sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90         stubs.api._rootDesc["revision"] = "20210628"
  91
  92         stubs.api.users().current().execute.return_value = {
  93             "uuid": stubs.fake_user_uuid,
  94         }
  95         stubs.api.collections().list().execute.return_value = {"items": []}
  96         stubs.api.containers().current().execute.return_value = {
  97             "uuid": stubs.fake_container_uuid,
  98         }
  99         stubs.api.config()["StorageClasses"].items.return_value = {
 100             "default": {
 101                 "Default": True
 102             }
 103         }.items()
 104
 105         class CollectionExecute(object):
 106             def __init__(self, exe):
 107                 self.exe = exe
 108             def execute(self, num_retries=None):
 109                 return self.exe
 110
 111         def collection_createstub(created_collections, body, ensure_unique_name=None):
 112             mt = body["manifest_text"].encode('utf-8')
 113             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 114             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 115             created_collections[uuid] = {
 116                 "uuid": uuid,
 117                 "portable_data_hash": pdh,
 118                 "manifest_text": mt.decode('utf-8')
 119             }
 120             return CollectionExecute(created_collections[uuid])
 121
 122         def collection_getstub(created_collections, uuid):
 123             for v in viewvalues(created_collections):
 124                 if uuid in (v["uuid"], v["portable_data_hash"]):
 125                     return CollectionExecute(v)
 126
 127         created_collections = {
 128             "99999999999999999999999999999998+99": {
 129                 "uuid": "",
 130                 "portable_data_hash": "99999999999999999999999999999998+99",
 131                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 132             },
 133             "99999999999999999999999999999997+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999997+99",
 136                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999994+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999994+99",
 141                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 142             },
 143             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 144                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 145                 "portable_data_hash": "999999999999999999999999999999d3+99",
 146                 "manifest_text": ""
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 150                 "portable_data_hash": "999999999999999999999999999999d4+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 155                 "portable_data_hash": "999999999999999999999999999999d5+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 160                 "portable_data_hash": "999999999999999999999999999999d6+99",
 161                 "manifest_text": ""
 162             }
 163         }
 164         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 165         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 166
 167         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 168         stubs.api.jobs().create().execute.return_value = {
 169             "uuid": stubs.expect_job_uuid,
 170             "state": "Queued",
 171         }
 172
 173         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 174         stubs.api.container_requests().create().execute.return_value = {
 175             "uuid": stubs.expect_container_request_uuid,
 176             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 177             "state": "Queued"
 178         }
 179
 180         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 181         stubs.api.pipeline_templates().create().execute.return_value = {
 182             "uuid": stubs.expect_pipeline_template_uuid,
 183         }
 184         stubs.expect_job_spec = {
 185             'runtime_constraints': {
 186                 'docker_image': '999999999999999999999999999999d3+99',
 187                 'min_ram_mb_per_node': 1024
 188             },
 189             'script_parameters': {
 190                 'x': {
 191                     'basename': 'blorp.txt',
 192                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 193                     'class': 'File'
 194                 },
 195                 'y': {
 196                     'basename': '99999999999999999999999999999998+99',
 197                     'location': 'keep:99999999999999999999999999999998+99',
 198                     'class': 'Directory'
 199                 },
 200                 'z': {
 201                     'basename': 'anonymous',
 202                     "listing": [{
 203                         "basename": "renamed.txt",
 204                         "class": "File",
 205                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 206                         "size": 0
 207                     }],
 208                     'class': 'Directory'
 209                 },
 210                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 211             },
 212             'repository': 'arvados',
 213             'script_version': 'master',
 214             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 215             'script': 'cwl-runner'
 216         }
 217         stubs.pipeline_component = stubs.expect_job_spec.copy()
 218         stubs.expect_pipeline_instance = {
 219             'name': 'submit_wf.cwl',
 220             'state': 'RunningOnServer',
 221             'owner_uuid': None,
 222             "components": {
 223                 "cwl-runner": {
 224                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 225                     'script_parameters': {
 226                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 227                         'x': {"value": {
 228                             'basename': 'blorp.txt',
 229                             'class': 'File',
 230                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 231                             "size": 16
 232                         }},
 233                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 234                               'listing': [
 235                                   {
 236                                       'basename': 'renamed.txt',
 237                                       'class': 'File', 'location':
 238                                       'keep:99999999999999999999999999999998+99/file1.txt',
 239                                       'size': 0
 240                                   }
 241                               ]}},
 242                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 243                         'arv:debug': True,
 244                         'arv:enable_reuse': True,
 245                         'arv:on_error': 'continue'
 246                     },
 247                     'repository': 'arvados',
 248                     'script_version': 'master',
 249                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 250                     'script': 'cwl-runner',
 251                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 252                 }
 253             }
 254         }
 255         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 256         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 257         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 258         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 259         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 260             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 261             "state": "Queued"
 262         }
 263         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 264         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 265
 266         with open("tests/wf/submit_wf_packed.cwl") as f:
 267             expect_packed_workflow = yaml.round_trip_load(f)
 268
 269         stubs.expect_container_spec = {
 270             'priority': 500,
 271             'mounts': {
 272                 '/var/spool/cwl': {
 273                     'writable': True,
 274                     'kind': 'collection'
 275                 },
 276                 '/var/lib/cwl/workflow.json': {
 277                     'content': expect_packed_workflow,
 278                     'kind': 'json'
 279                 },
 280                 'stdout': {
 281                     'path': '/var/spool/cwl/cwl.output.json',
 282                     'kind': 'file'
 283                 },
 284                 '/var/lib/cwl/cwl.input.json': {
 285                     'kind': 'json',
 286                     'content': {
 287                         'y': {
 288                             'basename': '99999999999999999999999999999998+99',
 289                             'location': 'keep:99999999999999999999999999999998+99',
 290                             'class': 'Directory'},
 291                         'x': {
 292                             'basename': u'blorp.txt',
 293                             'class': 'File',
 294                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 295                             "size": 16
 296                         },
 297                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 298                             {'basename': 'renamed.txt',
 299                              'class': 'File',
 300                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 301                              'size': 0
 302                             }
 303                         ]}
 304                     },
 305                     'kind': 'json'
 306                 }
 307             },
 308             'secret_mounts': {},
 309             'state': 'Committed',
 310             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 311                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 312                         '--eval-timeout=20', '--thread-count=0',
 313                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 314                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 315             'name': 'submit_wf.cwl',
 316             'container_image': '999999999999999999999999999999d3+99',
 317             'output_path': '/var/spool/cwl',
 318             'cwd': '/var/spool/cwl',
 319             'runtime_constraints': {
 320                 'API': True,
 321                 'vcpus': 1,
 322                 'ram': (1024+256)*1024*1024
 323             },
 324             'use_existing': False,
 325             'properties': {},
 326             'secret_mounts': {}
 327         }
 328
 329         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 330         stubs.api.workflows().create().execute.return_value = {
 331             "uuid": stubs.expect_workflow_uuid,
 332         }
 333         def update_mock(**kwargs):
 334             stubs.updated_uuid = kwargs.get('uuid')
 335             return mock.DEFAULT
 336         stubs.api.workflows().update.side_effect = update_mock
 337         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 338             "uuid": stubs.updated_uuid,
 339         }
 340
 341         return func(self, stubs, *args, **kwargs)
 342     return wrapped
 343
 344
 345 class TestSubmit(unittest.TestCase):
 346
 347     def setUp(self):
 348         cwltool.process._names = set()
 349         arvados_cwl.arvdocker.arv_docker_clear_cache()
 350
 351     def tearDown(self):
 352         root_logger = logging.getLogger('')
 353
 354         # Remove existing RuntimeStatusLoggingHandlers if they exist
 355         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 356         root_logger.handlers = handlers
 357
 358     @mock.patch("time.sleep")
 359     @stubs
 360     def test_submit_invalid_runner_ram(self, stubs, tm):
 361         exited = arvados_cwl.main(
 362             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 363              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 364             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 365         self.assertEqual(exited, 1)
 366
 367
 368     @stubs
 369     def test_submit_container(self, stubs):
 370         exited = arvados_cwl.main(
 371             ["--submit", "--no-wait", "--api=containers", "--debug",
 372                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 373             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 374
 375         stubs.api.collections().create.assert_has_calls([
 376             mock.call(body=JsonDiffMatcher({
 377                 'manifest_text':
 378                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 379                 'replication_desired': None,
 380                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 381             }), ensure_unique_name=False),
 382             mock.call(body=JsonDiffMatcher({
 383                 'manifest_text':
 384                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 385                 'replication_desired': None,
 386                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 387             }), ensure_unique_name=False),
 388             ])
 389
 390         expect_container = copy.deepcopy(stubs.expect_container_spec)
 391         stubs.api.container_requests().create.assert_called_with(
 392             body=JsonDiffMatcher(expect_container))
 393         self.assertEqual(stubs.capture_stdout.getvalue(),
 394                          stubs.expect_container_request_uuid + '\n')
 395         self.assertEqual(exited, 0)
 396
 397
 398     @stubs
 399     def test_submit_container_tool(self, stubs):
 400         # test for issue #16139
 401         exited = arvados_cwl.main(
 402             ["--submit", "--no-wait", "--api=containers", "--debug",
 403                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 404             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 405
 406         self.assertEqual(stubs.capture_stdout.getvalue(),
 407                          stubs.expect_container_request_uuid + '\n')
 408         self.assertEqual(exited, 0)
 409
 410     @stubs
 411     def test_submit_container_no_reuse(self, stubs):
 412         exited = arvados_cwl.main(
 413             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 414                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 415             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 416
 417         expect_container = copy.deepcopy(stubs.expect_container_spec)
 418         expect_container["command"] = [
 419             'arvados-cwl-runner', '--local', '--api=containers',
 420             '--no-log-timestamps', '--disable-validate', '--disable-color',
 421             '--eval-timeout=20', '--thread-count=0',
 422             '--disable-reuse', "--collection-cache-size=256",
 423             '--debug', '--on-error=continue',
 424             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 425         expect_container["use_existing"] = False
 426
 427         stubs.api.container_requests().create.assert_called_with(
 428             body=JsonDiffMatcher(expect_container))
 429         self.assertEqual(stubs.capture_stdout.getvalue(),
 430                          stubs.expect_container_request_uuid + '\n')
 431         self.assertEqual(exited, 0)
 432
 433     @stubs
 434     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 435         exited = arvados_cwl.main(
 436             ["--submit", "--no-wait", "--api=containers", "--debug",
 437              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 438             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 439         self.assertEqual(exited, 0)
 440
 441         expect_container = copy.deepcopy(stubs.expect_container_spec)
 442         expect_container["command"] = [
 443             'arvados-cwl-runner', '--local', '--api=containers',
 444             '--no-log-timestamps', '--disable-validate', '--disable-color',
 445             '--eval-timeout=20', '--thread-count=0',
 446             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 447             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 448         expect_container["use_existing"] = False
 449         expect_container["name"] = "submit_wf_no_reuse.cwl"
 450         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 451             {
 452                 "class": "http://arvados.org/cwl#ReuseRequirement",
 453                 "enableReuse": False,
 454             },
 455         ]
 456         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 457             "arv": "http://arvados.org/cwl#",
 458             "cwltool": "http://commonwl.org/cwltool#"
 459         }
 460
 461         stubs.api.container_requests().create.assert_called_with(
 462             body=JsonDiffMatcher(expect_container))
 463         self.assertEqual(stubs.capture_stdout.getvalue(),
 464                          stubs.expect_container_request_uuid + '\n')
 465
 466
 467     @stubs
 468     def test_submit_container_on_error(self, stubs):
 469         exited = arvados_cwl.main(
 470             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 471                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 472             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 473
 474         expect_container = copy.deepcopy(stubs.expect_container_spec)
 475         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 476                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 477                                        '--eval-timeout=20', '--thread-count=0',
 478                                        '--enable-reuse', "--collection-cache-size=256",
 479                                        '--debug', '--on-error=stop',
 480                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 481
 482         stubs.api.container_requests().create.assert_called_with(
 483             body=JsonDiffMatcher(expect_container))
 484         self.assertEqual(stubs.capture_stdout.getvalue(),
 485                          stubs.expect_container_request_uuid + '\n')
 486         self.assertEqual(exited, 0)
 487
 488     @stubs
 489     def test_submit_container_output_name(self, stubs):
 490         output_name = "test_output_name"
 491
 492         exited = arvados_cwl.main(
 493             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 494                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 495             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 496
 497         expect_container = copy.deepcopy(stubs.expect_container_spec)
 498         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 499                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 500                                        '--eval-timeout=20', '--thread-count=0',
 501                                        '--enable-reuse', "--collection-cache-size=256",
 502                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 503                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 504         expect_container["output_name"] = output_name
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(stubs.capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510         self.assertEqual(exited, 0)
 511
 512     @stubs
 513     def test_submit_storage_classes(self, stubs):
 514         exited = arvados_cwl.main(
 515             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 516                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 517             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 518
 519         expect_container = copy.deepcopy(stubs.expect_container_spec)
 520         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 521                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 522                                        '--eval-timeout=20', '--thread-count=0',
 523                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 524                                        "--storage-classes=foo", '--on-error=continue',
 525                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 526
 527         stubs.api.container_requests().create.assert_called_with(
 528             body=JsonDiffMatcher(expect_container))
 529         self.assertEqual(stubs.capture_stdout.getvalue(),
 530                          stubs.expect_container_request_uuid + '\n')
 531         self.assertEqual(exited, 0)
 532
 533     @stubs
 534     def test_submit_multiple_storage_classes(self, stubs):
 535         exited = arvados_cwl.main(
 536             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 537                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 538             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 539
 540         expect_container = copy.deepcopy(stubs.expect_container_spec)
 541         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 542                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 543                                        '--eval-timeout=20', '--thread-count=0',
 544                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 545                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 546                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 547
 548         stubs.api.container_requests().create.assert_called_with(
 549             body=JsonDiffMatcher(expect_container))
 550         self.assertEqual(stubs.capture_stdout.getvalue(),
 551                          stubs.expect_container_request_uuid + '\n')
 552         self.assertEqual(exited, 0)
 553
 554     @mock.patch("cwltool.task_queue.TaskQueue")
 555     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 556     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 557     @stubs
 558     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 559         final_output_c = arvados.collection.Collection()
 560         make_output.return_value = ({},final_output_c)
 561
 562         def set_final_output(job_order, output_callback, runtimeContext):
 563             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 564             return []
 565         job.side_effect = set_final_output
 566
 567         exited = arvados_cwl.main(
 568             ["--debug", "--local", "--storage-classes=foo",
 569                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 570             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 571
 572         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 573         self.assertEqual(exited, 0)
 574
 575     @mock.patch("cwltool.task_queue.TaskQueue")
 576     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 577     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 578     @stubs
 579     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 580         final_output_c = arvados.collection.Collection()
 581         make_output.return_value = ({},final_output_c)
 582         stubs.api.config().get.return_value = {"default": {"Default": True}}
 583
 584         def set_final_output(job_order, output_callback, runtimeContext):
 585             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 586             return []
 587         job.side_effect = set_final_output
 588
 589         exited = arvados_cwl.main(
 590             ["--debug", "--local",
 591                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 592             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 593
 594         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 595         self.assertEqual(exited, 0)
 596
 597     @mock.patch("cwltool.task_queue.TaskQueue")
 598     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 599     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 600     @stubs
 601     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 602         final_output_c = arvados.collection.Collection()
 603         make_output.return_value = ({},final_output_c)
 604
 605         def set_final_output(job_order, output_callback, runtimeContext):
 606             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 607             return []
 608         job.side_effect = set_final_output
 609
 610         exited = arvados_cwl.main(
 611             ["--debug", "--local",
 612                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 613             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 614
 615         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 616         self.assertEqual(exited, 0)
 617
 618     @stubs
 619     def test_submit_container_output_ttl(self, stubs):
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 627                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 628                                        '--eval-timeout=20', '--thread-count=0',
 629                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 630                                        '--on-error=continue',
 631                                        "--intermediate-output-ttl=3600",
 632                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 633
 634         stubs.api.container_requests().create.assert_called_with(
 635             body=JsonDiffMatcher(expect_container))
 636         self.assertEqual(stubs.capture_stdout.getvalue(),
 637                          stubs.expect_container_request_uuid + '\n')
 638         self.assertEqual(exited, 0)
 639
 640     @stubs
 641     def test_submit_container_trash_intermediate(self, stubs):
 642         exited = arvados_cwl.main(
 643             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 644                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 645             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 646
 647
 648         expect_container = copy.deepcopy(stubs.expect_container_spec)
 649         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 650                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 651                                        '--eval-timeout=20', '--thread-count=0',
 652                                        '--enable-reuse', "--collection-cache-size=256",
 653                                        '--debug', '--on-error=continue',
 654                                        "--trash-intermediate",
 655                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 656
 657         stubs.api.container_requests().create.assert_called_with(
 658             body=JsonDiffMatcher(expect_container))
 659         self.assertEqual(stubs.capture_stdout.getvalue(),
 660                          stubs.expect_container_request_uuid + '\n')
 661         self.assertEqual(exited, 0)
 662
 663     @stubs
 664     def test_submit_container_output_tags(self, stubs):
 665         output_tags = "tag0,tag1,tag2"
 666
 667         exited = arvados_cwl.main(
 668             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 669                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 670             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 671
 672         expect_container = copy.deepcopy(stubs.expect_container_spec)
 673         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 674                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 675                                        '--eval-timeout=20', '--thread-count=0',
 676                                        '--enable-reuse', "--collection-cache-size=256",
 677                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 678                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 679
 680         stubs.api.container_requests().create.assert_called_with(
 681             body=JsonDiffMatcher(expect_container))
 682         self.assertEqual(stubs.capture_stdout.getvalue(),
 683                          stubs.expect_container_request_uuid + '\n')
 684         self.assertEqual(exited, 0)
 685
 686     @stubs
 687     def test_submit_container_runner_ram(self, stubs):
 688         exited = arvados_cwl.main(
 689             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 690                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 691             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 692
 693         expect_container = copy.deepcopy(stubs.expect_container_spec)
 694         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 695
 696         stubs.api.container_requests().create.assert_called_with(
 697             body=JsonDiffMatcher(expect_container))
 698         self.assertEqual(stubs.capture_stdout.getvalue(),
 699                          stubs.expect_container_request_uuid + '\n')
 700         self.assertEqual(exited, 0)
 701
 702     @mock.patch("arvados.collection.CollectionReader")
 703     @mock.patch("time.sleep")
 704     @stubs
 705     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 706         collectionReader().exists.return_value = True
 707         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 708         exited = arvados_cwl.main(
 709             ["--submit", "--no-wait", "--api=containers", "--debug",
 710              "tests/wf/submit_keepref_wf.cwl"],
 711             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 712         self.assertEqual(exited, 0)
 713
 714     @mock.patch("arvados.collection.CollectionReader")
 715     @mock.patch("time.sleep")
 716     @stubs
 717     def test_submit_keepref(self, stubs, tm, reader):
 718         with open("tests/wf/expect_arvworkflow.cwl") as f:
 719             reader().open().__enter__().read.return_value = f.read()
 720
 721         exited = arvados_cwl.main(
 722             ["--submit", "--no-wait", "--api=containers", "--debug",
 723              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 724             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 725
 726         expect_container = {
 727             'priority': 500,
 728             'mounts': {
 729                 '/var/spool/cwl': {
 730                     'writable': True,
 731                     'kind': 'collection'
 732                 },
 733                 'stdout': {
 734                     'path': '/var/spool/cwl/cwl.output.json',
 735                     'kind': 'file'
 736                 },
 737                 '/var/lib/cwl/workflow': {
 738                     'portable_data_hash': '99999999999999999999999999999994+99',
 739                     'kind': 'collection'
 740                 },
 741                 '/var/lib/cwl/cwl.input.json': {
 742                     'content': {
 743                         'x': 'XxX'
 744                     },
 745                     'kind': 'json'
 746                 }
 747             }, 'state': 'Committed',
 748             'output_path': '/var/spool/cwl',
 749             'name': 'expect_arvworkflow.cwl#main',
 750             'container_image': '999999999999999999999999999999d3+99',
 751             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 752                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 753                         '--eval-timeout=20', '--thread-count=0',
 754                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 755                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 756             'cwd': '/var/spool/cwl',
 757             'runtime_constraints': {
 758                 'API': True,
 759                 'vcpus': 1,
 760                 'ram': 1342177280
 761             },
 762             'use_existing': False,
 763             'properties': {},
 764             'secret_mounts': {}
 765         }
 766
 767         stubs.api.container_requests().create.assert_called_with(
 768             body=JsonDiffMatcher(expect_container))
 769         self.assertEqual(stubs.capture_stdout.getvalue(),
 770                          stubs.expect_container_request_uuid + '\n')
 771         self.assertEqual(exited, 0)
 772
 773     @mock.patch("time.sleep")
 774     @stubs
 775     def test_submit_arvworkflow(self, stubs, tm):
 776         with open("tests/wf/expect_arvworkflow.cwl") as f:
 777             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 778
 779         exited = arvados_cwl.main(
 780             ["--submit", "--no-wait", "--api=containers", "--debug",
 781              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 782             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 783
 784         expect_container = {
 785             'priority': 500,
 786             'mounts': {
 787                 '/var/spool/cwl': {
 788                     'writable': True,
 789                     'kind': 'collection'
 790                 },
 791                 'stdout': {
 792                     'path': '/var/spool/cwl/cwl.output.json',
 793                     'kind': 'file'
 794                 },
 795                 '/var/lib/cwl/workflow.json': {
 796                     'kind': 'json',
 797                     'content': {
 798                         'cwlVersion': 'v1.0',
 799                         '$graph': [
 800                             {
 801                                 'id': '#main',
 802                                 'inputs': [
 803                                     {'type': 'string', 'id': '#main/x'}
 804                                 ],
 805                                 'steps': [
 806                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 807                                      'run': '#submit_tool.cwl',
 808                                      'id': '#main/step1',
 809                                      'out': []}
 810                                 ],
 811                                 'class': 'Workflow',
 812                                 'outputs': []
 813                             },
 814                             {
 815                                 'inputs': [
 816                                     {
 817                                         'inputBinding': {'position': 1},
 818                                         'type': 'string',
 819                                         'id': '#submit_tool.cwl/x'}
 820                                 ],
 821                                 'requirements': [
 822                                     {
 823                                         'dockerPull': 'debian:buster-slim',
 824                                         'class': 'DockerRequirement',
 825                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 826                                     }
 827                                 ],
 828                                 'id': '#submit_tool.cwl',
 829                                 'outputs': [],
 830                                 'baseCommand': 'cat',
 831                                 'class': 'CommandLineTool'
 832                             }
 833                         ]
 834                     }
 835                 },
 836                 '/var/lib/cwl/cwl.input.json': {
 837                     'content': {
 838                         'x': 'XxX'
 839                     },
 840                     'kind': 'json'
 841                 }
 842             }, 'state': 'Committed',
 843             'output_path': '/var/spool/cwl',
 844             'name': 'a test workflow',
 845             'container_image': "999999999999999999999999999999d3+99",
 846             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 847                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 848                         '--eval-timeout=20', '--thread-count=0',
 849                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 850                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 851             'cwd': '/var/spool/cwl',
 852             'runtime_constraints': {
 853                 'API': True,
 854                 'vcpus': 1,
 855                 'ram': 1342177280
 856             },
 857             'use_existing': False,
 858             'properties': {
 859                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 860             },
 861             'secret_mounts': {}
 862         }
 863
 864         stubs.api.container_requests().create.assert_called_with(
 865             body=JsonDiffMatcher(expect_container))
 866         self.assertEqual(stubs.capture_stdout.getvalue(),
 867                          stubs.expect_container_request_uuid + '\n')
 868         self.assertEqual(exited, 0)
 869
 870     @stubs
 871     def test_submit_container_name(self, stubs):
 872         exited = arvados_cwl.main(
 873             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 874                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 875             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 876
 877         expect_container = copy.deepcopy(stubs.expect_container_spec)
 878         expect_container["name"] = "hello container 123"
 879
 880         stubs.api.container_requests().create.assert_called_with(
 881             body=JsonDiffMatcher(expect_container))
 882         self.assertEqual(stubs.capture_stdout.getvalue(),
 883                          stubs.expect_container_request_uuid + '\n')
 884         self.assertEqual(exited, 0)
 885
 886     @stubs
 887     def test_submit_missing_input(self, stubs):
 888         exited = arvados_cwl.main(
 889             ["--submit", "--no-wait", "--api=containers", "--debug",
 890              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 891             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 892         self.assertEqual(exited, 0)
 893
 894         exited = arvados_cwl.main(
 895             ["--submit", "--no-wait", "--api=containers", "--debug",
 896              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 897             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 898         self.assertEqual(exited, 1)
 899
 900     @stubs
 901     def test_submit_container_project(self, stubs):
 902         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 903         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 904         exited = arvados_cwl.main(
 905             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 906                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 907             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 908
 909         expect_container = copy.deepcopy(stubs.expect_container_spec)
 910         expect_container["owner_uuid"] = project_uuid
 911         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 912                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 913                                        "--eval-timeout=20", "--thread-count=0",
 914                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 915                                        '--on-error=continue',
 916                                        '--project-uuid='+project_uuid,
 917                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 918
 919         stubs.api.container_requests().create.assert_called_with(
 920             body=JsonDiffMatcher(expect_container))
 921         self.assertEqual(stubs.capture_stdout.getvalue(),
 922                          stubs.expect_container_request_uuid + '\n')
 923         self.assertEqual(exited, 0)
 924
 925     @stubs
 926     def test_submit_container_eval_timeout(self, stubs):
 927         exited = arvados_cwl.main(
 928             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 929                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 930             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 931
 932         expect_container = copy.deepcopy(stubs.expect_container_spec)
 933         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 934                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 935                                        '--eval-timeout=60.0', '--thread-count=0',
 936                                        '--enable-reuse', "--collection-cache-size=256",
 937                                        '--debug', '--on-error=continue',
 938                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 939
 940         stubs.api.container_requests().create.assert_called_with(
 941             body=JsonDiffMatcher(expect_container))
 942         self.assertEqual(stubs.capture_stdout.getvalue(),
 943                          stubs.expect_container_request_uuid + '\n')
 944         self.assertEqual(exited, 0)
 945
 946     @stubs
 947     def test_submit_container_collection_cache(self, stubs):
 948         exited = arvados_cwl.main(
 949             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 950                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 951             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 952
 953         expect_container = copy.deepcopy(stubs.expect_container_spec)
 954         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 955                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 956                                        '--eval-timeout=20', '--thread-count=0',
 957                                        '--enable-reuse', "--collection-cache-size=500",
 958                                        '--debug', '--on-error=continue',
 959                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 960         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 961
 962         stubs.api.container_requests().create.assert_called_with(
 963             body=JsonDiffMatcher(expect_container))
 964         self.assertEqual(stubs.capture_stdout.getvalue(),
 965                          stubs.expect_container_request_uuid + '\n')
 966         self.assertEqual(exited, 0)
 967
 968     @stubs
 969     def test_submit_container_thread_count(self, stubs):
 970         exited = arvados_cwl.main(
 971             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 972                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 973             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 974
 975         expect_container = copy.deepcopy(stubs.expect_container_spec)
 976         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 977                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 978                                        '--eval-timeout=20', '--thread-count=20',
 979                                        '--enable-reuse', "--collection-cache-size=256",
 980                                        '--debug', '--on-error=continue',
 981                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 982
 983         stubs.api.container_requests().create.assert_called_with(
 984             body=JsonDiffMatcher(expect_container))
 985         self.assertEqual(stubs.capture_stdout.getvalue(),
 986                          stubs.expect_container_request_uuid + '\n')
 987         self.assertEqual(exited, 0)
 988
 989     @stubs
 990     def test_submit_container_runner_image(self, stubs):
 991         exited = arvados_cwl.main(
 992             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 993                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 995
 996         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 997
 998         expect_container = copy.deepcopy(stubs.expect_container_spec)
 999         stubs.api.container_requests().create.assert_called_with(
1000             body=JsonDiffMatcher(expect_container))
1001         self.assertEqual(stubs.capture_stdout.getvalue(),
1002                          stubs.expect_container_request_uuid + '\n')
1003         self.assertEqual(exited, 0)
1004
1005     @stubs
1006     def test_submit_priority(self, stubs):
1007         exited = arvados_cwl.main(
1008             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1009                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1010             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1011
1012         stubs.expect_container_spec["priority"] = 669
1013
1014         expect_container = copy.deepcopy(stubs.expect_container_spec)
1015         stubs.api.container_requests().create.assert_called_with(
1016             body=JsonDiffMatcher(expect_container))
1017         self.assertEqual(stubs.capture_stdout.getvalue(),
1018                          stubs.expect_container_request_uuid + '\n')
1019         self.assertEqual(exited, 0)
1020
1021     @stubs
1022     def test_submit_wf_runner_resources(self, stubs):
1023         exited = arvados_cwl.main(
1024             ["--submit", "--no-wait", "--api=containers", "--debug",
1025                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1026             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1027
1028         expect_container = copy.deepcopy(stubs.expect_container_spec)
1029         expect_container["runtime_constraints"] = {
1030             "API": True,
1031             "vcpus": 2,
1032             "ram": (2000+512) * 2**20
1033         }
1034         expect_container["name"] = "submit_wf_runner_resources.cwl"
1035         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1036             {
1037                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1038                 "coresMin": 2,
1039                 "ramMin": 2000,
1040                 "keep_cache": 512
1041             }
1042         ]
1043         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1044             "arv": "http://arvados.org/cwl#",
1045         }
1046         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1047                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1048                         '--eval-timeout=20', '--thread-count=0',
1049                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1050                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1051
1052         stubs.api.container_requests().create.assert_called_with(
1053             body=JsonDiffMatcher(expect_container))
1054         self.assertEqual(stubs.capture_stdout.getvalue(),
1055                          stubs.expect_container_request_uuid + '\n')
1056         self.assertEqual(exited, 0)
1057
1058     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1059     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1060     @mock.patch("arvados.api")
1061     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1062         arvados_cwl.arvdocker.arv_docker_clear_cache()
1063
1064         arvrunner = mock.MagicMock()
1065         arvrunner.project_uuid = ""
1066         api.return_value = mock.MagicMock()
1067         arvrunner.api = api.return_value
1068         arvrunner.runtimeContext.match_local_docker = False
1069         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1070                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1071                                                                         "link_class": "docker_image_repo+tag",
1072                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1073                                                                         "owner_uuid": "",
1074                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1075                                                             {"items": [{"created_at": "",
1076                                                                         "head_uuid": "",
1077                                                                         "link_class": "docker_image_hash",
1078                                                                         "name": "123456",
1079                                                                         "owner_uuid": "",
1080                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1081         )
1082         find_one_image_hash.return_value = "123456"
1083
1084         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1085                                                                               "owner_uuid": "",
1086                                                                               "manifest_text": "",
1087                                                                               "properties": ""
1088                                                                           }], "items_available": 1, "offset": 0},)
1089         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1090         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1091                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1092         self.assertEqual("9999999999999999999999999999999b+99",
1093                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1094
1095
1096     @stubs
1097     def test_submit_secrets(self, stubs):
1098         exited = arvados_cwl.main(
1099             ["--submit", "--no-wait", "--api=containers", "--debug",
1100                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1101             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1102
1103         expect_container = {
1104             "command": [
1105                 "arvados-cwl-runner",
1106                 "--local",
1107                 "--api=containers",
1108                 "--no-log-timestamps",
1109                 "--disable-validate",
1110                 "--disable-color",
1111                 "--eval-timeout=20",
1112                 '--thread-count=0',
1113                 "--enable-reuse",
1114                 "--collection-cache-size=256",
1115                 '--debug',
1116                 "--on-error=continue",
1117                 "/var/lib/cwl/workflow.json#main",
1118                 "/var/lib/cwl/cwl.input.json"
1119             ],
1120             "container_image": "999999999999999999999999999999d3+99",
1121             "cwd": "/var/spool/cwl",
1122             "mounts": {
1123                 "/var/lib/cwl/cwl.input.json": {
1124                     "content": {
1125                         "pw": {
1126                             "$include": "/secrets/s0"
1127                         }
1128                     },
1129                     "kind": "json"
1130                 },
1131                 "/var/lib/cwl/workflow.json": {
1132                     "content": {
1133                         "$graph": [
1134                             {
1135                                 "arguments": [
1136                                     "md5sum",
1137                                     "example.conf"
1138                                 ],
1139                                 "class": "CommandLineTool",
1140                                 "hints": [
1141                                     {
1142                                         "class": "http://commonwl.org/cwltool#Secrets",
1143                                         "secrets": [
1144                                             "#secret_job.cwl/pw"
1145                                         ]
1146                                     }
1147                                 ],
1148                                 "id": "#secret_job.cwl",
1149                                 "inputs": [
1150                                     {
1151                                         "id": "#secret_job.cwl/pw",
1152                                         "type": "string"
1153                                     }
1154                                 ],
1155                                 "outputs": [
1156                                     {
1157                                         "id": "#secret_job.cwl/out",
1158                                         "type": "File",
1159                                         "outputBinding": {
1160                                               "glob": "hashed_example.txt"
1161                                         }
1162                                     }
1163                                 ],
1164                                 "stdout": "hashed_example.txt",
1165                                 "requirements": [
1166                                     {
1167                                         "class": "InitialWorkDirRequirement",
1168                                         "listing": [
1169                                             {
1170                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1171                                                 "entryname": "example.conf"
1172                                             }
1173                                         ]
1174                                     }
1175                                 ]
1176                             },
1177                             {
1178                                 "class": "Workflow",
1179                                 "hints": [
1180                                     {
1181                                         "class": "DockerRequirement",
1182                                         "dockerPull": "debian:buster-slim",
1183                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1184                                     },
1185                                     {
1186                                         "class": "http://commonwl.org/cwltool#Secrets",
1187                                         "secrets": [
1188                                             "#main/pw"
1189                                         ]
1190                                     }
1191                                 ],
1192                                 "id": "#main",
1193                                 "inputs": [
1194                                     {
1195                                         "id": "#main/pw",
1196                                         "type": "string"
1197                                     }
1198                                 ],
1199                                 "outputs": [
1200                                     {
1201                                         "id": "#main/out",
1202                                         "outputSource": "#main/step1/out",
1203                                         "type": "File"
1204                                     }
1205                                 ],
1206                                 "steps": [
1207                                     {
1208                                         "id": "#main/step1",
1209                                         "in": [
1210                                             {
1211                                                 "id": "#main/step1/pw",
1212                                                 "source": "#main/pw"
1213                                             }
1214                                         ],
1215                                         "out": [
1216                                             "#main/step1/out"
1217                                         ],
1218                                         "run": "#secret_job.cwl"
1219                                     }
1220                                 ]
1221                             }
1222                         ],
1223                         "$namespaces": {
1224                             "cwltool": "http://commonwl.org/cwltool#"
1225                         },
1226                         "cwlVersion": "v1.0"
1227                     },
1228                     "kind": "json"
1229                 },
1230                 "/var/spool/cwl": {
1231                     "kind": "collection",
1232                     "writable": True
1233                 },
1234                 "stdout": {
1235                     "kind": "file",
1236                     "path": "/var/spool/cwl/cwl.output.json"
1237                 }
1238             },
1239             "name": "secret_wf.cwl",
1240             "output_path": "/var/spool/cwl",
1241             "priority": 500,
1242             "properties": {},
1243             "runtime_constraints": {
1244                 "API": True,
1245                 "ram": 1342177280,
1246                 "vcpus": 1
1247             },
1248             "secret_mounts": {
1249                 "/secrets/s0": {
1250                     "content": "blorp",
1251                     "kind": "text"
1252                 }
1253             },
1254             "state": "Committed",
1255             "use_existing": False
1256         }
1257
1258         stubs.api.container_requests().create.assert_called_with(
1259             body=JsonDiffMatcher(expect_container))
1260         self.assertEqual(stubs.capture_stdout.getvalue(),
1261                          stubs.expect_container_request_uuid + '\n')
1262         self.assertEqual(exited, 0)
1263
1264     @stubs
1265     def test_submit_request_uuid(self, stubs):
1266         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1267         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1268
1269         stubs.api.container_requests().update().execute.return_value = {
1270             "uuid": stubs.expect_container_request_uuid,
1271             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1272             "state": "Queued"
1273         }
1274
1275         exited = arvados_cwl.main(
1276             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1277                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1278             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1279
1280         stubs.api.container_requests().update.assert_called_with(
1281             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1282         self.assertEqual(stubs.capture_stdout.getvalue(),
1283                          stubs.expect_container_request_uuid + '\n')
1284         self.assertEqual(exited, 0)
1285
1286     @stubs
1287     def test_submit_container_cluster_id(self, stubs):
1288         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1289
1290         exited = arvados_cwl.main(
1291             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1292                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1293             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1294
1295         expect_container = copy.deepcopy(stubs.expect_container_spec)
1296
1297         stubs.api.container_requests().create.assert_called_with(
1298             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1299         self.assertEqual(stubs.capture_stdout.getvalue(),
1300                          stubs.expect_container_request_uuid + '\n')
1301         self.assertEqual(exited, 0)
1302
1303     @stubs
1304     def test_submit_validate_cluster_id(self, stubs):
1305         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1306         exited = arvados_cwl.main(
1307             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1308              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1309             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1310         self.assertEqual(exited, 1)
1311
1312     @stubs
1313     def test_submit_validate_project_uuid(self, stubs):
1314         # Fails with bad cluster prefix
1315         exited = arvados_cwl.main(
1316             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1317              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1318             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1319         self.assertEqual(exited, 1)
1320
1321         # Project lookup fails
1322         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1323         exited = arvados_cwl.main(
1324             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1325              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1326             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1327         self.assertEqual(exited, 1)
1328
1329         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1330         exited = arvados_cwl.main(
1331             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1332              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1333             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1334         self.assertEqual(exited, 0)
1335
1336
1337     @mock.patch("arvados.collection.CollectionReader")
1338     @stubs
1339     def test_submit_uuid_inputs(self, stubs, collectionReader):
1340         collectionReader().exists.return_value = True
1341         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1342         def list_side_effect(**kwargs):
1343             m = mock.MagicMock()
1344             if "count" in kwargs:
1345                 m.execute.return_value = {"items": [
1346                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1347                 ]}
1348             else:
1349                 m.execute.return_value = {"items": []}
1350             return m
1351         stubs.api.collections().list.side_effect = list_side_effect
1352
1353         exited = arvados_cwl.main(
1354             ["--submit", "--no-wait", "--api=containers", "--debug",
1355                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1356             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1357
1358         expect_container = copy.deepcopy(stubs.expect_container_spec)
1359         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1360         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1361         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1362
1363         stubs.api.collections().list.assert_has_calls([
1364             mock.call(count='none',
1365                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1366                       select=['uuid', 'portable_data_hash'])])
1367         stubs.api.container_requests().create.assert_called_with(
1368             body=JsonDiffMatcher(expect_container))
1369         self.assertEqual(stubs.capture_stdout.getvalue(),
1370                          stubs.expect_container_request_uuid + '\n')
1371         self.assertEqual(exited, 0)
1372
1373     @stubs
1374     def test_submit_mismatched_uuid_inputs(self, stubs):
1375         def list_side_effect(**kwargs):
1376             m = mock.MagicMock()
1377             if "count" in kwargs:
1378                 m.execute.return_value = {"items": [
1379                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1380                 ]}
1381             else:
1382                 m.execute.return_value = {"items": []}
1383             return m
1384         stubs.api.collections().list.side_effect = list_side_effect
1385
1386         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1387             capture_stderr = StringIO()
1388             cwltool_logger = logging.getLogger('cwltool')
1389             stderr_logger = logging.StreamHandler(capture_stderr)
1390             cwltool_logger.addHandler(stderr_logger)
1391
1392             try:
1393                 exited = arvados_cwl.main(
1394                     ["--submit", "--no-wait", "--api=containers", "--debug",
1395                         "tests/wf/submit_wf.cwl", infile],
1396                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1397
1398                 self.assertEqual(exited, 1)
1399                 self.assertRegex(
1400                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1401                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1402             finally:
1403                 cwltool_logger.removeHandler(stderr_logger)
1404
1405     @mock.patch("arvados.collection.CollectionReader")
1406     @stubs
1407     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1408         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1409         capture_stderr = StringIO()
1410
1411         cwltool_logger = logging.getLogger('cwltool')
1412         stderr_logger = logging.StreamHandler(capture_stderr)
1413         cwltool_logger.addHandler(stderr_logger)
1414
1415         exited = arvados_cwl.main(
1416             ["--submit", "--no-wait", "--api=containers", "--debug",
1417                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1418             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1419
1420         try:
1421             self.assertEqual(exited, 1)
1422             self.assertRegex(
1423                 capture_stderr.getvalue(),
1424                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1425         finally:
1426             cwltool_logger.removeHandler(stderr_logger)
1427
1428     @stubs
1429     def test_submit_set_process_properties(self, stubs):
1430         exited = arvados_cwl.main(
1431             ["--submit", "--no-wait", "--api=containers", "--debug",
1432                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1433             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1434
1435         expect_container = copy.deepcopy(stubs.expect_container_spec)
1436         expect_container["name"] = "submit_wf_process_properties.cwl"
1437         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1438             {
1439                 "class": "http://arvados.org/cwl#ProcessProperties",
1440                 "processProperties": [
1441                     {"propertyName": "baz",
1442                      "propertyValue": "$(inputs.x.basename)"},
1443                     {"propertyName": "foo",
1444                      "propertyValue": "bar"},
1445                     {"propertyName": "quux",
1446                      "propertyValue": {
1447                          "q1": 1,
1448                          "q2": 2
1449                      }
1450                     }
1451                 ],
1452             }
1453         ]
1454         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1455             "arv": "http://arvados.org/cwl#"
1456         }
1457
1458         expect_container["properties"] = {
1459             "baz": "blorp.txt",
1460             "foo": "bar",
1461             "quux": {
1462                 "q1": 1,
1463                 "q2": 2
1464             }
1465         }
1466
1467         stubs.api.container_requests().create.assert_called_with(
1468             body=JsonDiffMatcher(expect_container))
1469         self.assertEqual(stubs.capture_stdout.getvalue(),
1470                          stubs.expect_container_request_uuid + '\n')
1471         self.assertEqual(exited, 0)
1472
1473
1474     @stubs
1475     def test_submit_enable_preemptible(self, stubs):
1476         exited = arvados_cwl.main(
1477             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1478                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1479             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1480
1481         expect_container = copy.deepcopy(stubs.expect_container_spec)
1482         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1483                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1484                         '--eval-timeout=20', '--thread-count=0',
1485                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1486                                        '--enable-preemptible',
1487                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1488
1489         stubs.api.container_requests().create.assert_called_with(
1490             body=JsonDiffMatcher(expect_container))
1491         self.assertEqual(stubs.capture_stdout.getvalue(),
1492                          stubs.expect_container_request_uuid + '\n')
1493         self.assertEqual(exited, 0)
1494
1495     @stubs
1496     def test_submit_disable_preemptible(self, stubs):
1497         exited = arvados_cwl.main(
1498             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1499                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1500             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1501
1502         expect_container = copy.deepcopy(stubs.expect_container_spec)
1503         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1504                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1505                         '--eval-timeout=20', '--thread-count=0',
1506                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1507                                        '--disable-preemptible',
1508                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1509
1510         stubs.api.container_requests().create.assert_called_with(
1511             body=JsonDiffMatcher(expect_container))
1512         self.assertEqual(stubs.capture_stdout.getvalue(),
1513                          stubs.expect_container_request_uuid + '\n')
1514         self.assertEqual(exited, 0)
1515
1516
1517 class TestCreateWorkflow(unittest.TestCase):
1518     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1519     expect_workflow = StripYAMLComments(
1520         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1521
1522     def setUp(self):
1523         cwltool.process._names = set()
1524         arvados_cwl.arvdocker.arv_docker_clear_cache()
1525
1526     def tearDown(self):
1527         root_logger = logging.getLogger('')
1528
1529         # Remove existing RuntimeStatusLoggingHandlers if they exist
1530         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1531         root_logger.handlers = handlers
1532
1533     @stubs
1534     def test_create(self, stubs):
1535         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1536         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1537
1538         exited = arvados_cwl.main(
1539             ["--create-workflow", "--debug",
1540              "--api=containers",
1541              "--project-uuid", project_uuid,
1542              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1543             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1544
1545         stubs.api.pipeline_templates().create.refute_called()
1546         stubs.api.container_requests().create.refute_called()
1547
1548         body = {
1549             "workflow": {
1550                 "owner_uuid": project_uuid,
1551                 "name": "submit_wf.cwl",
1552                 "description": "",
1553                 "definition": self.expect_workflow,
1554             }
1555         }
1556         stubs.api.workflows().create.assert_called_with(
1557             body=JsonDiffMatcher(body))
1558
1559         self.assertEqual(stubs.capture_stdout.getvalue(),
1560                          stubs.expect_workflow_uuid + '\n')
1561         self.assertEqual(exited, 0)
1562
1563     @stubs
1564     def test_create_name(self, stubs):
1565         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1566         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1567
1568         exited = arvados_cwl.main(
1569             ["--create-workflow", "--debug",
1570              "--api=containers",
1571              "--project-uuid", project_uuid,
1572              "--name", "testing 123",
1573              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1574             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1575
1576         stubs.api.pipeline_templates().create.refute_called()
1577         stubs.api.container_requests().create.refute_called()
1578
1579         body = {
1580             "workflow": {
1581                 "owner_uuid": project_uuid,
1582                 "name": "testing 123",
1583                 "description": "",
1584                 "definition": self.expect_workflow,
1585             }
1586         }
1587         stubs.api.workflows().create.assert_called_with(
1588             body=JsonDiffMatcher(body))
1589
1590         self.assertEqual(stubs.capture_stdout.getvalue(),
1591                          stubs.expect_workflow_uuid + '\n')
1592         self.assertEqual(exited, 0)
1593
1594
1595     @stubs
1596     def test_update(self, stubs):
1597         exited = arvados_cwl.main(
1598             ["--update-workflow", self.existing_workflow_uuid,
1599              "--debug",
1600              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1601             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1602
1603         body = {
1604             "workflow": {
1605                 "name": "submit_wf.cwl",
1606                 "description": "",
1607                 "definition": self.expect_workflow,
1608             }
1609         }
1610         stubs.api.workflows().update.assert_called_with(
1611             uuid=self.existing_workflow_uuid,
1612             body=JsonDiffMatcher(body))
1613         self.assertEqual(stubs.capture_stdout.getvalue(),
1614                          self.existing_workflow_uuid + '\n')
1615         self.assertEqual(exited, 0)
1616
1617     @stubs
1618     def test_update_name(self, stubs):
1619         exited = arvados_cwl.main(
1620             ["--update-workflow", self.existing_workflow_uuid,
1621              "--debug", "--name", "testing 123",
1622              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1623             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1624
1625         body = {
1626             "workflow": {
1627                 "name": "testing 123",
1628                 "description": "",
1629                 "definition": self.expect_workflow,
1630             }
1631         }
1632         stubs.api.workflows().update.assert_called_with(
1633             uuid=self.existing_workflow_uuid,
1634             body=JsonDiffMatcher(body))
1635         self.assertEqual(stubs.capture_stdout.getvalue(),
1636                          self.existing_workflow_uuid + '\n')
1637         self.assertEqual(exited, 0)
1638
1639     @stubs
1640     def test_create_collection_per_tool(self, stubs):
1641         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1642         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1643
1644         exited = arvados_cwl.main(
1645             ["--create-workflow", "--debug",
1646              "--api=containers",
1647              "--project-uuid", project_uuid,
1648              "tests/collection_per_tool/collection_per_tool.cwl"],
1649             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1650
1651         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1652         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1653
1654         body = {
1655             "workflow": {
1656                 "owner_uuid": project_uuid,
1657                 "name": "collection_per_tool.cwl",
1658                 "description": "",
1659                 "definition": expect_workflow,
1660             }
1661         }
1662         stubs.api.workflows().create.assert_called_with(
1663             body=JsonDiffMatcher(body))
1664
1665         self.assertEqual(stubs.capture_stdout.getvalue(),
1666                          stubs.expect_workflow_uuid + '\n')
1667         self.assertEqual(exited, 0)
1668
1669     @stubs
1670     def test_create_with_imports(self, stubs):
1671         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1672         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1673
1674         exited = arvados_cwl.main(
1675             ["--create-workflow", "--debug",
1676              "--api=containers",
1677              "--project-uuid", project_uuid,
1678              "tests/wf/feddemo/feddemo.cwl"],
1679             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1680
1681         stubs.api.pipeline_templates().create.refute_called()
1682         stubs.api.container_requests().create.refute_called()
1683
1684         self.assertEqual(stubs.capture_stdout.getvalue(),
1685                          stubs.expect_workflow_uuid + '\n')
1686         self.assertEqual(exited, 0)
1687
1688     @stubs
1689     def test_create_with_no_input(self, stubs):
1690         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1691         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1692
1693         exited = arvados_cwl.main(
1694             ["--create-workflow", "--debug",
1695              "--api=containers",
1696              "--project-uuid", project_uuid,
1697              "tests/wf/revsort/revsort.cwl"],
1698             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1699
1700         stubs.api.pipeline_templates().create.refute_called()
1701         stubs.api.container_requests().create.refute_called()
1702
1703         self.assertEqual(stubs.capture_stdout.getvalue(),
1704                          stubs.expect_workflow_uuid + '\n')
1705         self.assertEqual(exited, 0)