sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  51     @mock.patch("uuid.uuid4")
  52     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  53     @mock.patch("arvados.collection.KeepClient")
  54     @mock.patch("arvados.keep.KeepClient")
  55     @mock.patch("arvados.events.subscribe")
  56     def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  57                 uuid4, determine_image_id, *args, **kwargs):
  58         class Stubs(object):
  59             pass
  60         stubs = Stubs()
  61         stubs.events = events
  62         stubs.keepdocker = keepdocker
  63
  64         uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  65                              "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  66                              "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  67
  68         determine_image_id.return_value = None
  69
  70         def putstub(p, **kwargs):
  71             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  72         keep_client1().put.side_effect = putstub
  73         keep_client1.put.side_effect = putstub
  74         keep_client2().put.side_effect = putstub
  75         keep_client2.put.side_effect = putstub
  76
  77         stubs.keep_client = keep_client2
  78         stubs.docker_images = {
  79             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  80             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  81             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  82             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  83         }
  84         def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  85             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  86         stubs.keepdocker.side_effect = kd
  87
  88         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  89         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  90
  91         if sys.version_info[0] < 3:
  92             stubs.capture_stdout = BytesIO()
  93         else:
  94             stubs.capture_stdout = StringIO()
  95
  96         stubs.api = mock.MagicMock()
  97         stubs.api._rootDesc = get_rootDesc()
  98         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  99         stubs.api._rootDesc["revision"] = "20210628"
 100
 101         stubs.api.users().current().execute.return_value = {
 102             "uuid": stubs.fake_user_uuid,
 103         }
 104         stubs.api.collections().list().execute.return_value = {"items": []}
 105         stubs.api.containers().current().execute.return_value = {
 106             "uuid": stubs.fake_container_uuid,
 107         }
 108         stubs.api.config()["StorageClasses"].items.return_value = {
 109             "default": {
 110                 "Default": True
 111             }
 112         }.items()
 113
 114         class CollectionExecute(object):
 115             def __init__(self, exe):
 116                 self.exe = exe
 117             def execute(self, num_retries=None):
 118                 return self.exe
 119
 120         def collection_createstub(created_collections, body, ensure_unique_name=None):
 121             mt = body["manifest_text"].encode('utf-8')
 122             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 123             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 124             created_collections[uuid] = {
 125                 "uuid": uuid,
 126                 "portable_data_hash": pdh,
 127                 "manifest_text": mt.decode('utf-8')
 128             }
 129             return CollectionExecute(created_collections[uuid])
 130
 131         def collection_getstub(created_collections, uuid):
 132             for v in viewvalues(created_collections):
 133                 if uuid in (v["uuid"], v["portable_data_hash"]):
 134                     return CollectionExecute(v)
 135
 136         created_collections = {
 137             "99999999999999999999999999999998+99": {
 138                 "uuid": "",
 139                 "portable_data_hash": "99999999999999999999999999999998+99",
 140                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 141             },
 142             "99999999999999999999999999999997+99": {
 143                 "uuid": "",
 144                 "portable_data_hash": "99999999999999999999999999999997+99",
 145                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 146             },
 147             "99999999999999999999999999999994+99": {
 148                 "uuid": "",
 149                 "portable_data_hash": "99999999999999999999999999999994+99",
 150                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 154                 "portable_data_hash": "999999999999999999999999999999d3+99",
 155                 "manifest_text": ""
 156             },
 157             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 158                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 159                 "portable_data_hash": "999999999999999999999999999999d4+99",
 160                 "manifest_text": ""
 161             },
 162             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 163                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 164                 "portable_data_hash": "999999999999999999999999999999d5+99",
 165                 "manifest_text": ""
 166             },
 167             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 168                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 169                 "portable_data_hash": "999999999999999999999999999999d6+99",
 170                 "manifest_text": ""
 171             }
 172         }
 173         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 174         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 175
 176         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 177         stubs.api.jobs().create().execute.return_value = {
 178             "uuid": stubs.expect_job_uuid,
 179             "state": "Queued",
 180         }
 181
 182         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 183         stubs.api.container_requests().create().execute.return_value = {
 184             "uuid": stubs.expect_container_request_uuid,
 185             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 186             "state": "Queued"
 187         }
 188
 189         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 190         stubs.api.pipeline_templates().create().execute.return_value = {
 191             "uuid": stubs.expect_pipeline_template_uuid,
 192         }
 193         stubs.expect_job_spec = {
 194             'runtime_constraints': {
 195                 'docker_image': '999999999999999999999999999999d3+99',
 196                 'min_ram_mb_per_node': 1024
 197             },
 198             'script_parameters': {
 199                 'x': {
 200                     'basename': 'blorp.txt',
 201                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 202                     'class': 'File'
 203                 },
 204                 'y': {
 205                     'basename': '99999999999999999999999999999998+99',
 206                     'location': 'keep:99999999999999999999999999999998+99',
 207                     'class': 'Directory'
 208                 },
 209                 'z': {
 210                     'basename': 'anonymous',
 211                     "listing": [{
 212                         "basename": "renamed.txt",
 213                         "class": "File",
 214                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 215                         "size": 0
 216                     }],
 217                     'class': 'Directory'
 218                 },
 219                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 220             },
 221             'repository': 'arvados',
 222             'script_version': 'master',
 223             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 224             'script': 'cwl-runner'
 225         }
 226         stubs.pipeline_component = stubs.expect_job_spec.copy()
 227         stubs.expect_pipeline_instance = {
 228             'name': 'submit_wf.cwl',
 229             'state': 'RunningOnServer',
 230             'owner_uuid': None,
 231             "components": {
 232                 "cwl-runner": {
 233                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 234                     'script_parameters': {
 235                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 236                         'x': {"value": {
 237                             'basename': 'blorp.txt',
 238                             'class': 'File',
 239                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 240                             "size": 16
 241                         }},
 242                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 243                               'listing': [
 244                                   {
 245                                       'basename': 'renamed.txt',
 246                                       'class': 'File', 'location':
 247                                       'keep:99999999999999999999999999999998+99/file1.txt',
 248                                       'size': 0
 249                                   }
 250                               ]}},
 251                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 252                         'arv:debug': True,
 253                         'arv:enable_reuse': True,
 254                         'arv:on_error': 'continue'
 255                     },
 256                     'repository': 'arvados',
 257                     'script_version': 'master',
 258                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 259                     'script': 'cwl-runner',
 260                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 261                 }
 262             }
 263         }
 264         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 265         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 266         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 267         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 268         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 269             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 270             "state": "Queued"
 271         }
 272         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 273         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 274
 275         with open("tests/wf/submit_wf_packed.cwl") as f:
 276             expect_packed_workflow = yaml.round_trip_load(f)
 277
 278         stubs.expect_container_spec = {
 279             'priority': 500,
 280             'mounts': {
 281                 '/var/spool/cwl': {
 282                     'writable': True,
 283                     'kind': 'collection'
 284                 },
 285                 '/var/lib/cwl/workflow.json': {
 286                     'content': expect_packed_workflow,
 287                     'kind': 'json'
 288                 },
 289                 'stdout': {
 290                     'path': '/var/spool/cwl/cwl.output.json',
 291                     'kind': 'file'
 292                 },
 293                 '/var/lib/cwl/cwl.input.json': {
 294                     'kind': 'json',
 295                     'content': {
 296                         'y': {
 297                             'basename': '99999999999999999999999999999998+99',
 298                             'location': 'keep:99999999999999999999999999999998+99',
 299                             'class': 'Directory'},
 300                         'x': {
 301                             'basename': u'blorp.txt',
 302                             'class': 'File',
 303                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 304                             "size": 16
 305                         },
 306                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 307                             {'basename': 'renamed.txt',
 308                              'class': 'File',
 309                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 310                              'size': 0
 311                             }
 312                         ]}
 313                     },
 314                     'kind': 'json'
 315                 }
 316             },
 317             'secret_mounts': {},
 318             'state': 'Committed',
 319             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 320                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 321                         '--eval-timeout=20', '--thread-count=0',
 322                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 323                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 324             'name': 'submit_wf.cwl',
 325             'container_image': '999999999999999999999999999999d3+99',
 326             'output_path': '/var/spool/cwl',
 327             'cwd': '/var/spool/cwl',
 328             'runtime_constraints': {
 329                 'API': True,
 330                 'vcpus': 1,
 331                 'ram': (1024+256)*1024*1024
 332             },
 333             'use_existing': False,
 334             'properties': {},
 335             'secret_mounts': {}
 336         }
 337
 338         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 339         stubs.api.workflows().create().execute.return_value = {
 340             "uuid": stubs.expect_workflow_uuid,
 341         }
 342         def update_mock(**kwargs):
 343             stubs.updated_uuid = kwargs.get('uuid')
 344             return mock.DEFAULT
 345         stubs.api.workflows().update.side_effect = update_mock
 346         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 347             "uuid": stubs.updated_uuid,
 348         }
 349
 350         return func(self, stubs, *args, **kwargs)
 351     return wrapped
 352
 353
 354 class TestSubmit(unittest.TestCase):
 355
 356     def setUp(self):
 357         cwltool.process._names = set()
 358         arvados_cwl.arvdocker.arv_docker_clear_cache()
 359
 360     def tearDown(self):
 361         root_logger = logging.getLogger('')
 362
 363         # Remove existing RuntimeStatusLoggingHandlers if they exist
 364         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 365         root_logger.handlers = handlers
 366
 367     @mock.patch("time.sleep")
 368     @stubs
 369     def test_submit_invalid_runner_ram(self, stubs, tm):
 370         exited = arvados_cwl.main(
 371             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 372              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 373             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 374         self.assertEqual(exited, 1)
 375
 376
 377     @stubs
 378     def test_submit_container(self, stubs):
 379         exited = arvados_cwl.main(
 380             ["--submit", "--no-wait", "--api=containers", "--debug",
 381                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 382             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 383
 384         stubs.api.collections().create.assert_has_calls([
 385             mock.call(body=JsonDiffMatcher({
 386                 'manifest_text':
 387                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 388                 'replication_desired': None,
 389                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 390             }), ensure_unique_name=False),
 391             mock.call(body=JsonDiffMatcher({
 392                 'manifest_text':
 393                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 394                 'replication_desired': None,
 395                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 396             }), ensure_unique_name=False),
 397             ])
 398
 399         expect_container = copy.deepcopy(stubs.expect_container_spec)
 400         stubs.api.container_requests().create.assert_called_with(
 401             body=JsonDiffMatcher(expect_container))
 402         self.assertEqual(stubs.capture_stdout.getvalue(),
 403                          stubs.expect_container_request_uuid + '\n')
 404         self.assertEqual(exited, 0)
 405
 406
 407     @stubs
 408     def test_submit_container_tool(self, stubs):
 409         # test for issue #16139
 410         exited = arvados_cwl.main(
 411             ["--submit", "--no-wait", "--api=containers", "--debug",
 412                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 413             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 414
 415         self.assertEqual(stubs.capture_stdout.getvalue(),
 416                          stubs.expect_container_request_uuid + '\n')
 417         self.assertEqual(exited, 0)
 418
 419     @stubs
 420     def test_submit_container_no_reuse(self, stubs):
 421         exited = arvados_cwl.main(
 422             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 423                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 424             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 425
 426         expect_container = copy.deepcopy(stubs.expect_container_spec)
 427         expect_container["command"] = [
 428             'arvados-cwl-runner', '--local', '--api=containers',
 429             '--no-log-timestamps', '--disable-validate', '--disable-color',
 430             '--eval-timeout=20', '--thread-count=0',
 431             '--disable-reuse', "--collection-cache-size=256",
 432             '--debug', '--on-error=continue',
 433             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 434         expect_container["use_existing"] = False
 435
 436         stubs.api.container_requests().create.assert_called_with(
 437             body=JsonDiffMatcher(expect_container))
 438         self.assertEqual(stubs.capture_stdout.getvalue(),
 439                          stubs.expect_container_request_uuid + '\n')
 440         self.assertEqual(exited, 0)
 441
 442     @stubs
 443     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 444         exited = arvados_cwl.main(
 445             ["--submit", "--no-wait", "--api=containers", "--debug",
 446              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 447             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 448         self.assertEqual(exited, 0)
 449
 450         expect_container = copy.deepcopy(stubs.expect_container_spec)
 451         expect_container["command"] = [
 452             'arvados-cwl-runner', '--local', '--api=containers',
 453             '--no-log-timestamps', '--disable-validate', '--disable-color',
 454             '--eval-timeout=20', '--thread-count=0',
 455             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 456             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 457         expect_container["use_existing"] = False
 458         expect_container["name"] = "submit_wf_no_reuse.cwl"
 459         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 460             {
 461                 "class": "http://arvados.org/cwl#ReuseRequirement",
 462                 "enableReuse": False,
 463             },
 464         ]
 465         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 466             "arv": "http://arvados.org/cwl#",
 467             "cwltool": "http://commonwl.org/cwltool#"
 468         }
 469
 470         stubs.api.container_requests().create.assert_called_with(
 471             body=JsonDiffMatcher(expect_container))
 472         self.assertEqual(stubs.capture_stdout.getvalue(),
 473                          stubs.expect_container_request_uuid + '\n')
 474
 475
 476     @stubs
 477     def test_submit_container_on_error(self, stubs):
 478         exited = arvados_cwl.main(
 479             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 480                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 481             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 482
 483         expect_container = copy.deepcopy(stubs.expect_container_spec)
 484         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 485                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 486                                        '--eval-timeout=20', '--thread-count=0',
 487                                        '--enable-reuse', "--collection-cache-size=256",
 488                                        '--debug', '--on-error=stop',
 489                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 490
 491         stubs.api.container_requests().create.assert_called_with(
 492             body=JsonDiffMatcher(expect_container))
 493         self.assertEqual(stubs.capture_stdout.getvalue(),
 494                          stubs.expect_container_request_uuid + '\n')
 495         self.assertEqual(exited, 0)
 496
 497     @stubs
 498     def test_submit_container_output_name(self, stubs):
 499         output_name = "test_output_name"
 500
 501         exited = arvados_cwl.main(
 502             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 503                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505
 506         expect_container = copy.deepcopy(stubs.expect_container_spec)
 507         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 508                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 509                                        '--eval-timeout=20', '--thread-count=0',
 510                                        '--enable-reuse', "--collection-cache-size=256",
 511                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 512                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 513         expect_container["output_name"] = output_name
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(stubs.capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519         self.assertEqual(exited, 0)
 520
 521     @stubs
 522     def test_submit_storage_classes(self, stubs):
 523         exited = arvados_cwl.main(
 524             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 525                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 526             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 527
 528         expect_container = copy.deepcopy(stubs.expect_container_spec)
 529         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 530                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 531                                        '--eval-timeout=20', '--thread-count=0',
 532                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 533                                        "--storage-classes=foo", '--on-error=continue',
 534                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 535
 536         stubs.api.container_requests().create.assert_called_with(
 537             body=JsonDiffMatcher(expect_container))
 538         self.assertEqual(stubs.capture_stdout.getvalue(),
 539                          stubs.expect_container_request_uuid + '\n')
 540         self.assertEqual(exited, 0)
 541
 542     @stubs
 543     def test_submit_multiple_storage_classes(self, stubs):
 544         exited = arvados_cwl.main(
 545             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 546                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 547             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 548
 549         expect_container = copy.deepcopy(stubs.expect_container_spec)
 550         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 551                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 552                                        '--eval-timeout=20', '--thread-count=0',
 553                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 554                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 555                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 556
 557         stubs.api.container_requests().create.assert_called_with(
 558             body=JsonDiffMatcher(expect_container))
 559         self.assertEqual(stubs.capture_stdout.getvalue(),
 560                          stubs.expect_container_request_uuid + '\n')
 561         self.assertEqual(exited, 0)
 562
 563     @mock.patch("cwltool.task_queue.TaskQueue")
 564     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 565     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 566     @stubs
 567     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 568         final_output_c = arvados.collection.Collection()
 569         make_output.return_value = ({},final_output_c)
 570
 571         def set_final_output(job_order, output_callback, runtimeContext):
 572             output_callback({"out": "zzzzz"}, "success")
 573             return []
 574         job.side_effect = set_final_output
 575
 576         exited = arvados_cwl.main(
 577             ["--debug", "--local", "--storage-classes=foo",
 578                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 579             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 580
 581         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 582         self.assertEqual(exited, 0)
 583
 584     @mock.patch("cwltool.task_queue.TaskQueue")
 585     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 586     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 587     @stubs
 588     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 589         final_output_c = arvados.collection.Collection()
 590         make_output.return_value = ({},final_output_c)
 591         stubs.api.config().get.return_value = {"default": {"Default": True}}
 592
 593         def set_final_output(job_order, output_callback, runtimeContext):
 594             output_callback({"out": "zzzzz"}, "success")
 595             return []
 596         job.side_effect = set_final_output
 597
 598         exited = arvados_cwl.main(
 599             ["--debug", "--local",
 600                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 601             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 602
 603         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 604         self.assertEqual(exited, 0)
 605
 606     @mock.patch("cwltool.task_queue.TaskQueue")
 607     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 608     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 609     @stubs
 610     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 611         final_output_c = arvados.collection.Collection()
 612         make_output.return_value = ({},final_output_c)
 613
 614         def set_final_output(job_order, output_callback, runtimeContext):
 615             output_callback({"out": "zzzzz"}, "success")
 616             return []
 617         job.side_effect = set_final_output
 618
 619         exited = arvados_cwl.main(
 620             ["--debug", "--local",
 621                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 622             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 623
 624         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 625         self.assertEqual(exited, 0)
 626
 627     @stubs
 628     def test_submit_container_output_ttl(self, stubs):
 629         exited = arvados_cwl.main(
 630             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 631                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 632             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 633
 634         expect_container = copy.deepcopy(stubs.expect_container_spec)
 635         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 636                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 637                                        '--eval-timeout=20', '--thread-count=0',
 638                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 639                                        '--on-error=continue',
 640                                        "--intermediate-output-ttl=3600",
 641                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 642
 643         stubs.api.container_requests().create.assert_called_with(
 644             body=JsonDiffMatcher(expect_container))
 645         self.assertEqual(stubs.capture_stdout.getvalue(),
 646                          stubs.expect_container_request_uuid + '\n')
 647         self.assertEqual(exited, 0)
 648
 649     @stubs
 650     def test_submit_container_trash_intermediate(self, stubs):
 651         exited = arvados_cwl.main(
 652             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 653                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 654             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 655
 656
 657         expect_container = copy.deepcopy(stubs.expect_container_spec)
 658         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 659                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 660                                        '--eval-timeout=20', '--thread-count=0',
 661                                        '--enable-reuse', "--collection-cache-size=256",
 662                                        '--debug', '--on-error=continue',
 663                                        "--trash-intermediate",
 664                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 665
 666         stubs.api.container_requests().create.assert_called_with(
 667             body=JsonDiffMatcher(expect_container))
 668         self.assertEqual(stubs.capture_stdout.getvalue(),
 669                          stubs.expect_container_request_uuid + '\n')
 670         self.assertEqual(exited, 0)
 671
 672     @stubs
 673     def test_submit_container_output_tags(self, stubs):
 674         output_tags = "tag0,tag1,tag2"
 675
 676         exited = arvados_cwl.main(
 677             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 678                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 679             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 680
 681         expect_container = copy.deepcopy(stubs.expect_container_spec)
 682         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 683                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 684                                        '--eval-timeout=20', '--thread-count=0',
 685                                        '--enable-reuse', "--collection-cache-size=256",
 686                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 687                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 688
 689         stubs.api.container_requests().create.assert_called_with(
 690             body=JsonDiffMatcher(expect_container))
 691         self.assertEqual(stubs.capture_stdout.getvalue(),
 692                          stubs.expect_container_request_uuid + '\n')
 693         self.assertEqual(exited, 0)
 694
 695     @stubs
 696     def test_submit_container_runner_ram(self, stubs):
 697         exited = arvados_cwl.main(
 698             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 699                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 700             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 701
 702         expect_container = copy.deepcopy(stubs.expect_container_spec)
 703         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 704
 705         stubs.api.container_requests().create.assert_called_with(
 706             body=JsonDiffMatcher(expect_container))
 707         self.assertEqual(stubs.capture_stdout.getvalue(),
 708                          stubs.expect_container_request_uuid + '\n')
 709         self.assertEqual(exited, 0)
 710
 711     @mock.patch("arvados.collection.CollectionReader")
 712     @mock.patch("time.sleep")
 713     @stubs
 714     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 715         collectionReader().exists.return_value = True
 716         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 717         exited = arvados_cwl.main(
 718             ["--submit", "--no-wait", "--api=containers", "--debug",
 719              "tests/wf/submit_keepref_wf.cwl"],
 720             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 721         self.assertEqual(exited, 0)
 722
 723     @mock.patch("arvados.collection.CollectionReader")
 724     @mock.patch("time.sleep")
 725     @stubs
 726     def test_submit_keepref(self, stubs, tm, reader):
 727         with open("tests/wf/expect_arvworkflow.cwl") as f:
 728             reader().open().__enter__().read.return_value = f.read()
 729
 730         exited = arvados_cwl.main(
 731             ["--submit", "--no-wait", "--api=containers", "--debug",
 732              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 733             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 734
 735         expect_container = {
 736             'priority': 500,
 737             'mounts': {
 738                 '/var/spool/cwl': {
 739                     'writable': True,
 740                     'kind': 'collection'
 741                 },
 742                 'stdout': {
 743                     'path': '/var/spool/cwl/cwl.output.json',
 744                     'kind': 'file'
 745                 },
 746                 '/var/lib/cwl/workflow': {
 747                     'portable_data_hash': '99999999999999999999999999999994+99',
 748                     'kind': 'collection'
 749                 },
 750                 '/var/lib/cwl/cwl.input.json': {
 751                     'content': {
 752                         'x': 'XxX'
 753                     },
 754                     'kind': 'json'
 755                 }
 756             }, 'state': 'Committed',
 757             'output_path': '/var/spool/cwl',
 758             'name': 'expect_arvworkflow.cwl#main',
 759             'container_image': '999999999999999999999999999999d3+99',
 760             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 761                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 762                         '--eval-timeout=20', '--thread-count=0',
 763                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 764                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 765             'cwd': '/var/spool/cwl',
 766             'runtime_constraints': {
 767                 'API': True,
 768                 'vcpus': 1,
 769                 'ram': 1342177280
 770             },
 771             'use_existing': False,
 772             'properties': {},
 773             'secret_mounts': {}
 774         }
 775
 776         stubs.api.container_requests().create.assert_called_with(
 777             body=JsonDiffMatcher(expect_container))
 778         self.assertEqual(stubs.capture_stdout.getvalue(),
 779                          stubs.expect_container_request_uuid + '\n')
 780         self.assertEqual(exited, 0)
 781
 782     @mock.patch("time.sleep")
 783     @stubs
 784     def test_submit_arvworkflow(self, stubs, tm):
 785         with open("tests/wf/expect_arvworkflow.cwl") as f:
 786             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 787
 788         exited = arvados_cwl.main(
 789             ["--submit", "--no-wait", "--api=containers", "--debug",
 790              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 791             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 792
 793         expect_container = {
 794             'priority': 500,
 795             'mounts': {
 796                 '/var/spool/cwl': {
 797                     'writable': True,
 798                     'kind': 'collection'
 799                 },
 800                 'stdout': {
 801                     'path': '/var/spool/cwl/cwl.output.json',
 802                     'kind': 'file'
 803                 },
 804                 '/var/lib/cwl/workflow.json': {
 805                     'kind': 'json',
 806                     'content': {
 807                         'cwlVersion': 'v1.0',
 808                         '$graph': [
 809                             {
 810                                 'id': '#main',
 811                                 'inputs': [
 812                                     {'type': 'string', 'id': '#main/x'}
 813                                 ],
 814                                 'steps': [
 815                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 816                                      'run': '#submit_tool.cwl',
 817                                      'id': '#main/step1',
 818                                      'out': []}
 819                                 ],
 820                                 'class': 'Workflow',
 821                                 'outputs': []
 822                             },
 823                             {
 824                                 'inputs': [
 825                                     {
 826                                         'inputBinding': {'position': 1},
 827                                         'type': 'string',
 828                                         'id': '#submit_tool.cwl/x'}
 829                                 ],
 830                                 'requirements': [
 831                                     {
 832                                         'dockerPull': 'debian:buster-slim',
 833                                         'class': 'DockerRequirement',
 834                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 835                                     }
 836                                 ],
 837                                 'id': '#submit_tool.cwl',
 838                                 'outputs': [],
 839                                 'baseCommand': 'cat',
 840                                 'class': 'CommandLineTool'
 841                             }
 842                         ]
 843                     }
 844                 },
 845                 '/var/lib/cwl/cwl.input.json': {
 846                     'content': {
 847                         'x': 'XxX'
 848                     },
 849                     'kind': 'json'
 850                 }
 851             }, 'state': 'Committed',
 852             'output_path': '/var/spool/cwl',
 853             'name': 'a test workflow',
 854             'container_image': "999999999999999999999999999999d3+99",
 855             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 856                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 857                         '--eval-timeout=20', '--thread-count=0',
 858                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 859                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 860             'cwd': '/var/spool/cwl',
 861             'runtime_constraints': {
 862                 'API': True,
 863                 'vcpus': 1,
 864                 'ram': 1342177280
 865             },
 866             'use_existing': False,
 867             'properties': {
 868                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 869             },
 870             'secret_mounts': {}
 871         }
 872
 873         stubs.api.container_requests().create.assert_called_with(
 874             body=JsonDiffMatcher(expect_container))
 875         self.assertEqual(stubs.capture_stdout.getvalue(),
 876                          stubs.expect_container_request_uuid + '\n')
 877         self.assertEqual(exited, 0)
 878
 879     @stubs
 880     def test_submit_container_name(self, stubs):
 881         exited = arvados_cwl.main(
 882             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 883                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 884             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 885
 886         expect_container = copy.deepcopy(stubs.expect_container_spec)
 887         expect_container["name"] = "hello container 123"
 888
 889         stubs.api.container_requests().create.assert_called_with(
 890             body=JsonDiffMatcher(expect_container))
 891         self.assertEqual(stubs.capture_stdout.getvalue(),
 892                          stubs.expect_container_request_uuid + '\n')
 893         self.assertEqual(exited, 0)
 894
 895     @stubs
 896     def test_submit_missing_input(self, stubs):
 897         exited = arvados_cwl.main(
 898             ["--submit", "--no-wait", "--api=containers", "--debug",
 899              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 900             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 901         self.assertEqual(exited, 0)
 902
 903         exited = arvados_cwl.main(
 904             ["--submit", "--no-wait", "--api=containers", "--debug",
 905              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 906             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 907         self.assertEqual(exited, 1)
 908
 909     @stubs
 910     def test_submit_container_project(self, stubs):
 911         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 912         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 913         exited = arvados_cwl.main(
 914             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 915                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 916             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 917
 918         expect_container = copy.deepcopy(stubs.expect_container_spec)
 919         expect_container["owner_uuid"] = project_uuid
 920         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 921                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 922                                        "--eval-timeout=20", "--thread-count=0",
 923                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 924                                        '--on-error=continue',
 925                                        '--project-uuid='+project_uuid,
 926                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 927
 928         stubs.api.container_requests().create.assert_called_with(
 929             body=JsonDiffMatcher(expect_container))
 930         self.assertEqual(stubs.capture_stdout.getvalue(),
 931                          stubs.expect_container_request_uuid + '\n')
 932         self.assertEqual(exited, 0)
 933
 934     @stubs
 935     def test_submit_container_eval_timeout(self, stubs):
 936         exited = arvados_cwl.main(
 937             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 938                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 939             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 940
 941         expect_container = copy.deepcopy(stubs.expect_container_spec)
 942         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 943                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 944                                        '--eval-timeout=60.0', '--thread-count=0',
 945                                        '--enable-reuse', "--collection-cache-size=256",
 946                                        '--debug', '--on-error=continue',
 947                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 948
 949         stubs.api.container_requests().create.assert_called_with(
 950             body=JsonDiffMatcher(expect_container))
 951         self.assertEqual(stubs.capture_stdout.getvalue(),
 952                          stubs.expect_container_request_uuid + '\n')
 953         self.assertEqual(exited, 0)
 954
 955     @stubs
 956     def test_submit_container_collection_cache(self, stubs):
 957         exited = arvados_cwl.main(
 958             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 959                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 960             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 961
 962         expect_container = copy.deepcopy(stubs.expect_container_spec)
 963         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 964                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 965                                        '--eval-timeout=20', '--thread-count=0',
 966                                        '--enable-reuse', "--collection-cache-size=500",
 967                                        '--debug', '--on-error=continue',
 968                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 969         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 970
 971         stubs.api.container_requests().create.assert_called_with(
 972             body=JsonDiffMatcher(expect_container))
 973         self.assertEqual(stubs.capture_stdout.getvalue(),
 974                          stubs.expect_container_request_uuid + '\n')
 975         self.assertEqual(exited, 0)
 976
 977     @stubs
 978     def test_submit_container_thread_count(self, stubs):
 979         exited = arvados_cwl.main(
 980             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 981                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 982             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 983
 984         expect_container = copy.deepcopy(stubs.expect_container_spec)
 985         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 986                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 987                                        '--eval-timeout=20', '--thread-count=20',
 988                                        '--enable-reuse', "--collection-cache-size=256",
 989                                        '--debug', '--on-error=continue',
 990                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 991
 992         stubs.api.container_requests().create.assert_called_with(
 993             body=JsonDiffMatcher(expect_container))
 994         self.assertEqual(stubs.capture_stdout.getvalue(),
 995                          stubs.expect_container_request_uuid + '\n')
 996         self.assertEqual(exited, 0)
 997
 998     @stubs
 999     def test_submit_container_runner_image(self, stubs):
1000         exited = arvados_cwl.main(
1001             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1002                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1003             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1004
1005         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1006
1007         expect_container = copy.deepcopy(stubs.expect_container_spec)
1008         stubs.api.container_requests().create.assert_called_with(
1009             body=JsonDiffMatcher(expect_container))
1010         self.assertEqual(stubs.capture_stdout.getvalue(),
1011                          stubs.expect_container_request_uuid + '\n')
1012         self.assertEqual(exited, 0)
1013
1014     @stubs
1015     def test_submit_priority(self, stubs):
1016         exited = arvados_cwl.main(
1017             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1018                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1019             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1020
1021         stubs.expect_container_spec["priority"] = 669
1022
1023         expect_container = copy.deepcopy(stubs.expect_container_spec)
1024         stubs.api.container_requests().create.assert_called_with(
1025             body=JsonDiffMatcher(expect_container))
1026         self.assertEqual(stubs.capture_stdout.getvalue(),
1027                          stubs.expect_container_request_uuid + '\n')
1028         self.assertEqual(exited, 0)
1029
1030     @stubs
1031     def test_submit_wf_runner_resources(self, stubs):
1032         exited = arvados_cwl.main(
1033             ["--submit", "--no-wait", "--api=containers", "--debug",
1034                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1035             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1036
1037         expect_container = copy.deepcopy(stubs.expect_container_spec)
1038         expect_container["runtime_constraints"] = {
1039             "API": True,
1040             "vcpus": 2,
1041             "ram": (2000+512) * 2**20
1042         }
1043         expect_container["name"] = "submit_wf_runner_resources.cwl"
1044         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1045             {
1046                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1047                 "coresMin": 2,
1048                 "ramMin": 2000,
1049                 "keep_cache": 512
1050             }
1051         ]
1052         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1053             "arv": "http://arvados.org/cwl#",
1054         }
1055         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1056                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1057                         '--eval-timeout=20', '--thread-count=0',
1058                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1059                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1060
1061         stubs.api.container_requests().create.assert_called_with(
1062             body=JsonDiffMatcher(expect_container))
1063         self.assertEqual(stubs.capture_stdout.getvalue(),
1064                          stubs.expect_container_request_uuid + '\n')
1065         self.assertEqual(exited, 0)
1066
1067     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1068     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1069     @mock.patch("arvados.api")
1070     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1071         arvados_cwl.arvdocker.arv_docker_clear_cache()
1072
1073         arvrunner = mock.MagicMock()
1074         arvrunner.project_uuid = ""
1075         api.return_value = mock.MagicMock()
1076         arvrunner.api = api.return_value
1077         arvrunner.runtimeContext.match_local_docker = False
1078         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1079                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1080                                                                         "link_class": "docker_image_repo+tag",
1081                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1082                                                                         "owner_uuid": "",
1083                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1084                                                             {"items": [{"created_at": "",
1085                                                                         "head_uuid": "",
1086                                                                         "link_class": "docker_image_hash",
1087                                                                         "name": "123456",
1088                                                                         "owner_uuid": "",
1089                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1090                                                             {"items": [{"created_at": "",
1091                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1092                                                                         "link_class": "docker_image_repo+tag",
1093                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1094                                                                         "owner_uuid": "",
1095                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1096                                                             {"items": [{"created_at": "",
1097                                                                         "head_uuid": "",
1098                                                                         "link_class": "docker_image_hash",
1099                                                                         "name": "123456",
1100                                                                         "owner_uuid": "",
1101                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1102         )
1103         find_one_image_hash.return_value = "123456"
1104
1105         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1106                                                                               "owner_uuid": "",
1107                                                                               "manifest_text": "",
1108                                                                               "properties": ""
1109                                                                               }], "items_available": 1, "offset": 0},
1110                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1111                                                                               "owner_uuid": "",
1112                                                                               "manifest_text": "",
1113                                                                               "properties": ""
1114                                                                           }], "items_available": 1, "offset": 0})
1115         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1116         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1117                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1118
1119         self.assertEqual("9999999999999999999999999999999b+99",
1120                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1121
1122
1123     @stubs
1124     def test_submit_secrets(self, stubs):
1125         exited = arvados_cwl.main(
1126             ["--submit", "--no-wait", "--api=containers", "--debug",
1127                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1128             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1129
1130         expect_container = {
1131             "command": [
1132                 "arvados-cwl-runner",
1133                 "--local",
1134                 "--api=containers",
1135                 "--no-log-timestamps",
1136                 "--disable-validate",
1137                 "--disable-color",
1138                 "--eval-timeout=20",
1139                 '--thread-count=0',
1140                 "--enable-reuse",
1141                 "--collection-cache-size=256",
1142                 '--debug',
1143                 "--on-error=continue",
1144                 "/var/lib/cwl/workflow.json#main",
1145                 "/var/lib/cwl/cwl.input.json"
1146             ],
1147             "container_image": "999999999999999999999999999999d3+99",
1148             "cwd": "/var/spool/cwl",
1149             "mounts": {
1150                 "/var/lib/cwl/cwl.input.json": {
1151                     "content": {
1152                         "pw": {
1153                             "$include": "/secrets/s0"
1154                         }
1155                     },
1156                     "kind": "json"
1157                 },
1158                 "/var/lib/cwl/workflow.json": {
1159                     "content": {
1160                         "$graph": [
1161                             {
1162                                 "arguments": [
1163                                     "md5sum",
1164                                     "example.conf"
1165                                 ],
1166                                 "class": "CommandLineTool",
1167                                 "hints": [
1168                                     {
1169                                         "class": "http://commonwl.org/cwltool#Secrets",
1170                                         "secrets": [
1171                                             "#secret_job.cwl/pw"
1172                                         ]
1173                                     }
1174                                 ],
1175                                 "id": "#secret_job.cwl",
1176                                 "inputs": [
1177                                     {
1178                                         "id": "#secret_job.cwl/pw",
1179                                         "type": "string"
1180                                     }
1181                                 ],
1182                                 "outputs": [
1183                                     {
1184                                         "id": "#secret_job.cwl/out",
1185                                         "type": "File",
1186                                         "outputBinding": {
1187                                               "glob": "hashed_example.txt"
1188                                         }
1189                                     }
1190                                 ],
1191                                 "stdout": "hashed_example.txt",
1192                                 "requirements": [
1193                                     {
1194                                         "class": "InitialWorkDirRequirement",
1195                                         "listing": [
1196                                             {
1197                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1198                                                 "entryname": "example.conf"
1199                                             }
1200                                         ]
1201                                     }
1202                                 ]
1203                             },
1204                             {
1205                                 "class": "Workflow",
1206                                 "hints": [
1207                                     {
1208                                         "class": "DockerRequirement",
1209                                         "dockerPull": "debian:buster-slim",
1210                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1211                                     },
1212                                     {
1213                                         "class": "http://commonwl.org/cwltool#Secrets",
1214                                         "secrets": [
1215                                             "#main/pw"
1216                                         ]
1217                                     }
1218                                 ],
1219                                 "id": "#main",
1220                                 "inputs": [
1221                                     {
1222                                         "id": "#main/pw",
1223                                         "type": "string"
1224                                     }
1225                                 ],
1226                                 "outputs": [
1227                                     {
1228                                         "id": "#main/out",
1229                                         "outputSource": "#main/step1/out",
1230                                         "type": "File"
1231                                     }
1232                                 ],
1233                                 "steps": [
1234                                     {
1235                                         "id": "#main/step1",
1236                                         "in": [
1237                                             {
1238                                                 "id": "#main/step1/pw",
1239                                                 "source": "#main/pw"
1240                                             }
1241                                         ],
1242                                         "out": [
1243                                             "#main/step1/out"
1244                                         ],
1245                                         "run": "#secret_job.cwl"
1246                                     }
1247                                 ]
1248                             }
1249                         ],
1250                         "$namespaces": {
1251                             "cwltool": "http://commonwl.org/cwltool#"
1252                         },
1253                         "cwlVersion": "v1.0"
1254                     },
1255                     "kind": "json"
1256                 },
1257                 "/var/spool/cwl": {
1258                     "kind": "collection",
1259                     "writable": True
1260                 },
1261                 "stdout": {
1262                     "kind": "file",
1263                     "path": "/var/spool/cwl/cwl.output.json"
1264                 }
1265             },
1266             "name": "secret_wf.cwl",
1267             "output_path": "/var/spool/cwl",
1268             "priority": 500,
1269             "properties": {},
1270             "runtime_constraints": {
1271                 "API": True,
1272                 "ram": 1342177280,
1273                 "vcpus": 1
1274             },
1275             "secret_mounts": {
1276                 "/secrets/s0": {
1277                     "content": "blorp",
1278                     "kind": "text"
1279                 }
1280             },
1281             "state": "Committed",
1282             "use_existing": False
1283         }
1284
1285         stubs.api.container_requests().create.assert_called_with(
1286             body=JsonDiffMatcher(expect_container))
1287         self.assertEqual(stubs.capture_stdout.getvalue(),
1288                          stubs.expect_container_request_uuid + '\n')
1289         self.assertEqual(exited, 0)
1290
1291     @stubs
1292     def test_submit_request_uuid(self, stubs):
1293         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1294         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1295
1296         stubs.api.container_requests().update().execute.return_value = {
1297             "uuid": stubs.expect_container_request_uuid,
1298             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1299             "state": "Queued"
1300         }
1301
1302         exited = arvados_cwl.main(
1303             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1304                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1305             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1306
1307         stubs.api.container_requests().update.assert_called_with(
1308             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1309         self.assertEqual(stubs.capture_stdout.getvalue(),
1310                          stubs.expect_container_request_uuid + '\n')
1311         self.assertEqual(exited, 0)
1312
1313     @stubs
1314     def test_submit_container_cluster_id(self, stubs):
1315         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1316
1317         exited = arvados_cwl.main(
1318             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1319                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1320             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1321
1322         expect_container = copy.deepcopy(stubs.expect_container_spec)
1323
1324         stubs.api.container_requests().create.assert_called_with(
1325             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1326         self.assertEqual(stubs.capture_stdout.getvalue(),
1327                          stubs.expect_container_request_uuid + '\n')
1328         self.assertEqual(exited, 0)
1329
1330     @stubs
1331     def test_submit_validate_cluster_id(self, stubs):
1332         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1333         exited = arvados_cwl.main(
1334             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1335              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1336             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1337         self.assertEqual(exited, 1)
1338
1339     @stubs
1340     def test_submit_validate_project_uuid(self, stubs):
1341         # Fails with bad cluster prefix
1342         exited = arvados_cwl.main(
1343             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1344              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1345             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1346         self.assertEqual(exited, 1)
1347
1348         # Project lookup fails
1349         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1350         exited = arvados_cwl.main(
1351             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1352              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1353             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1354         self.assertEqual(exited, 1)
1355
1356         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1357         exited = arvados_cwl.main(
1358             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1360             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1361         self.assertEqual(exited, 0)
1362
1363
1364     @mock.patch("arvados.collection.CollectionReader")
1365     @stubs
1366     def test_submit_uuid_inputs(self, stubs, collectionReader):
1367         collectionReader().exists.return_value = True
1368         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1369         def list_side_effect(**kwargs):
1370             m = mock.MagicMock()
1371             if "count" in kwargs:
1372                 m.execute.return_value = {"items": [
1373                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1374                 ]}
1375             else:
1376                 m.execute.return_value = {"items": []}
1377             return m
1378         stubs.api.collections().list.side_effect = list_side_effect
1379
1380         exited = arvados_cwl.main(
1381             ["--submit", "--no-wait", "--api=containers", "--debug",
1382                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1383             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1384
1385         expect_container = copy.deepcopy(stubs.expect_container_spec)
1386         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1387         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1388         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1389
1390         stubs.api.collections().list.assert_has_calls([
1391             mock.call(count='none',
1392                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1393                       select=['uuid', 'portable_data_hash'])])
1394         stubs.api.container_requests().create.assert_called_with(
1395             body=JsonDiffMatcher(expect_container))
1396         self.assertEqual(stubs.capture_stdout.getvalue(),
1397                          stubs.expect_container_request_uuid + '\n')
1398         self.assertEqual(exited, 0)
1399
1400     @stubs
1401     def test_submit_mismatched_uuid_inputs(self, stubs):
1402         def list_side_effect(**kwargs):
1403             m = mock.MagicMock()
1404             if "count" in kwargs:
1405                 m.execute.return_value = {"items": [
1406                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1407                 ]}
1408             else:
1409                 m.execute.return_value = {"items": []}
1410             return m
1411         stubs.api.collections().list.side_effect = list_side_effect
1412
1413         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1414             capture_stderr = StringIO()
1415             cwltool_logger = logging.getLogger('cwltool')
1416             stderr_logger = logging.StreamHandler(capture_stderr)
1417             cwltool_logger.addHandler(stderr_logger)
1418
1419             try:
1420                 exited = arvados_cwl.main(
1421                     ["--submit", "--no-wait", "--api=containers", "--debug",
1422                         "tests/wf/submit_wf.cwl", infile],
1423                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1424
1425                 self.assertEqual(exited, 1)
1426                 self.assertRegex(
1427                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1428                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1429             finally:
1430                 cwltool_logger.removeHandler(stderr_logger)
1431
1432     @mock.patch("arvados.collection.CollectionReader")
1433     @stubs
1434     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1435         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1436         capture_stderr = StringIO()
1437
1438         cwltool_logger = logging.getLogger('cwltool')
1439         stderr_logger = logging.StreamHandler(capture_stderr)
1440         cwltool_logger.addHandler(stderr_logger)
1441
1442         exited = arvados_cwl.main(
1443             ["--submit", "--no-wait", "--api=containers", "--debug",
1444                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1445             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1446
1447         try:
1448             self.assertEqual(exited, 1)
1449             self.assertRegex(
1450                 capture_stderr.getvalue(),
1451                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1452         finally:
1453             cwltool_logger.removeHandler(stderr_logger)
1454
1455     @stubs
1456     def test_submit_set_process_properties(self, stubs):
1457         exited = arvados_cwl.main(
1458             ["--submit", "--no-wait", "--api=containers", "--debug",
1459                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1460             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1461
1462         expect_container = copy.deepcopy(stubs.expect_container_spec)
1463         expect_container["name"] = "submit_wf_process_properties.cwl"
1464         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1465             {
1466                 "class": "http://arvados.org/cwl#ProcessProperties",
1467                 "processProperties": [
1468                     {"propertyName": "baz",
1469                      "propertyValue": "$(inputs.x.basename)"},
1470                     {"propertyName": "foo",
1471                      "propertyValue": "bar"},
1472                     {"propertyName": "quux",
1473                      "propertyValue": {
1474                          "q1": 1,
1475                          "q2": 2
1476                      }
1477                     }
1478                 ],
1479             }
1480         ]
1481         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1482             "arv": "http://arvados.org/cwl#"
1483         }
1484
1485         expect_container["properties"] = {
1486             "baz": "blorp.txt",
1487             "foo": "bar",
1488             "quux": {
1489                 "q1": 1,
1490                 "q2": 2
1491             }
1492         }
1493
1494         stubs.api.container_requests().create.assert_called_with(
1495             body=JsonDiffMatcher(expect_container))
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_container_request_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500
1501     @stubs
1502     def test_submit_enable_preemptible(self, stubs):
1503         exited = arvados_cwl.main(
1504             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1505                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1506             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1507
1508         expect_container = copy.deepcopy(stubs.expect_container_spec)
1509         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1510                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1511                         '--eval-timeout=20', '--thread-count=0',
1512                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1513                                        '--enable-preemptible',
1514                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1515
1516         stubs.api.container_requests().create.assert_called_with(
1517             body=JsonDiffMatcher(expect_container))
1518         self.assertEqual(stubs.capture_stdout.getvalue(),
1519                          stubs.expect_container_request_uuid + '\n')
1520         self.assertEqual(exited, 0)
1521
1522     @stubs
1523     def test_submit_disable_preemptible(self, stubs):
1524         exited = arvados_cwl.main(
1525             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1526                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1527             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1528
1529         expect_container = copy.deepcopy(stubs.expect_container_spec)
1530         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1531                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1532                         '--eval-timeout=20', '--thread-count=0',
1533                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1534                                        '--disable-preemptible',
1535                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1536
1537         stubs.api.container_requests().create.assert_called_with(
1538             body=JsonDiffMatcher(expect_container))
1539         self.assertEqual(stubs.capture_stdout.getvalue(),
1540                          stubs.expect_container_request_uuid + '\n')
1541         self.assertEqual(exited, 0)
1542
1543
1544 class TestCreateWorkflow(unittest.TestCase):
1545     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1546     expect_workflow = StripYAMLComments(
1547         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1548
1549     def setUp(self):
1550         cwltool.process._names = set()
1551         arvados_cwl.arvdocker.arv_docker_clear_cache()
1552
1553     def tearDown(self):
1554         root_logger = logging.getLogger('')
1555
1556         # Remove existing RuntimeStatusLoggingHandlers if they exist
1557         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1558         root_logger.handlers = handlers
1559
1560     @stubs
1561     def test_create(self, stubs):
1562         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1563         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1564
1565         exited = arvados_cwl.main(
1566             ["--create-workflow", "--debug",
1567              "--api=containers",
1568              "--project-uuid", project_uuid,
1569              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1570             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1571
1572         stubs.api.pipeline_templates().create.refute_called()
1573         stubs.api.container_requests().create.refute_called()
1574
1575         body = {
1576             "workflow": {
1577                 "owner_uuid": project_uuid,
1578                 "name": "submit_wf.cwl",
1579                 "description": "",
1580                 "definition": self.expect_workflow,
1581             }
1582         }
1583         stubs.api.workflows().create.assert_called_with(
1584             body=JsonDiffMatcher(body))
1585
1586         self.assertEqual(stubs.capture_stdout.getvalue(),
1587                          stubs.expect_workflow_uuid + '\n')
1588         self.assertEqual(exited, 0)
1589
1590     @stubs
1591     def test_create_name(self, stubs):
1592         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1593         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1594
1595         exited = arvados_cwl.main(
1596             ["--create-workflow", "--debug",
1597              "--api=containers",
1598              "--project-uuid", project_uuid,
1599              "--name", "testing 123",
1600              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1601             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1602
1603         stubs.api.pipeline_templates().create.refute_called()
1604         stubs.api.container_requests().create.refute_called()
1605
1606         body = {
1607             "workflow": {
1608                 "owner_uuid": project_uuid,
1609                 "name": "testing 123",
1610                 "description": "",
1611                 "definition": self.expect_workflow,
1612             }
1613         }
1614         stubs.api.workflows().create.assert_called_with(
1615             body=JsonDiffMatcher(body))
1616
1617         self.assertEqual(stubs.capture_stdout.getvalue(),
1618                          stubs.expect_workflow_uuid + '\n')
1619         self.assertEqual(exited, 0)
1620
1621
1622     @stubs
1623     def test_update(self, stubs):
1624         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1625         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1626
1627         exited = arvados_cwl.main(
1628             ["--update-workflow", self.existing_workflow_uuid,
1629              "--debug",
1630              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1631             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1632
1633         body = {
1634             "workflow": {
1635                 "name": "submit_wf.cwl",
1636                 "description": "",
1637                 "definition": self.expect_workflow,
1638                 "owner_uuid": project_uuid
1639             }
1640         }
1641         stubs.api.workflows().update.assert_called_with(
1642             uuid=self.existing_workflow_uuid,
1643             body=JsonDiffMatcher(body))
1644         self.assertEqual(stubs.capture_stdout.getvalue(),
1645                          self.existing_workflow_uuid + '\n')
1646         self.assertEqual(exited, 0)
1647
1648
1649     @stubs
1650     def test_update_name(self, stubs):
1651         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1652         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1653
1654         exited = arvados_cwl.main(
1655             ["--update-workflow", self.existing_workflow_uuid,
1656              "--debug", "--name", "testing 123",
1657              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1658             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1659
1660         body = {
1661             "workflow": {
1662                 "name": "testing 123",
1663                 "description": "",
1664                 "definition": self.expect_workflow,
1665                 "owner_uuid": project_uuid
1666             }
1667         }
1668         stubs.api.workflows().update.assert_called_with(
1669             uuid=self.existing_workflow_uuid,
1670             body=JsonDiffMatcher(body))
1671         self.assertEqual(stubs.capture_stdout.getvalue(),
1672                          self.existing_workflow_uuid + '\n')
1673         self.assertEqual(exited, 0)
1674
1675     @stubs
1676     def test_create_collection_per_tool(self, stubs):
1677         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1678         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1679
1680         exited = arvados_cwl.main(
1681             ["--create-workflow", "--debug",
1682              "--api=containers",
1683              "--project-uuid", project_uuid,
1684              "tests/collection_per_tool/collection_per_tool.cwl"],
1685             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1686
1687         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1688         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1689
1690         body = {
1691             "workflow": {
1692                 "owner_uuid": project_uuid,
1693                 "name": "collection_per_tool.cwl",
1694                 "description": "",
1695                 "definition": expect_workflow,
1696             }
1697         }
1698         stubs.api.workflows().create.assert_called_with(
1699             body=JsonDiffMatcher(body))
1700
1701         self.assertEqual(stubs.capture_stdout.getvalue(),
1702                          stubs.expect_workflow_uuid + '\n')
1703         self.assertEqual(exited, 0)
1704
1705     @stubs
1706     def test_create_with_imports(self, stubs):
1707         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1708         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1709
1710         exited = arvados_cwl.main(
1711             ["--create-workflow", "--debug",
1712              "--api=containers",
1713              "--project-uuid", project_uuid,
1714              "tests/wf/feddemo/feddemo.cwl"],
1715             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1716
1717         stubs.api.pipeline_templates().create.refute_called()
1718         stubs.api.container_requests().create.refute_called()
1719
1720         self.assertEqual(stubs.capture_stdout.getvalue(),
1721                          stubs.expect_workflow_uuid + '\n')
1722         self.assertEqual(exited, 0)
1723
1724     @stubs
1725     def test_create_with_no_input(self, stubs):
1726         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1727         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1728
1729         exited = arvados_cwl.main(
1730             ["--create-workflow", "--debug",
1731              "--api=containers",
1732              "--project-uuid", project_uuid,
1733              "tests/wf/revsort/revsort.cwl"],
1734             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1735
1736         stubs.api.pipeline_templates().create.refute_called()
1737         stubs.api.container_requests().create.refute_called()
1738
1739         self.assertEqual(stubs.capture_stdout.getvalue(),
1740                          stubs.expect_workflow_uuid + '\n')
1741         self.assertEqual(exited, 0)