sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(wfname='submit_wf.cwl'):
  49     def outer_wrapper(func, *rest):
  50         @functools.wraps(func)
  51         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  52         @mock.patch("uuid.uuid4")
  53         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  54         @mock.patch("arvados.collection.KeepClient")
  55         @mock.patch("arvados.keep.KeepClient")
  56         @mock.patch("arvados.events.subscribe")
  57         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  58                     uuid4, determine_image_id, *args, **kwargs):
  59             class Stubs(object):
  60                 pass
  61             stubs = Stubs()
  62             stubs.events = events
  63             stubs.keepdocker = keepdocker
  64
  65             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  66                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  67                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  68
  69             determine_image_id.return_value = None
  70
  71             def putstub(p, **kwargs):
  72                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  73             keep_client1().put.side_effect = putstub
  74             keep_client1.put.side_effect = putstub
  75             keep_client2().put.side_effect = putstub
  76             keep_client2.put.side_effect = putstub
  77
  78             stubs.keep_client = keep_client2
  79             stubs.docker_images = {
  80                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  81                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  82                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  83                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  84             }
  85             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  86                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  87             stubs.keepdocker.side_effect = kd
  88
  89             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  90             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  91
  92             if sys.version_info[0] < 3:
  93                 stubs.capture_stdout = BytesIO()
  94             else:
  95                 stubs.capture_stdout = StringIO()
  96
  97             stubs.api = mock.MagicMock()
  98             stubs.api._rootDesc = get_rootDesc()
  99             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 100             stubs.api._rootDesc["revision"] = "20210628"
 101
 102             stubs.api.users().current().execute.return_value = {
 103                 "uuid": stubs.fake_user_uuid,
 104             }
 105             stubs.api.collections().list().execute.return_value = {"items": []}
 106             stubs.api.containers().current().execute.return_value = {
 107                 "uuid": stubs.fake_container_uuid,
 108             }
 109             stubs.api.config()["StorageClasses"].items.return_value = {
 110                 "default": {
 111                     "Default": True
 112                 }
 113             }.items()
 114
 115             class CollectionExecute(object):
 116                 def __init__(self, exe):
 117                     self.exe = exe
 118                 def execute(self, num_retries=None):
 119                     return self.exe
 120
 121             def collection_createstub(created_collections, body, ensure_unique_name=None):
 122                 mt = body["manifest_text"].encode('utf-8')
 123                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 124                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 125                 created_collections[uuid] = {
 126                     "uuid": uuid,
 127                     "portable_data_hash": pdh,
 128                     "manifest_text": mt.decode('utf-8')
 129                 }
 130                 return CollectionExecute(created_collections[uuid])
 131
 132             def collection_getstub(created_collections, uuid):
 133                 for v in viewvalues(created_collections):
 134                     if uuid in (v["uuid"], v["portable_data_hash"]):
 135                         return CollectionExecute(v)
 136
 137             created_collections = {
 138                 "99999999999999999999999999999998+99": {
 139                     "uuid": "",
 140                     "portable_data_hash": "99999999999999999999999999999998+99",
 141                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 142                 },
 143                 "99999999999999999999999999999997+99": {
 144                     "uuid": "",
 145                     "portable_data_hash": "99999999999999999999999999999997+99",
 146                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 147                 },
 148                 "99999999999999999999999999999994+99": {
 149                     "uuid": "",
 150                     "portable_data_hash": "99999999999999999999999999999994+99",
 151                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 152                 },
 153                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 154                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 155                     "portable_data_hash": "999999999999999999999999999999d3+99",
 156                     "manifest_text": ""
 157                 },
 158                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 159                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 160                     "portable_data_hash": "999999999999999999999999999999d4+99",
 161                     "manifest_text": ""
 162                 },
 163                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 164                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 165                     "portable_data_hash": "999999999999999999999999999999d5+99",
 166                     "manifest_text": ""
 167                 },
 168                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 169                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 170                     "portable_data_hash": "999999999999999999999999999999d6+99",
 171                     "manifest_text": ""
 172                 }
 173             }
 174             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 175             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 176
 177             stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 178             stubs.api.jobs().create().execute.return_value = {
 179                 "uuid": stubs.expect_job_uuid,
 180                 "state": "Queued",
 181             }
 182
 183             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 184             stubs.api.container_requests().create().execute.return_value = {
 185                 "uuid": stubs.expect_container_request_uuid,
 186                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 187                 "state": "Queued"
 188             }
 189
 190             stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 191             stubs.api.pipeline_templates().create().execute.return_value = {
 192                 "uuid": stubs.expect_pipeline_template_uuid,
 193             }
 194             stubs.expect_job_spec = {
 195                 'runtime_constraints': {
 196                     'docker_image': '999999999999999999999999999999d3+99',
 197                     'min_ram_mb_per_node': 1024
 198                 },
 199                 'script_parameters': {
 200                     'x': {
 201                         'basename': 'blorp.txt',
 202                         'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 203                         'class': 'File'
 204                     },
 205                     'y': {
 206                         'basename': '99999999999999999999999999999998+99',
 207                         'location': 'keep:99999999999999999999999999999998+99',
 208                         'class': 'Directory'
 209                     },
 210                     'z': {
 211                         'basename': 'anonymous',
 212                         "listing": [{
 213                             "basename": "renamed.txt",
 214                             "class": "File",
 215                             "location": "keep:99999999999999999999999999999998+99/file1.txt",
 216                             "size": 0
 217                         }],
 218                         'class': 'Directory'
 219                     },
 220                     'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 221                 },
 222                 'repository': 'arvados',
 223                 'script_version': 'master',
 224                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 225                 'script': 'cwl-runner'
 226             }
 227             stubs.pipeline_component = stubs.expect_job_spec.copy()
 228             stubs.expect_pipeline_instance = {
 229                 'name': 'submit_wf.cwl',
 230                 'state': 'RunningOnServer',
 231                 'owner_uuid': None,
 232                 "components": {
 233                     "cwl-runner": {
 234                         'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 235                         'script_parameters': {
 236                             'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 237                             'x': {"value": {
 238                                 'basename': 'blorp.txt',
 239                                 'class': 'File',
 240                                 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 241                                 "size": 16
 242                             }},
 243                             'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 244                                   'listing': [
 245                                       {
 246                                           'basename': 'renamed.txt',
 247                                           'class': 'File', 'location':
 248                                           'keep:99999999999999999999999999999998+99/file1.txt',
 249                                           'size': 0
 250                                       }
 251                                   ]}},
 252                             'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 253                             'arv:debug': True,
 254                             'arv:enable_reuse': True,
 255                             'arv:on_error': 'continue'
 256                         },
 257                         'repository': 'arvados',
 258                         'script_version': 'master',
 259                         'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 260                         'script': 'cwl-runner',
 261                         'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 262                     }
 263                 }
 264             }
 265             stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 266             stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 267             stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 268             stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 269             stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 270                 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 271                 "state": "Queued"
 272             }
 273             stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 274             stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 275
 276             with open("tests/wf/submit_wf_packed.cwl") as f:
 277                 expect_packed_workflow = yaml.round_trip_load(f)
 278
 279             stubs.expect_container_spec = {
 280                 'priority': 500,
 281                 'mounts': {
 282                     '/var/spool/cwl': {
 283                         'writable': True,
 284                         'kind': 'collection'
 285                     },
 286                     '/var/lib/cwl/workflow.json': {
 287                         'content': expect_packed_workflow,
 288                         'kind': 'json'
 289                     },
 290                     'stdout': {
 291                         'path': '/var/spool/cwl/cwl.output.json',
 292                         'kind': 'file'
 293                     },
 294                     '/var/lib/cwl/cwl.input.json': {
 295                         'kind': 'json',
 296                         'content': {
 297                             'y': {
 298                                 'basename': '99999999999999999999999999999998+99',
 299                                 'location': 'keep:99999999999999999999999999999998+99',
 300                                 'class': 'Directory'},
 301                             'x': {
 302                                 'basename': u'blorp.txt',
 303                                 'class': 'File',
 304                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 305                                 "size": 16
 306                             },
 307                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 308                                 {'basename': 'renamed.txt',
 309                                  'class': 'File',
 310                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 311                                  'size': 0
 312                                 }
 313                             ]}
 314                         },
 315                         'kind': 'json'
 316                     }
 317                 },
 318                 'secret_mounts': {},
 319                 'state': 'Committed',
 320                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 321                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 322                             '--eval-timeout=20', '--thread-count=0',
 323                             '--enable-reuse', "--collection-cache-size=256",
 324                             '--output-name=Output from workflow '+wfname,
 325                             '--debug', '--on-error=continue',
 326                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 327                 'name': wfname,
 328                 'container_image': '999999999999999999999999999999d3+99',
 329                 'output_name': 'Output from workflow '+wfname,
 330                 'output_path': '/var/spool/cwl',
 331                 'cwd': '/var/spool/cwl',
 332                 'runtime_constraints': {
 333                     'API': True,
 334                     'vcpus': 1,
 335                     'ram': (1024+256)*1024*1024
 336                 },
 337                 'use_existing': False,
 338                 'properties': {},
 339                 'secret_mounts': {}
 340             }
 341
 342             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 343             stubs.api.workflows().create().execute.return_value = {
 344                 "uuid": stubs.expect_workflow_uuid,
 345             }
 346             def update_mock(**kwargs):
 347                 stubs.updated_uuid = kwargs.get('uuid')
 348                 return mock.DEFAULT
 349             stubs.api.workflows().update.side_effect = update_mock
 350             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 351                 "uuid": stubs.updated_uuid,
 352             }
 353
 354             return func(self, stubs, *args, **kwargs)
 355         return wrapped
 356     return outer_wrapper
 357
 358 class TestSubmit(unittest.TestCase):
 359
 360     def setUp(self):
 361         cwltool.process._names = set()
 362         arvados_cwl.arvdocker.arv_docker_clear_cache()
 363
 364     def tearDown(self):
 365         root_logger = logging.getLogger('')
 366
 367         # Remove existing RuntimeStatusLoggingHandlers if they exist
 368         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 369         root_logger.handlers = handlers
 370
 371     @mock.patch("time.sleep")
 372     @stubs
 373     def test_submit_invalid_runner_ram(self, stubs, tm):
 374         exited = arvados_cwl.main(
 375             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 376              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 377             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 378         self.assertEqual(exited, 1)
 379
 380
 381     @stubs
 382     def test_submit_container(self, stubs):
 383         exited = arvados_cwl.main(
 384             ["--submit", "--no-wait", "--api=containers", "--debug",
 385                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 386             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 387
 388         stubs.api.collections().create.assert_has_calls([
 389             mock.call(body=JsonDiffMatcher({
 390                 'manifest_text':
 391                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 392                 'replication_desired': None,
 393                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 394             }), ensure_unique_name=False),
 395             mock.call(body=JsonDiffMatcher({
 396                 'manifest_text':
 397                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 398                 'replication_desired': None,
 399                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 400             }), ensure_unique_name=False),
 401             ])
 402
 403         expect_container = copy.deepcopy(stubs.expect_container_spec)
 404         stubs.api.container_requests().create.assert_called_with(
 405             body=JsonDiffMatcher(expect_container))
 406         self.assertEqual(stubs.capture_stdout.getvalue(),
 407                          stubs.expect_container_request_uuid + '\n')
 408         self.assertEqual(exited, 0)
 409
 410
 411     @stubs
 412     def test_submit_container_tool(self, stubs):
 413         # test for issue #16139
 414         exited = arvados_cwl.main(
 415             ["--submit", "--no-wait", "--api=containers", "--debug",
 416                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 417             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 418
 419         self.assertEqual(stubs.capture_stdout.getvalue(),
 420                          stubs.expect_container_request_uuid + '\n')
 421         self.assertEqual(exited, 0)
 422
 423     @stubs
 424     def test_submit_container_no_reuse(self, stubs):
 425         exited = arvados_cwl.main(
 426             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 427                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 428             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 429
 430         expect_container = copy.deepcopy(stubs.expect_container_spec)
 431         expect_container["command"] = [
 432             'arvados-cwl-runner', '--local', '--api=containers',
 433             '--no-log-timestamps', '--disable-validate', '--disable-color',
 434             '--eval-timeout=20', '--thread-count=0',
 435             '--disable-reuse', "--collection-cache-size=256",
 436             "--output-name=Output from workflow submit_wf.cwl",
 437             '--debug', '--on-error=continue',
 438             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 439         expect_container["use_existing"] = False
 440
 441         stubs.api.container_requests().create.assert_called_with(
 442             body=JsonDiffMatcher(expect_container))
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_container_request_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @stubs('submit_wf_no_reuse.cwl')
 448     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 449         exited = arvados_cwl.main(
 450             ["--submit", "--no-wait", "--api=containers", "--debug",
 451              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 452             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 453         self.assertEqual(exited, 0)
 454
 455         expect_container = copy.deepcopy(stubs.expect_container_spec)
 456         expect_container["command"] = [
 457             'arvados-cwl-runner', '--local', '--api=containers',
 458             '--no-log-timestamps', '--disable-validate', '--disable-color',
 459             '--eval-timeout=20', '--thread-count=0',
 460             '--disable-reuse', "--collection-cache-size=256",
 461             '--output-name=Output from workflow submit_wf_no_reuse.cwl', '--debug', '--on-error=continue',
 462             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 463         expect_container["use_existing"] = False
 464         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 465             {
 466                 "class": "http://arvados.org/cwl#ReuseRequirement",
 467                 "enableReuse": False,
 468             },
 469         ]
 470         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 471             "arv": "http://arvados.org/cwl#",
 472             "cwltool": "http://commonwl.org/cwltool#"
 473         }
 474
 475         stubs.api.container_requests().create.assert_called_with(
 476             body=JsonDiffMatcher(expect_container))
 477         self.assertEqual(stubs.capture_stdout.getvalue(),
 478                          stubs.expect_container_request_uuid + '\n')
 479
 480
 481     @stubs
 482     def test_submit_container_on_error(self, stubs):
 483         exited = arvados_cwl.main(
 484             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 485                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 486             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 487
 488         expect_container = copy.deepcopy(stubs.expect_container_spec)
 489         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 490                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 491                                        '--eval-timeout=20', '--thread-count=0',
 492                                        '--enable-reuse', "--collection-cache-size=256",
 493                                        "--output-name=Output from workflow submit_wf.cwl",
 494                                        '--debug', '--on-error=stop',
 495                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 496
 497         stubs.api.container_requests().create.assert_called_with(
 498             body=JsonDiffMatcher(expect_container))
 499         self.assertEqual(stubs.capture_stdout.getvalue(),
 500                          stubs.expect_container_request_uuid + '\n')
 501         self.assertEqual(exited, 0)
 502
 503     @stubs
 504     def test_submit_container_output_name(self, stubs):
 505         output_name = "test_output_name"
 506
 507         exited = arvados_cwl.main(
 508             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 509                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 510             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 511
 512         expect_container = copy.deepcopy(stubs.expect_container_spec)
 513         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 514                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 515                                        '--eval-timeout=20', '--thread-count=0',
 516                                        '--enable-reuse', "--collection-cache-size=256",
 517                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 518                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 519         expect_container["output_name"] = output_name
 520
 521         stubs.api.container_requests().create.assert_called_with(
 522             body=JsonDiffMatcher(expect_container))
 523         self.assertEqual(stubs.capture_stdout.getvalue(),
 524                          stubs.expect_container_request_uuid + '\n')
 525         self.assertEqual(exited, 0)
 526
 527     @stubs
 528     def test_submit_storage_classes(self, stubs):
 529         exited = arvados_cwl.main(
 530             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 531                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533
 534         expect_container = copy.deepcopy(stubs.expect_container_spec)
 535         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 536                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 537                                        '--eval-timeout=20', '--thread-count=0',
 538                                        '--enable-reuse', "--collection-cache-size=256",
 539                                        '--output-name=Output from workflow submit_wf.cwl',
 540                                        "--debug",
 541                                        "--storage-classes=foo", '--on-error=continue',
 542                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 543
 544         stubs.api.container_requests().create.assert_called_with(
 545             body=JsonDiffMatcher(expect_container))
 546         self.assertEqual(stubs.capture_stdout.getvalue(),
 547                          stubs.expect_container_request_uuid + '\n')
 548         self.assertEqual(exited, 0)
 549
 550     @stubs
 551     def test_submit_multiple_storage_classes(self, stubs):
 552         exited = arvados_cwl.main(
 553             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 554                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 555             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 556
 557         expect_container = copy.deepcopy(stubs.expect_container_spec)
 558         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 559                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 560                                        '--eval-timeout=20', '--thread-count=0',
 561                                        '--enable-reuse', "--collection-cache-size=256",
 562                                        "--output-name=Output from workflow submit_wf.cwl",
 563                                        "--debug",
 564                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 565                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 566
 567         stubs.api.container_requests().create.assert_called_with(
 568             body=JsonDiffMatcher(expect_container))
 569         self.assertEqual(stubs.capture_stdout.getvalue(),
 570                          stubs.expect_container_request_uuid + '\n')
 571         self.assertEqual(exited, 0)
 572
 573     @mock.patch("cwltool.task_queue.TaskQueue")
 574     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 575     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 576     @stubs
 577     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 578         final_output_c = arvados.collection.Collection()
 579         make_output.return_value = ({},final_output_c)
 580
 581         def set_final_output(job_order, output_callback, runtimeContext):
 582             output_callback({"out": "zzzzz"}, "success")
 583             return []
 584         job.side_effect = set_final_output
 585
 586         exited = arvados_cwl.main(
 587             ["--debug", "--local", "--storage-classes=foo",
 588                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 589             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 590
 591         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 592         self.assertEqual(exited, 0)
 593
 594     @mock.patch("cwltool.task_queue.TaskQueue")
 595     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 596     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 597     @stubs
 598     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 599         final_output_c = arvados.collection.Collection()
 600         make_output.return_value = ({},final_output_c)
 601         stubs.api.config().get.return_value = {"default": {"Default": True}}
 602
 603         def set_final_output(job_order, output_callback, runtimeContext):
 604             output_callback({"out": "zzzzz"}, "success")
 605             return []
 606         job.side_effect = set_final_output
 607
 608         exited = arvados_cwl.main(
 609             ["--debug", "--local",
 610                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 611             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 612
 613         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 614         self.assertEqual(exited, 0)
 615
 616     @mock.patch("cwltool.task_queue.TaskQueue")
 617     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 618     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 619     @stubs
 620     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 621         final_output_c = arvados.collection.Collection()
 622         make_output.return_value = ({},final_output_c)
 623
 624         def set_final_output(job_order, output_callback, runtimeContext):
 625             output_callback({"out": "zzzzz"}, "success")
 626             return []
 627         job.side_effect = set_final_output
 628
 629         exited = arvados_cwl.main(
 630             ["--debug", "--local",
 631                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 632             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 633
 634         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 635         self.assertEqual(exited, 0)
 636
 637     @stubs
 638     def test_submit_container_output_ttl(self, stubs):
 639         exited = arvados_cwl.main(
 640             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 641                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 642             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 643
 644         expect_container = copy.deepcopy(stubs.expect_container_spec)
 645         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 646                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 647                                        '--eval-timeout=20', '--thread-count=0',
 648                                        '--enable-reuse', "--collection-cache-size=256",
 649                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 650                                        '--on-error=continue',
 651                                        "--intermediate-output-ttl=3600",
 652                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 653
 654         stubs.api.container_requests().create.assert_called_with(
 655             body=JsonDiffMatcher(expect_container))
 656         self.assertEqual(stubs.capture_stdout.getvalue(),
 657                          stubs.expect_container_request_uuid + '\n')
 658         self.assertEqual(exited, 0)
 659
 660     @stubs
 661     def test_submit_container_trash_intermediate(self, stubs):
 662         exited = arvados_cwl.main(
 663             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 664                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 665             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 666
 667
 668         expect_container = copy.deepcopy(stubs.expect_container_spec)
 669         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 670                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 671                                        '--eval-timeout=20', '--thread-count=0',
 672                                        '--enable-reuse', "--collection-cache-size=256",
 673                                        '--debug', '--on-error=continue',
 674                                        "--trash-intermediate",
 675                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 676
 677         stubs.api.container_requests().create.assert_called_with(
 678             body=JsonDiffMatcher(expect_container))
 679         self.assertEqual(stubs.capture_stdout.getvalue(),
 680                          stubs.expect_container_request_uuid + '\n')
 681         self.assertEqual(exited, 0)
 682
 683     @stubs
 684     def test_submit_container_output_tags(self, stubs):
 685         output_tags = "tag0,tag1,tag2"
 686
 687         exited = arvados_cwl.main(
 688             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 689                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 690             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 691
 692         expect_container = copy.deepcopy(stubs.expect_container_spec)
 693         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 694                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 695                                        '--eval-timeout=20', '--thread-count=0',
 696                                        '--enable-reuse', "--collection-cache-size=256",
 697                                        "--output-name=Output from workflow submit_wf.cwl",
 698                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 699                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 700
 701         stubs.api.container_requests().create.assert_called_with(
 702             body=JsonDiffMatcher(expect_container))
 703         self.assertEqual(stubs.capture_stdout.getvalue(),
 704                          stubs.expect_container_request_uuid + '\n')
 705         self.assertEqual(exited, 0)
 706
 707     @stubs
 708     def test_submit_container_runner_ram(self, stubs):
 709         exited = arvados_cwl.main(
 710             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 711                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 712             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 713
 714         expect_container = copy.deepcopy(stubs.expect_container_spec)
 715         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 716
 717         stubs.api.container_requests().create.assert_called_with(
 718             body=JsonDiffMatcher(expect_container))
 719         self.assertEqual(stubs.capture_stdout.getvalue(),
 720                          stubs.expect_container_request_uuid + '\n')
 721         self.assertEqual(exited, 0)
 722
 723     @mock.patch("arvados.collection.CollectionReader")
 724     @mock.patch("time.sleep")
 725     @stubs
 726     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 727         collectionReader().exists.return_value = True
 728         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 729         exited = arvados_cwl.main(
 730             ["--submit", "--no-wait", "--api=containers", "--debug",
 731              "tests/wf/submit_keepref_wf.cwl"],
 732             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 733         self.assertEqual(exited, 0)
 734
 735     @mock.patch("arvados.collection.CollectionReader")
 736     @mock.patch("time.sleep")
 737     @stubs
 738     def test_submit_keepref(self, stubs, tm, reader):
 739         with open("tests/wf/expect_arvworkflow.cwl") as f:
 740             reader().open().__enter__().read.return_value = f.read()
 741
 742         exited = arvados_cwl.main(
 743             ["--submit", "--no-wait", "--api=containers", "--debug",
 744              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 745             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 746
 747         expect_container = {
 748             'priority': 500,
 749             'mounts': {
 750                 '/var/spool/cwl': {
 751                     'writable': True,
 752                     'kind': 'collection'
 753                 },
 754                 'stdout': {
 755                     'path': '/var/spool/cwl/cwl.output.json',
 756                     'kind': 'file'
 757                 },
 758                 '/var/lib/cwl/workflow': {
 759                     'portable_data_hash': '99999999999999999999999999999994+99',
 760                     'kind': 'collection'
 761                 },
 762                 '/var/lib/cwl/cwl.input.json': {
 763                     'content': {
 764                         'x': 'XxX'
 765                     },
 766                     'kind': 'json'
 767                 }
 768             }, 'state': 'Committed',
 769             'output_path': '/var/spool/cwl',
 770             'name': 'expect_arvworkflow.cwl#main',
 771             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 772             'container_image': '999999999999999999999999999999d3+99',
 773             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 774                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 775                         '--eval-timeout=20', '--thread-count=0',
 776                         '--enable-reuse', "--collection-cache-size=256",
 777                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 778                         '--debug', '--on-error=continue',
 779                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 780             'cwd': '/var/spool/cwl',
 781             'runtime_constraints': {
 782                 'API': True,
 783                 'vcpus': 1,
 784                 'ram': 1342177280
 785             },
 786             'use_existing': False,
 787             'properties': {},
 788             'secret_mounts': {}
 789         }
 790
 791         stubs.api.container_requests().create.assert_called_with(
 792             body=JsonDiffMatcher(expect_container))
 793         self.assertEqual(stubs.capture_stdout.getvalue(),
 794                          stubs.expect_container_request_uuid + '\n')
 795         self.assertEqual(exited, 0)
 796
 797     @mock.patch("time.sleep")
 798     @stubs
 799     def test_submit_arvworkflow(self, stubs, tm):
 800         with open("tests/wf/expect_arvworkflow.cwl") as f:
 801             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 802
 803         exited = arvados_cwl.main(
 804             ["--submit", "--no-wait", "--api=containers", "--debug",
 805              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 806             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 807
 808         expect_container = {
 809             'priority': 500,
 810             'mounts': {
 811                 '/var/spool/cwl': {
 812                     'writable': True,
 813                     'kind': 'collection'
 814                 },
 815                 'stdout': {
 816                     'path': '/var/spool/cwl/cwl.output.json',
 817                     'kind': 'file'
 818                 },
 819                 '/var/lib/cwl/workflow.json': {
 820                     'kind': 'json',
 821                     'content': {
 822                         'cwlVersion': 'v1.0',
 823                         '$graph': [
 824                             {
 825                                 'id': '#main',
 826                                 'inputs': [
 827                                     {'type': 'string', 'id': '#main/x'}
 828                                 ],
 829                                 'steps': [
 830                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 831                                      'run': '#submit_tool.cwl',
 832                                      'id': '#main/step1',
 833                                      'out': []}
 834                                 ],
 835                                 'class': 'Workflow',
 836                                 'outputs': []
 837                             },
 838                             {
 839                                 'inputs': [
 840                                     {
 841                                         'inputBinding': {'position': 1},
 842                                         'type': 'string',
 843                                         'id': '#submit_tool.cwl/x'}
 844                                 ],
 845                                 'requirements': [
 846                                     {
 847                                         'dockerPull': 'debian:buster-slim',
 848                                         'class': 'DockerRequirement',
 849                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 850                                     }
 851                                 ],
 852                                 'id': '#submit_tool.cwl',
 853                                 'outputs': [],
 854                                 'baseCommand': 'cat',
 855                                 'class': 'CommandLineTool'
 856                             }
 857                         ]
 858                     }
 859                 },
 860                 '/var/lib/cwl/cwl.input.json': {
 861                     'content': {
 862                         'x': 'XxX'
 863                     },
 864                     'kind': 'json'
 865                 }
 866             }, 'state': 'Committed',
 867             'output_path': '/var/spool/cwl',
 868             'name': 'a test workflow',
 869             'container_image': "999999999999999999999999999999d3+99",
 870             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 871                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 872                         '--eval-timeout=20', '--thread-count=0',
 873                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 874                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 875             'cwd': '/var/spool/cwl',
 876             'runtime_constraints': {
 877                 'API': True,
 878                 'vcpus': 1,
 879                 'ram': 1342177280
 880             },
 881             'use_existing': False,
 882             'properties': {
 883                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 884             },
 885             'secret_mounts': {}
 886         }
 887
 888         stubs.api.container_requests().create.assert_called_with(
 889             body=JsonDiffMatcher(expect_container))
 890         self.assertEqual(stubs.capture_stdout.getvalue(),
 891                          stubs.expect_container_request_uuid + '\n')
 892         self.assertEqual(exited, 0)
 893
 894     @stubs('hello container 123')
 895     def test_submit_container_name(self, stubs):
 896         exited = arvados_cwl.main(
 897             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 898                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 899             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 900
 901         expect_container = copy.deepcopy(stubs.expect_container_spec)
 902
 903         stubs.api.container_requests().create.assert_called_with(
 904             body=JsonDiffMatcher(expect_container))
 905         self.assertEqual(stubs.capture_stdout.getvalue(),
 906                          stubs.expect_container_request_uuid + '\n')
 907         self.assertEqual(exited, 0)
 908
 909     @stubs
 910     def test_submit_missing_input(self, stubs):
 911         exited = arvados_cwl.main(
 912             ["--submit", "--no-wait", "--api=containers", "--debug",
 913              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 914             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 915         self.assertEqual(exited, 0)
 916
 917         exited = arvados_cwl.main(
 918             ["--submit", "--no-wait", "--api=containers", "--debug",
 919              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 920             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 921         self.assertEqual(exited, 1)
 922
 923     @stubs
 924     def test_submit_container_project(self, stubs):
 925         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 926         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 927         exited = arvados_cwl.main(
 928             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 929                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 930             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 931
 932         expect_container = copy.deepcopy(stubs.expect_container_spec)
 933         expect_container["owner_uuid"] = project_uuid
 934         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 935                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 936                                        "--eval-timeout=20", "--thread-count=0",
 937                                        '--enable-reuse', "--collection-cache-size=256",
 938                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 939                                        '--on-error=continue',
 940                                        '--project-uuid='+project_uuid,
 941                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 942
 943         stubs.api.container_requests().create.assert_called_with(
 944             body=JsonDiffMatcher(expect_container))
 945         self.assertEqual(stubs.capture_stdout.getvalue(),
 946                          stubs.expect_container_request_uuid + '\n')
 947         self.assertEqual(exited, 0)
 948
 949     @stubs
 950     def test_submit_container_eval_timeout(self, stubs):
 951         exited = arvados_cwl.main(
 952             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 953                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 954             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 955
 956         expect_container = copy.deepcopy(stubs.expect_container_spec)
 957         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 958                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 959                                        '--eval-timeout=60.0', '--thread-count=0',
 960                                        '--enable-reuse', "--collection-cache-size=256",
 961                                        '--debug', '--on-error=continue',
 962                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 963
 964         stubs.api.container_requests().create.assert_called_with(
 965             body=JsonDiffMatcher(expect_container))
 966         self.assertEqual(stubs.capture_stdout.getvalue(),
 967                          stubs.expect_container_request_uuid + '\n')
 968         self.assertEqual(exited, 0)
 969
 970     @stubs
 971     def test_submit_container_collection_cache(self, stubs):
 972         exited = arvados_cwl.main(
 973             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 974                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 975             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 976
 977         expect_container = copy.deepcopy(stubs.expect_container_spec)
 978         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 979                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 980                                        '--eval-timeout=20', '--thread-count=0',
 981                                        '--enable-reuse', "--collection-cache-size=500",
 982                                        '--debug', '--on-error=continue',
 983                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 984         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 985
 986         stubs.api.container_requests().create.assert_called_with(
 987             body=JsonDiffMatcher(expect_container))
 988         self.assertEqual(stubs.capture_stdout.getvalue(),
 989                          stubs.expect_container_request_uuid + '\n')
 990         self.assertEqual(exited, 0)
 991
 992     @stubs
 993     def test_submit_container_thread_count(self, stubs):
 994         exited = arvados_cwl.main(
 995             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 996                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 997             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 998
 999         expect_container = copy.deepcopy(stubs.expect_container_spec)
1000         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1001                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1002                                        '--eval-timeout=20', '--thread-count=20',
1003                                        '--enable-reuse', "--collection-cache-size=256",
1004                                        '--debug', '--on-error=continue',
1005                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1006
1007         stubs.api.container_requests().create.assert_called_with(
1008             body=JsonDiffMatcher(expect_container))
1009         self.assertEqual(stubs.capture_stdout.getvalue(),
1010                          stubs.expect_container_request_uuid + '\n')
1011         self.assertEqual(exited, 0)
1012
1013     @stubs
1014     def test_submit_container_runner_image(self, stubs):
1015         exited = arvados_cwl.main(
1016             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1017                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1018             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1019
1020         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1021
1022         expect_container = copy.deepcopy(stubs.expect_container_spec)
1023         stubs.api.container_requests().create.assert_called_with(
1024             body=JsonDiffMatcher(expect_container))
1025         self.assertEqual(stubs.capture_stdout.getvalue(),
1026                          stubs.expect_container_request_uuid + '\n')
1027         self.assertEqual(exited, 0)
1028
1029     @stubs
1030     def test_submit_priority(self, stubs):
1031         exited = arvados_cwl.main(
1032             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1033                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1034             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1035
1036         stubs.expect_container_spec["priority"] = 669
1037
1038         expect_container = copy.deepcopy(stubs.expect_container_spec)
1039         stubs.api.container_requests().create.assert_called_with(
1040             body=JsonDiffMatcher(expect_container))
1041         self.assertEqual(stubs.capture_stdout.getvalue(),
1042                          stubs.expect_container_request_uuid + '\n')
1043         self.assertEqual(exited, 0)
1044
1045     @stubs('submit_wf_runner_resources.cwl')
1046     def test_submit_wf_runner_resources(self, stubs):
1047         exited = arvados_cwl.main(
1048             ["--submit", "--no-wait", "--api=containers", "--debug",
1049                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1050             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1051
1052         expect_container = copy.deepcopy(stubs.expect_container_spec)
1053         expect_container["runtime_constraints"] = {
1054             "API": True,
1055             "vcpus": 2,
1056             "ram": (2000+512) * 2**20
1057         }
1058         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1059             {
1060                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1061                 "coresMin": 2,
1062                 "ramMin": 2000,
1063                 "keep_cache": 512
1064             }
1065         ]
1066         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1067             "arv": "http://arvados.org/cwl#",
1068         }
1069         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1070                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1071                         '--eval-timeout=20', '--thread-count=0',
1072                         '--enable-reuse', "--collection-cache-size=512",
1073                                        '--output-name=Output from workflow submit_wf_runner_resources.cwl',
1074                                        '--debug', '--on-error=continue',
1075                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1076
1077         stubs.api.container_requests().create.assert_called_with(
1078             body=JsonDiffMatcher(expect_container))
1079         self.assertEqual(stubs.capture_stdout.getvalue(),
1080                          stubs.expect_container_request_uuid + '\n')
1081         self.assertEqual(exited, 0)
1082
1083     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1084     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1085     @mock.patch("arvados.api")
1086     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1087         arvados_cwl.arvdocker.arv_docker_clear_cache()
1088
1089         arvrunner = mock.MagicMock()
1090         arvrunner.project_uuid = ""
1091         api.return_value = mock.MagicMock()
1092         arvrunner.api = api.return_value
1093         arvrunner.runtimeContext.match_local_docker = False
1094         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1095                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1096                                                                         "link_class": "docker_image_repo+tag",
1097                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1098                                                                         "owner_uuid": "",
1099                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1100                                                             {"items": [{"created_at": "",
1101                                                                         "head_uuid": "",
1102                                                                         "link_class": "docker_image_hash",
1103                                                                         "name": "123456",
1104                                                                         "owner_uuid": "",
1105                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1106                                                             {"items": [{"created_at": "",
1107                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1108                                                                         "link_class": "docker_image_repo+tag",
1109                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1110                                                                         "owner_uuid": "",
1111                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1112                                                             {"items": [{"created_at": "",
1113                                                                         "head_uuid": "",
1114                                                                         "link_class": "docker_image_hash",
1115                                                                         "name": "123456",
1116                                                                         "owner_uuid": "",
1117                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1118         )
1119         find_one_image_hash.return_value = "123456"
1120
1121         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1122                                                                               "owner_uuid": "",
1123                                                                               "manifest_text": "",
1124                                                                               "properties": ""
1125                                                                               }], "items_available": 1, "offset": 0},
1126                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1127                                                                               "owner_uuid": "",
1128                                                                               "manifest_text": "",
1129                                                                               "properties": ""
1130                                                                           }], "items_available": 1, "offset": 0})
1131         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1132         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1133                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1134
1135         self.assertEqual("9999999999999999999999999999999b+99",
1136                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1137
1138
1139     @stubs
1140     def test_submit_secrets(self, stubs):
1141         exited = arvados_cwl.main(
1142             ["--submit", "--no-wait", "--api=containers", "--debug",
1143                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1144             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1145
1146         expect_container = {
1147             "command": [
1148                 "arvados-cwl-runner",
1149                 "--local",
1150                 "--api=containers",
1151                 "--no-log-timestamps",
1152                 "--disable-validate",
1153                 "--disable-color",
1154                 "--eval-timeout=20",
1155                 '--thread-count=0',
1156                 "--enable-reuse",
1157                 "--collection-cache-size=256",
1158                 '--output-name=Output from workflow secret_wf.cwl'
1159                 '--debug',
1160                 "--on-error=continue",
1161                 "/var/lib/cwl/workflow.json#main",
1162                 "/var/lib/cwl/cwl.input.json"
1163             ],
1164             "container_image": "999999999999999999999999999999d3+99",
1165             "cwd": "/var/spool/cwl",
1166             "mounts": {
1167                 "/var/lib/cwl/cwl.input.json": {
1168                     "content": {
1169                         "pw": {
1170                             "$include": "/secrets/s0"
1171                         }
1172                     },
1173                     "kind": "json"
1174                 },
1175                 "/var/lib/cwl/workflow.json": {
1176                     "content": {
1177                         "$graph": [
1178                             {
1179                                 "arguments": [
1180                                     "md5sum",
1181                                     "example.conf"
1182                                 ],
1183                                 "class": "CommandLineTool",
1184                                 "hints": [
1185                                     {
1186                                         "class": "http://commonwl.org/cwltool#Secrets",
1187                                         "secrets": [
1188                                             "#secret_job.cwl/pw"
1189                                         ]
1190                                     }
1191                                 ],
1192                                 "id": "#secret_job.cwl",
1193                                 "inputs": [
1194                                     {
1195                                         "id": "#secret_job.cwl/pw",
1196                                         "type": "string"
1197                                     }
1198                                 ],
1199                                 "outputs": [
1200                                     {
1201                                         "id": "#secret_job.cwl/out",
1202                                         "type": "File",
1203                                         "outputBinding": {
1204                                               "glob": "hashed_example.txt"
1205                                         }
1206                                     }
1207                                 ],
1208                                 "stdout": "hashed_example.txt",
1209                                 "requirements": [
1210                                     {
1211                                         "class": "InitialWorkDirRequirement",
1212                                         "listing": [
1213                                             {
1214                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1215                                                 "entryname": "example.conf"
1216                                             }
1217                                         ]
1218                                     }
1219                                 ]
1220                             },
1221                             {
1222                                 "class": "Workflow",
1223                                 "hints": [
1224                                     {
1225                                         "class": "DockerRequirement",
1226                                         "dockerPull": "debian:buster-slim",
1227                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1228                                     },
1229                                     {
1230                                         "class": "http://commonwl.org/cwltool#Secrets",
1231                                         "secrets": [
1232                                             "#main/pw"
1233                                         ]
1234                                     }
1235                                 ],
1236                                 "id": "#main",
1237                                 "inputs": [
1238                                     {
1239                                         "id": "#main/pw",
1240                                         "type": "string"
1241                                     }
1242                                 ],
1243                                 "outputs": [
1244                                     {
1245                                         "id": "#main/out",
1246                                         "outputSource": "#main/step1/out",
1247                                         "type": "File"
1248                                     }
1249                                 ],
1250                                 "steps": [
1251                                     {
1252                                         "id": "#main/step1",
1253                                         "in": [
1254                                             {
1255                                                 "id": "#main/step1/pw",
1256                                                 "source": "#main/pw"
1257                                             }
1258                                         ],
1259                                         "out": [
1260                                             "#main/step1/out"
1261                                         ],
1262                                         "run": "#secret_job.cwl"
1263                                     }
1264                                 ]
1265                             }
1266                         ],
1267                         "$namespaces": {
1268                             "cwltool": "http://commonwl.org/cwltool#"
1269                         },
1270                         "cwlVersion": "v1.0"
1271                     },
1272                     "kind": "json"
1273                 },
1274                 "/var/spool/cwl": {
1275                     "kind": "collection",
1276                     "writable": True
1277                 },
1278                 "stdout": {
1279                     "kind": "file",
1280                     "path": "/var/spool/cwl/cwl.output.json"
1281                 }
1282             },
1283             "name": "secret_wf.cwl",
1284             "output_name": "Output from workflow secret_wf.cwl",
1285             "output_path": "/var/spool/cwl",
1286             "priority": 500,
1287             "properties": {},
1288             "runtime_constraints": {
1289                 "API": True,
1290                 "ram": 1342177280,
1291                 "vcpus": 1
1292             },
1293             "secret_mounts": {
1294                 "/secrets/s0": {
1295                     "content": "blorp",
1296                     "kind": "text"
1297                 }
1298             },
1299             "state": "Committed",
1300             "use_existing": False
1301         }
1302
1303         stubs.api.container_requests().create.assert_called_with(
1304             body=JsonDiffMatcher(expect_container))
1305         self.assertEqual(stubs.capture_stdout.getvalue(),
1306                          stubs.expect_container_request_uuid + '\n')
1307         self.assertEqual(exited, 0)
1308
1309     @stubs
1310     def test_submit_request_uuid(self, stubs):
1311         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1312         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1313
1314         stubs.api.container_requests().update().execute.return_value = {
1315             "uuid": stubs.expect_container_request_uuid,
1316             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1317             "state": "Queued"
1318         }
1319
1320         exited = arvados_cwl.main(
1321             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1322                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1323             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1324
1325         stubs.api.container_requests().update.assert_called_with(
1326             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1327         self.assertEqual(stubs.capture_stdout.getvalue(),
1328                          stubs.expect_container_request_uuid + '\n')
1329         self.assertEqual(exited, 0)
1330
1331     @stubs
1332     def test_submit_container_cluster_id(self, stubs):
1333         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1334
1335         exited = arvados_cwl.main(
1336             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1337                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1338             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1339
1340         expect_container = copy.deepcopy(stubs.expect_container_spec)
1341
1342         stubs.api.container_requests().create.assert_called_with(
1343             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1344         self.assertEqual(stubs.capture_stdout.getvalue(),
1345                          stubs.expect_container_request_uuid + '\n')
1346         self.assertEqual(exited, 0)
1347
1348     @stubs
1349     def test_submit_validate_cluster_id(self, stubs):
1350         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1351         exited = arvados_cwl.main(
1352             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1353              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1354             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1355         self.assertEqual(exited, 1)
1356
1357     @stubs
1358     def test_submit_validate_project_uuid(self, stubs):
1359         # Fails with bad cluster prefix
1360         exited = arvados_cwl.main(
1361             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1362              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1363             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1364         self.assertEqual(exited, 1)
1365
1366         # Project lookup fails
1367         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1368         exited = arvados_cwl.main(
1369             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1370              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1371             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1372         self.assertEqual(exited, 1)
1373
1374         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1375         exited = arvados_cwl.main(
1376             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1377              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1378             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1379         self.assertEqual(exited, 0)
1380
1381
1382     @mock.patch("arvados.collection.CollectionReader")
1383     @stubs
1384     def test_submit_uuid_inputs(self, stubs, collectionReader):
1385         collectionReader().exists.return_value = True
1386         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1387         def list_side_effect(**kwargs):
1388             m = mock.MagicMock()
1389             if "count" in kwargs:
1390                 m.execute.return_value = {"items": [
1391                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1392                 ]}
1393             else:
1394                 m.execute.return_value = {"items": []}
1395             return m
1396         stubs.api.collections().list.side_effect = list_side_effect
1397
1398         exited = arvados_cwl.main(
1399             ["--submit", "--no-wait", "--api=containers", "--debug",
1400                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1401             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1402
1403         expect_container = copy.deepcopy(stubs.expect_container_spec)
1404         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1405         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1406         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1407
1408         stubs.api.collections().list.assert_has_calls([
1409             mock.call(count='none',
1410                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1411                       select=['uuid', 'portable_data_hash'])])
1412         stubs.api.container_requests().create.assert_called_with(
1413             body=JsonDiffMatcher(expect_container))
1414         self.assertEqual(stubs.capture_stdout.getvalue(),
1415                          stubs.expect_container_request_uuid + '\n')
1416         self.assertEqual(exited, 0)
1417
1418     @stubs
1419     def test_submit_mismatched_uuid_inputs(self, stubs):
1420         def list_side_effect(**kwargs):
1421             m = mock.MagicMock()
1422             if "count" in kwargs:
1423                 m.execute.return_value = {"items": [
1424                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1425                 ]}
1426             else:
1427                 m.execute.return_value = {"items": []}
1428             return m
1429         stubs.api.collections().list.side_effect = list_side_effect
1430
1431         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1432             capture_stderr = StringIO()
1433             cwltool_logger = logging.getLogger('cwltool')
1434             stderr_logger = logging.StreamHandler(capture_stderr)
1435             cwltool_logger.addHandler(stderr_logger)
1436
1437             try:
1438                 exited = arvados_cwl.main(
1439                     ["--submit", "--no-wait", "--api=containers", "--debug",
1440                         "tests/wf/submit_wf.cwl", infile],
1441                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1442
1443                 self.assertEqual(exited, 1)
1444                 self.assertRegex(
1445                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1446                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1447             finally:
1448                 cwltool_logger.removeHandler(stderr_logger)
1449
1450     @mock.patch("arvados.collection.CollectionReader")
1451     @stubs
1452     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1453         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1454         capture_stderr = StringIO()
1455
1456         cwltool_logger = logging.getLogger('cwltool')
1457         stderr_logger = logging.StreamHandler(capture_stderr)
1458         cwltool_logger.addHandler(stderr_logger)
1459
1460         exited = arvados_cwl.main(
1461             ["--submit", "--no-wait", "--api=containers", "--debug",
1462                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1463             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1464
1465         try:
1466             self.assertEqual(exited, 1)
1467             self.assertRegex(
1468                 capture_stderr.getvalue(),
1469                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1470         finally:
1471             cwltool_logger.removeHandler(stderr_logger)
1472
1473     @stubs('submit_wf_process_properties.cwl')
1474     def test_submit_set_process_properties(self, stubs):
1475         exited = arvados_cwl.main(
1476             ["--submit", "--no-wait", "--api=containers", "--debug",
1477                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1478             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1479
1480         expect_container = copy.deepcopy(stubs.expect_container_spec)
1481
1482         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1483             {
1484                 "class": "http://arvados.org/cwl#ProcessProperties",
1485                 "processProperties": [
1486                     {"propertyName": "baz",
1487                      "propertyValue": "$(inputs.x.basename)"},
1488                     {"propertyName": "foo",
1489                      "propertyValue": "bar"},
1490                     {"propertyName": "quux",
1491                      "propertyValue": {
1492                          "q1": 1,
1493                          "q2": 2
1494                      }
1495                     }
1496                 ],
1497             }
1498         ]
1499         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1500             "arv": "http://arvados.org/cwl#"
1501         }
1502
1503         expect_container["properties"] = {
1504             "baz": "blorp.txt",
1505             "foo": "bar",
1506             "quux": {
1507                 "q1": 1,
1508                 "q2": 2
1509             }
1510         }
1511
1512         stubs.api.container_requests().create.assert_called_with(
1513             body=JsonDiffMatcher(expect_container))
1514         self.assertEqual(stubs.capture_stdout.getvalue(),
1515                          stubs.expect_container_request_uuid + '\n')
1516         self.assertEqual(exited, 0)
1517
1518
1519     @stubs
1520     def test_submit_enable_preemptible(self, stubs):
1521         exited = arvados_cwl.main(
1522             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1523                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1524             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1525
1526         expect_container = copy.deepcopy(stubs.expect_container_spec)
1527         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1528                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1529                         '--eval-timeout=20', '--thread-count=0',
1530                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1531                                        '--enable-preemptible',
1532                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1533
1534         stubs.api.container_requests().create.assert_called_with(
1535             body=JsonDiffMatcher(expect_container))
1536         self.assertEqual(stubs.capture_stdout.getvalue(),
1537                          stubs.expect_container_request_uuid + '\n')
1538         self.assertEqual(exited, 0)
1539
1540     @stubs
1541     def test_submit_disable_preemptible(self, stubs):
1542         exited = arvados_cwl.main(
1543             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1545             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1546
1547         expect_container = copy.deepcopy(stubs.expect_container_spec)
1548         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1549                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1550                         '--eval-timeout=20', '--thread-count=0',
1551                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1552                                        '--disable-preemptible',
1553                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1554
1555         stubs.api.container_requests().create.assert_called_with(
1556             body=JsonDiffMatcher(expect_container))
1557         self.assertEqual(stubs.capture_stdout.getvalue(),
1558                          stubs.expect_container_request_uuid + '\n')
1559         self.assertEqual(exited, 0)
1560
1561
1562 class TestCreateWorkflow(unittest.TestCase):
1563     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1564     expect_workflow = StripYAMLComments(
1565         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1566
1567     def setUp(self):
1568         cwltool.process._names = set()
1569         arvados_cwl.arvdocker.arv_docker_clear_cache()
1570
1571     def tearDown(self):
1572         root_logger = logging.getLogger('')
1573
1574         # Remove existing RuntimeStatusLoggingHandlers if they exist
1575         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1576         root_logger.handlers = handlers
1577
1578     @stubs
1579     def test_create(self, stubs):
1580         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1581         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1582
1583         exited = arvados_cwl.main(
1584             ["--create-workflow", "--debug",
1585              "--api=containers",
1586              "--project-uuid", project_uuid,
1587              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1588             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1589
1590         stubs.api.pipeline_templates().create.refute_called()
1591         stubs.api.container_requests().create.refute_called()
1592
1593         body = {
1594             "workflow": {
1595                 "owner_uuid": project_uuid,
1596                 "name": "submit_wf.cwl",
1597                 "description": "",
1598                 "definition": self.expect_workflow,
1599             }
1600         }
1601         stubs.api.workflows().create.assert_called_with(
1602             body=JsonDiffMatcher(body))
1603
1604         self.assertEqual(stubs.capture_stdout.getvalue(),
1605                          stubs.expect_workflow_uuid + '\n')
1606         self.assertEqual(exited, 0)
1607
1608     @stubs
1609     def test_create_name(self, stubs):
1610         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1611         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1612
1613         exited = arvados_cwl.main(
1614             ["--create-workflow", "--debug",
1615              "--api=containers",
1616              "--project-uuid", project_uuid,
1617              "--name", "testing 123",
1618              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1619             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1620
1621         stubs.api.pipeline_templates().create.refute_called()
1622         stubs.api.container_requests().create.refute_called()
1623
1624         body = {
1625             "workflow": {
1626                 "owner_uuid": project_uuid,
1627                 "name": "testing 123",
1628                 "description": "",
1629                 "definition": self.expect_workflow,
1630             }
1631         }
1632         stubs.api.workflows().create.assert_called_with(
1633             body=JsonDiffMatcher(body))
1634
1635         self.assertEqual(stubs.capture_stdout.getvalue(),
1636                          stubs.expect_workflow_uuid + '\n')
1637         self.assertEqual(exited, 0)
1638
1639
1640     @stubs
1641     def test_update(self, stubs):
1642         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1643         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1644
1645         exited = arvados_cwl.main(
1646             ["--update-workflow", self.existing_workflow_uuid,
1647              "--debug",
1648              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1649             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1650
1651         body = {
1652             "workflow": {
1653                 "name": "submit_wf.cwl",
1654                 "description": "",
1655                 "definition": self.expect_workflow,
1656                 "owner_uuid": project_uuid
1657             }
1658         }
1659         stubs.api.workflows().update.assert_called_with(
1660             uuid=self.existing_workflow_uuid,
1661             body=JsonDiffMatcher(body))
1662         self.assertEqual(stubs.capture_stdout.getvalue(),
1663                          self.existing_workflow_uuid + '\n')
1664         self.assertEqual(exited, 0)
1665
1666
1667     @stubs
1668     def test_update_name(self, stubs):
1669         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1670         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1671
1672         exited = arvados_cwl.main(
1673             ["--update-workflow", self.existing_workflow_uuid,
1674              "--debug", "--name", "testing 123",
1675              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1676             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1677
1678         body = {
1679             "workflow": {
1680                 "name": "testing 123",
1681                 "description": "",
1682                 "definition": self.expect_workflow,
1683                 "owner_uuid": project_uuid
1684             }
1685         }
1686         stubs.api.workflows().update.assert_called_with(
1687             uuid=self.existing_workflow_uuid,
1688             body=JsonDiffMatcher(body))
1689         self.assertEqual(stubs.capture_stdout.getvalue(),
1690                          self.existing_workflow_uuid + '\n')
1691         self.assertEqual(exited, 0)
1692
1693     @stubs
1694     def test_create_collection_per_tool(self, stubs):
1695         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1696         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1697
1698         exited = arvados_cwl.main(
1699             ["--create-workflow", "--debug",
1700              "--api=containers",
1701              "--project-uuid", project_uuid,
1702              "tests/collection_per_tool/collection_per_tool.cwl"],
1703             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1704
1705         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1706         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1707
1708         body = {
1709             "workflow": {
1710                 "owner_uuid": project_uuid,
1711                 "name": "collection_per_tool.cwl",
1712                 "description": "",
1713                 "definition": expect_workflow,
1714             }
1715         }
1716         stubs.api.workflows().create.assert_called_with(
1717             body=JsonDiffMatcher(body))
1718
1719         self.assertEqual(stubs.capture_stdout.getvalue(),
1720                          stubs.expect_workflow_uuid + '\n')
1721         self.assertEqual(exited, 0)
1722
1723     @stubs
1724     def test_create_with_imports(self, stubs):
1725         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1726         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1727
1728         exited = arvados_cwl.main(
1729             ["--create-workflow", "--debug",
1730              "--api=containers",
1731              "--project-uuid", project_uuid,
1732              "tests/wf/feddemo/feddemo.cwl"],
1733             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1734
1735         stubs.api.pipeline_templates().create.refute_called()
1736         stubs.api.container_requests().create.refute_called()
1737
1738         self.assertEqual(stubs.capture_stdout.getvalue(),
1739                          stubs.expect_workflow_uuid + '\n')
1740         self.assertEqual(exited, 0)
1741
1742     @stubs
1743     def test_create_with_no_input(self, stubs):
1744         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1745         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1746
1747         exited = arvados_cwl.main(
1748             ["--create-workflow", "--debug",
1749              "--api=containers",
1750              "--project-uuid", project_uuid,
1751              "tests/wf/revsort/revsort.cwl"],
1752             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1753
1754         stubs.api.pipeline_templates().create.refute_called()
1755         stubs.api.container_requests().create.refute_called()
1756
1757         self.assertEqual(stubs.capture_stdout.getvalue(),
1758                          stubs.expect_workflow_uuid + '\n')
1759         self.assertEqual(exited, 0)