sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml as yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfname='submit_wf.cwl'):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62             stubs = Stubs()
  63             stubs.events = events
  64             stubs.keepdocker = keepdocker
  65
  66             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  67                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  68                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  69
  70             determine_image_id.return_value = None
  71
  72             def putstub(p, **kwargs):
  73                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  74             keep_client1().put.side_effect = putstub
  75             keep_client1.put.side_effect = putstub
  76             keep_client2().put.side_effect = putstub
  77             keep_client2.put.side_effect = putstub
  78
  79             stubs.keep_client = keep_client2
  80             stubs.docker_images = {
  81                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  82                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  83                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  84                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  85             }
  86             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  87                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  88             stubs.keepdocker.side_effect = kd
  89
  90             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  91             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  92
  93             if sys.version_info[0] < 3:
  94                 stubs.capture_stdout = BytesIO()
  95             else:
  96                 stubs.capture_stdout = StringIO()
  97
  98             stubs.api = mock.MagicMock()
  99             stubs.api._rootDesc = get_rootDesc()
 100             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 101             stubs.api._rootDesc["revision"] = "20210628"
 102
 103             stubs.api.users().current().execute.return_value = {
 104                 "uuid": stubs.fake_user_uuid,
 105             }
 106             stubs.api.collections().list().execute.return_value = {"items": []}
 107             stubs.api.containers().current().execute.return_value = {
 108                 "uuid": stubs.fake_container_uuid,
 109             }
 110             stubs.api.config()["StorageClasses"].items.return_value = {
 111                 "default": {
 112                     "Default": True
 113                 }
 114             }.items()
 115
 116             class CollectionExecute(object):
 117                 def __init__(self, exe):
 118                     self.exe = exe
 119                 def execute(self, num_retries=None):
 120                     return self.exe
 121
 122             def collection_createstub(created_collections, body, ensure_unique_name=None):
 123                 mt = body["manifest_text"].encode('utf-8')
 124                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 125                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 126                 created_collections[uuid] = {
 127                     "uuid": uuid,
 128                     "portable_data_hash": pdh,
 129                     "manifest_text": mt.decode('utf-8')
 130                 }
 131                 return CollectionExecute(created_collections[uuid])
 132
 133             def collection_getstub(created_collections, uuid):
 134                 for v in viewvalues(created_collections):
 135                     if uuid in (v["uuid"], v["portable_data_hash"]):
 136                         return CollectionExecute(v)
 137
 138             created_collections = {
 139                 "99999999999999999999999999999998+99": {
 140                     "uuid": "",
 141                     "portable_data_hash": "99999999999999999999999999999998+99",
 142                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 143                 },
 144                 "99999999999999999999999999999997+99": {
 145                     "uuid": "",
 146                     "portable_data_hash": "99999999999999999999999999999997+99",
 147                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 148                 },
 149                 "99999999999999999999999999999994+99": {
 150                     "uuid": "",
 151                     "portable_data_hash": "99999999999999999999999999999994+99",
 152                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 153                 },
 154                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 155                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 156                     "portable_data_hash": "999999999999999999999999999999d3+99",
 157                     "manifest_text": ""
 158                 },
 159                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 160                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 161                     "portable_data_hash": "999999999999999999999999999999d4+99",
 162                     "manifest_text": ""
 163                 },
 164                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 165                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 166                     "portable_data_hash": "999999999999999999999999999999d5+99",
 167                     "manifest_text": ""
 168                 },
 169                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 170                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 171                     "portable_data_hash": "999999999999999999999999999999d6+99",
 172                     "manifest_text": ""
 173                 }
 174             }
 175             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 176             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 177
 178             stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 179             stubs.api.jobs().create().execute.return_value = {
 180                 "uuid": stubs.expect_job_uuid,
 181                 "state": "Queued",
 182             }
 183
 184             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 185             stubs.api.container_requests().create().execute.return_value = {
 186                 "uuid": stubs.expect_container_request_uuid,
 187                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 188                 "state": "Queued"
 189             }
 190
 191             stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 192             stubs.api.pipeline_templates().create().execute.return_value = {
 193                 "uuid": stubs.expect_pipeline_template_uuid,
 194             }
 195             stubs.expect_job_spec = {
 196                 'runtime_constraints': {
 197                     'docker_image': '999999999999999999999999999999d3+99',
 198                     'min_ram_mb_per_node': 1024
 199                 },
 200                 'script_parameters': {
 201                     'x': {
 202                         'basename': 'blorp.txt',
 203                         'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 204                         'class': 'File'
 205                     },
 206                     'y': {
 207                         'basename': '99999999999999999999999999999998+99',
 208                         'location': 'keep:99999999999999999999999999999998+99',
 209                         'class': 'Directory'
 210                     },
 211                     'z': {
 212                         'basename': 'anonymous',
 213                         "listing": [{
 214                             "basename": "renamed.txt",
 215                             "class": "File",
 216                             "location": "keep:99999999999999999999999999999998+99/file1.txt",
 217                             "size": 0
 218                         }],
 219                         'class': 'Directory'
 220                     },
 221                     'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 222                 },
 223                 'repository': 'arvados',
 224                 'script_version': 'master',
 225                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 226                 'script': 'cwl-runner'
 227             }
 228             stubs.pipeline_component = stubs.expect_job_spec.copy()
 229             stubs.expect_pipeline_instance = {
 230                 'name': 'submit_wf.cwl',
 231                 'state': 'RunningOnServer',
 232                 'owner_uuid': None,
 233                 "components": {
 234                     "cwl-runner": {
 235                         'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 236                         'script_parameters': {
 237                             'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 238                             'x': {"value": {
 239                                 'basename': 'blorp.txt',
 240                                 'class': 'File',
 241                                 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 242                                 "size": 16
 243                             }},
 244                             'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 245                                   'listing': [
 246                                       {
 247                                           'basename': 'renamed.txt',
 248                                           'class': 'File', 'location':
 249                                           'keep:99999999999999999999999999999998+99/file1.txt',
 250                                           'size': 0
 251                                       }
 252                                   ]}},
 253                             'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 254                             'arv:debug': True,
 255                             'arv:enable_reuse': True,
 256                             'arv:on_error': 'continue'
 257                         },
 258                         'repository': 'arvados',
 259                         'script_version': 'master',
 260                         'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 261                         'script': 'cwl-runner',
 262                         'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 263                     }
 264                 }
 265             }
 266             stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 267             stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 268             stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 269             stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 270             stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 271                 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 272                 "state": "Queued"
 273             }
 274             stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 275             stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 276
 277             cwd = os.getcwd()
 278             filepath = os.path.join(cwd, "tests/wf/submit_wf_packed.cwl")
 279             with open(filepath) as f:
 280                 expect_packed_workflow = yaml.round_trip_load(f)
 281
 282             expect_packed_workflow["id"] = "file://" + filepath
 283             mocktool = mock.NonCallableMock(tool=expect_packed_workflow, metadata=expect_packed_workflow)
 284
 285             git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 286             expect_packed_workflow.update(git_info)
 287
 288             stubs.expect_container_spec = {
 289                 'priority': 500,
 290                 'mounts': {
 291                     '/var/spool/cwl': {
 292                         'writable': True,
 293                         'kind': 'collection'
 294                     },
 295                     '/var/lib/cwl/workflow.json': {
 296                         'content': expect_packed_workflow,
 297                         'kind': 'json'
 298                     },
 299                     'stdout': {
 300                         'path': '/var/spool/cwl/cwl.output.json',
 301                         'kind': 'file'
 302                     },
 303                     '/var/lib/cwl/cwl.input.json': {
 304                         'kind': 'json',
 305                         'content': {
 306                             'y': {
 307                                 'basename': '99999999999999999999999999999998+99',
 308                                 'location': 'keep:99999999999999999999999999999998+99',
 309                                 'class': 'Directory'},
 310                             'x': {
 311                                 'basename': u'blorp.txt',
 312                                 'class': 'File',
 313                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 314                                 "size": 16
 315                             },
 316                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 317                                 {'basename': 'renamed.txt',
 318                                  'class': 'File',
 319                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 320                                  'size': 0
 321                                 }
 322                             ]}
 323                         },
 324                         'kind': 'json'
 325                     }
 326                 },
 327                 'secret_mounts': {},
 328                 'state': 'Committed',
 329                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 330                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 331                             '--eval-timeout=20', '--thread-count=0',
 332                             '--enable-reuse', "--collection-cache-size=256",
 333                             '--output-name=Output from workflow '+wfname,
 334                             '--debug', '--on-error=continue',
 335                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 336                 'name': wfname,
 337                 'container_image': '999999999999999999999999999999d3+99',
 338                 'output_name': 'Output from workflow '+wfname,
 339                 'output_path': '/var/spool/cwl',
 340                 'cwd': '/var/spool/cwl',
 341                 'runtime_constraints': {
 342                     'API': True,
 343                     'vcpus': 1,
 344                     'ram': (1024+256)*1024*1024
 345                 },
 346                 'use_existing': False,
 347                 'properties': {},
 348                 'secret_mounts': {}
 349             }
 350
 351             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 352             stubs.api.workflows().create().execute.return_value = {
 353                 "uuid": stubs.expect_workflow_uuid,
 354             }
 355             def update_mock(**kwargs):
 356                 stubs.updated_uuid = kwargs.get('uuid')
 357                 return mock.DEFAULT
 358             stubs.api.workflows().update.side_effect = update_mock
 359             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 360                 "uuid": stubs.updated_uuid,
 361             }
 362
 363             return func(self, stubs, *args, **kwargs)
 364         return wrapped
 365     return outer_wrapper
 366
 367 class TestSubmit(unittest.TestCase):
 368
 369     def setUp(self):
 370         cwltool.process._names = set()
 371         arvados_cwl.arvdocker.arv_docker_clear_cache()
 372
 373     def tearDown(self):
 374         root_logger = logging.getLogger('')
 375
 376         # Remove existing RuntimeStatusLoggingHandlers if they exist
 377         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 378         root_logger.handlers = handlers
 379
 380     @mock.patch("time.sleep")
 381     @stubs
 382     def test_submit_invalid_runner_ram(self, stubs, tm):
 383         exited = arvados_cwl.main(
 384             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 385              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 386             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 387         self.assertEqual(exited, 1)
 388
 389
 390     @stubs
 391     def test_submit_container(self, stubs):
 392         exited = arvados_cwl.main(
 393             ["--submit", "--no-wait", "--api=containers", "--debug",
 394                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 395             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 396
 397         stubs.api.collections().create.assert_has_calls([
 398             mock.call(body=JsonDiffMatcher({
 399                 'manifest_text':
 400                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 401                 'replication_desired': None,
 402                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 403             }), ensure_unique_name=False),
 404             mock.call(body=JsonDiffMatcher({
 405                 'manifest_text':
 406                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 407                 'replication_desired': None,
 408                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 409             }), ensure_unique_name=False),
 410             ])
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         stubs.api.container_requests().create.assert_called_with(
 414             body=JsonDiffMatcher(expect_container))
 415         self.assertEqual(stubs.capture_stdout.getvalue(),
 416                          stubs.expect_container_request_uuid + '\n')
 417         self.assertEqual(exited, 0)
 418
 419
 420     @stubs
 421     def test_submit_container_tool(self, stubs):
 422         # test for issue #16139
 423         exited = arvados_cwl.main(
 424             ["--submit", "--no-wait", "--api=containers", "--debug",
 425                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 426             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 427
 428         self.assertEqual(stubs.capture_stdout.getvalue(),
 429                          stubs.expect_container_request_uuid + '\n')
 430         self.assertEqual(exited, 0)
 431
 432     @stubs
 433     def test_submit_container_no_reuse(self, stubs):
 434         exited = arvados_cwl.main(
 435             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 436                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 437             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 438
 439         expect_container = copy.deepcopy(stubs.expect_container_spec)
 440         expect_container["command"] = [
 441             'arvados-cwl-runner', '--local', '--api=containers',
 442             '--no-log-timestamps', '--disable-validate', '--disable-color',
 443             '--eval-timeout=20', '--thread-count=0',
 444             '--disable-reuse', "--collection-cache-size=256",
 445             "--output-name=Output from workflow submit_wf.cwl",
 446             '--debug', '--on-error=continue',
 447             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 448         expect_container["use_existing"] = False
 449
 450         stubs.api.container_requests().create.assert_called_with(
 451             body=JsonDiffMatcher(expect_container))
 452         self.assertEqual(stubs.capture_stdout.getvalue(),
 453                          stubs.expect_container_request_uuid + '\n')
 454         self.assertEqual(exited, 0)
 455
 456     @stubs('submit_wf_no_reuse.cwl')
 457     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 458         exited = arvados_cwl.main(
 459             ["--submit", "--no-wait", "--api=containers", "--debug",
 460              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 461             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 462         self.assertEqual(exited, 0)
 463
 464         expect_container = copy.deepcopy(stubs.expect_container_spec)
 465         expect_container["command"] = [
 466             'arvados-cwl-runner', '--local', '--api=containers',
 467             '--no-log-timestamps', '--disable-validate', '--disable-color',
 468             '--eval-timeout=20', '--thread-count=0',
 469             '--disable-reuse', "--collection-cache-size=256",
 470             '--output-name=Output from workflow submit_wf_no_reuse.cwl', '--debug', '--on-error=continue',
 471             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 472         expect_container["use_existing"] = False
 473         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 474             {
 475                 "class": "http://arvados.org/cwl#ReuseRequirement",
 476                 "enableReuse": False,
 477             },
 478         ]
 479         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 480             "arv": "http://arvados.org/cwl#",
 481             "cwltool": "http://commonwl.org/cwltool#"
 482         }
 483
 484         stubs.api.container_requests().create.assert_called_with(
 485             body=JsonDiffMatcher(expect_container))
 486         self.assertEqual(stubs.capture_stdout.getvalue(),
 487                          stubs.expect_container_request_uuid + '\n')
 488
 489
 490     @stubs
 491     def test_submit_container_on_error(self, stubs):
 492         exited = arvados_cwl.main(
 493             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 494                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 495             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 496
 497         expect_container = copy.deepcopy(stubs.expect_container_spec)
 498         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 499                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 500                                        '--eval-timeout=20', '--thread-count=0',
 501                                        '--enable-reuse', "--collection-cache-size=256",
 502                                        "--output-name=Output from workflow submit_wf.cwl",
 503                                        '--debug', '--on-error=stop',
 504                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(stubs.capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510         self.assertEqual(exited, 0)
 511
 512     @stubs
 513     def test_submit_container_output_name(self, stubs):
 514         output_name = "test_output_name"
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 518                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 519             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 520
 521         expect_container = copy.deepcopy(stubs.expect_container_spec)
 522         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 523                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 524                                        '--eval-timeout=20', '--thread-count=0',
 525                                        '--enable-reuse', "--collection-cache-size=256",
 526                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 527                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 528         expect_container["output_name"] = output_name
 529
 530         stubs.api.container_requests().create.assert_called_with(
 531             body=JsonDiffMatcher(expect_container))
 532         self.assertEqual(stubs.capture_stdout.getvalue(),
 533                          stubs.expect_container_request_uuid + '\n')
 534         self.assertEqual(exited, 0)
 535
 536     @stubs
 537     def test_submit_storage_classes(self, stubs):
 538         exited = arvados_cwl.main(
 539             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 540                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 541             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 542
 543         expect_container = copy.deepcopy(stubs.expect_container_spec)
 544         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 545                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 546                                        '--eval-timeout=20', '--thread-count=0',
 547                                        '--enable-reuse', "--collection-cache-size=256",
 548                                        '--output-name=Output from workflow submit_wf.cwl',
 549                                        "--debug",
 550                                        "--storage-classes=foo", '--on-error=continue',
 551                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 552
 553         stubs.api.container_requests().create.assert_called_with(
 554             body=JsonDiffMatcher(expect_container))
 555         self.assertEqual(stubs.capture_stdout.getvalue(),
 556                          stubs.expect_container_request_uuid + '\n')
 557         self.assertEqual(exited, 0)
 558
 559     @stubs
 560     def test_submit_multiple_storage_classes(self, stubs):
 561         exited = arvados_cwl.main(
 562             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 563                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 564             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 565
 566         expect_container = copy.deepcopy(stubs.expect_container_spec)
 567         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 568                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 569                                        '--eval-timeout=20', '--thread-count=0',
 570                                        '--enable-reuse', "--collection-cache-size=256",
 571                                        "--output-name=Output from workflow submit_wf.cwl",
 572                                        "--debug",
 573                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 574                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 575
 576         stubs.api.container_requests().create.assert_called_with(
 577             body=JsonDiffMatcher(expect_container))
 578         self.assertEqual(stubs.capture_stdout.getvalue(),
 579                          stubs.expect_container_request_uuid + '\n')
 580         self.assertEqual(exited, 0)
 581
 582     @mock.patch("cwltool.task_queue.TaskQueue")
 583     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 584     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 585     @stubs
 586     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 587         final_output_c = arvados.collection.Collection()
 588         make_output.return_value = ({},final_output_c)
 589
 590         def set_final_output(job_order, output_callback, runtimeContext):
 591             output_callback({"out": "zzzzz"}, "success")
 592             return []
 593         job.side_effect = set_final_output
 594
 595         exited = arvados_cwl.main(
 596             ["--debug", "--local", "--storage-classes=foo",
 597                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599
 600         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 601         self.assertEqual(exited, 0)
 602
 603     @mock.patch("cwltool.task_queue.TaskQueue")
 604     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 605     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 606     @stubs
 607     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 608         final_output_c = arvados.collection.Collection()
 609         make_output.return_value = ({},final_output_c)
 610         stubs.api.config().get.return_value = {"default": {"Default": True}}
 611
 612         def set_final_output(job_order, output_callback, runtimeContext):
 613             output_callback({"out": "zzzzz"}, "success")
 614             return []
 615         job.side_effect = set_final_output
 616
 617         exited = arvados_cwl.main(
 618             ["--debug", "--local",
 619                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 620             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 621
 622         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 623         self.assertEqual(exited, 0)
 624
 625     @mock.patch("cwltool.task_queue.TaskQueue")
 626     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 627     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 628     @stubs
 629     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 630         final_output_c = arvados.collection.Collection()
 631         make_output.return_value = ({},final_output_c)
 632
 633         def set_final_output(job_order, output_callback, runtimeContext):
 634             output_callback({"out": "zzzzz"}, "success")
 635             return []
 636         job.side_effect = set_final_output
 637
 638         exited = arvados_cwl.main(
 639             ["--debug", "--local",
 640                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 641             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 642
 643         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 644         self.assertEqual(exited, 0)
 645
 646     @stubs
 647     def test_submit_container_output_ttl(self, stubs):
 648         exited = arvados_cwl.main(
 649             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 650                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 651             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 652
 653         expect_container = copy.deepcopy(stubs.expect_container_spec)
 654         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 655                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 656                                        '--eval-timeout=20', '--thread-count=0',
 657                                        '--enable-reuse', "--collection-cache-size=256",
 658                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 659                                        '--on-error=continue',
 660                                        "--intermediate-output-ttl=3600",
 661                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 662
 663         stubs.api.container_requests().create.assert_called_with(
 664             body=JsonDiffMatcher(expect_container))
 665         self.assertEqual(stubs.capture_stdout.getvalue(),
 666                          stubs.expect_container_request_uuid + '\n')
 667         self.assertEqual(exited, 0)
 668
 669     @stubs
 670     def test_submit_container_trash_intermediate(self, stubs):
 671         exited = arvados_cwl.main(
 672             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 673                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 674             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 675
 676
 677         expect_container = copy.deepcopy(stubs.expect_container_spec)
 678         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 679                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 680                                        '--eval-timeout=20', '--thread-count=0',
 681                                        '--enable-reuse', "--collection-cache-size=256",
 682                                        '--debug', '--on-error=continue',
 683                                        "--trash-intermediate",
 684                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 685
 686         stubs.api.container_requests().create.assert_called_with(
 687             body=JsonDiffMatcher(expect_container))
 688         self.assertEqual(stubs.capture_stdout.getvalue(),
 689                          stubs.expect_container_request_uuid + '\n')
 690         self.assertEqual(exited, 0)
 691
 692     @stubs
 693     def test_submit_container_output_tags(self, stubs):
 694         output_tags = "tag0,tag1,tag2"
 695
 696         exited = arvados_cwl.main(
 697             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 698                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 699             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 700
 701         expect_container = copy.deepcopy(stubs.expect_container_spec)
 702         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 703                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 704                                        '--eval-timeout=20', '--thread-count=0',
 705                                        '--enable-reuse', "--collection-cache-size=256",
 706                                        "--output-name=Output from workflow submit_wf.cwl",
 707                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 708                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 709
 710         stubs.api.container_requests().create.assert_called_with(
 711             body=JsonDiffMatcher(expect_container))
 712         self.assertEqual(stubs.capture_stdout.getvalue(),
 713                          stubs.expect_container_request_uuid + '\n')
 714         self.assertEqual(exited, 0)
 715
 716     @stubs
 717     def test_submit_container_runner_ram(self, stubs):
 718         exited = arvados_cwl.main(
 719             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 720                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 721             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 722
 723         expect_container = copy.deepcopy(stubs.expect_container_spec)
 724         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 725
 726         stubs.api.container_requests().create.assert_called_with(
 727             body=JsonDiffMatcher(expect_container))
 728         self.assertEqual(stubs.capture_stdout.getvalue(),
 729                          stubs.expect_container_request_uuid + '\n')
 730         self.assertEqual(exited, 0)
 731
 732     @mock.patch("arvados.collection.CollectionReader")
 733     @mock.patch("time.sleep")
 734     @stubs
 735     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 736         collectionReader().exists.return_value = True
 737         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 738         exited = arvados_cwl.main(
 739             ["--submit", "--no-wait", "--api=containers", "--debug",
 740              "tests/wf/submit_keepref_wf.cwl"],
 741             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 742         self.assertEqual(exited, 0)
 743
 744     @mock.patch("arvados.collection.CollectionReader")
 745     @mock.patch("time.sleep")
 746     @stubs
 747     def test_submit_keepref(self, stubs, tm, reader):
 748         with open("tests/wf/expect_arvworkflow.cwl") as f:
 749             reader().open().__enter__().read.return_value = f.read()
 750
 751         exited = arvados_cwl.main(
 752             ["--submit", "--no-wait", "--api=containers", "--debug",
 753              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 754             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 755
 756         expect_container = {
 757             'priority': 500,
 758             'mounts': {
 759                 '/var/spool/cwl': {
 760                     'writable': True,
 761                     'kind': 'collection'
 762                 },
 763                 'stdout': {
 764                     'path': '/var/spool/cwl/cwl.output.json',
 765                     'kind': 'file'
 766                 },
 767                 '/var/lib/cwl/workflow': {
 768                     'portable_data_hash': '99999999999999999999999999999994+99',
 769                     'kind': 'collection'
 770                 },
 771                 '/var/lib/cwl/cwl.input.json': {
 772                     'content': {
 773                         'x': 'XxX'
 774                     },
 775                     'kind': 'json'
 776                 }
 777             }, 'state': 'Committed',
 778             'output_path': '/var/spool/cwl',
 779             'name': 'expect_arvworkflow.cwl#main',
 780             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 781             'container_image': '999999999999999999999999999999d3+99',
 782             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 783                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 784                         '--eval-timeout=20', '--thread-count=0',
 785                         '--enable-reuse', "--collection-cache-size=256",
 786                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 787                         '--debug', '--on-error=continue',
 788                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 789             'cwd': '/var/spool/cwl',
 790             'runtime_constraints': {
 791                 'API': True,
 792                 'vcpus': 1,
 793                 'ram': 1342177280
 794             },
 795             'use_existing': False,
 796             'properties': {},
 797             'secret_mounts': {}
 798         }
 799
 800         stubs.api.container_requests().create.assert_called_with(
 801             body=JsonDiffMatcher(expect_container))
 802         self.assertEqual(stubs.capture_stdout.getvalue(),
 803                          stubs.expect_container_request_uuid + '\n')
 804         self.assertEqual(exited, 0)
 805
 806     @mock.patch("time.sleep")
 807     @stubs
 808     def test_submit_arvworkflow(self, stubs, tm):
 809         with open("tests/wf/expect_arvworkflow.cwl") as f:
 810             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 811
 812         exited = arvados_cwl.main(
 813             ["--submit", "--no-wait", "--api=containers", "--debug",
 814              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 815             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 816
 817         expect_container = {
 818             'priority': 500,
 819             'mounts': {
 820                 '/var/spool/cwl': {
 821                     'writable': True,
 822                     'kind': 'collection'
 823                 },
 824                 'stdout': {
 825                     'path': '/var/spool/cwl/cwl.output.json',
 826                     'kind': 'file'
 827                 },
 828                 '/var/lib/cwl/workflow.json': {
 829                     'kind': 'json',
 830                     'content': {
 831                         'cwlVersion': 'v1.0',
 832                         '$graph': [
 833                             {
 834                                 'id': '#main',
 835                                 'inputs': [
 836                                     {'type': 'string', 'id': '#main/x'}
 837                                 ],
 838                                 'steps': [
 839                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 840                                      'run': '#submit_tool.cwl',
 841                                      'id': '#main/step1',
 842                                      'out': []}
 843                                 ],
 844                                 'class': 'Workflow',
 845                                 'outputs': []
 846                             },
 847                             {
 848                                 'inputs': [
 849                                     {
 850                                         'inputBinding': {'position': 1},
 851                                         'type': 'string',
 852                                         'id': '#submit_tool.cwl/x'}
 853                                 ],
 854                                 'requirements': [
 855                                     {
 856                                         'dockerPull': 'debian:buster-slim',
 857                                         'class': 'DockerRequirement',
 858                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 859                                     }
 860                                 ],
 861                                 'id': '#submit_tool.cwl',
 862                                 'outputs': [],
 863                                 'baseCommand': 'cat',
 864                                 'class': 'CommandLineTool'
 865                             }
 866                         ]
 867                     }
 868                 },
 869                 '/var/lib/cwl/cwl.input.json': {
 870                     'content': {
 871                         'x': 'XxX'
 872                     },
 873                     'kind': 'json'
 874                 }
 875             }, 'state': 'Committed',
 876             'output_path': '/var/spool/cwl',
 877             'name': 'a test workflow',
 878             'container_image': "999999999999999999999999999999d3+99",
 879             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 880                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 881                         '--eval-timeout=20', '--thread-count=0',
 882                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 883                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 884             'cwd': '/var/spool/cwl',
 885             'runtime_constraints': {
 886                 'API': True,
 887                 'vcpus': 1,
 888                 'ram': 1342177280
 889             },
 890             'use_existing': False,
 891             'properties': {
 892                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 893             },
 894             'secret_mounts': {}
 895         }
 896
 897         stubs.api.container_requests().create.assert_called_with(
 898             body=JsonDiffMatcher(expect_container))
 899         self.assertEqual(stubs.capture_stdout.getvalue(),
 900                          stubs.expect_container_request_uuid + '\n')
 901         self.assertEqual(exited, 0)
 902
 903     @stubs('hello container 123')
 904     def test_submit_container_name(self, stubs):
 905         exited = arvados_cwl.main(
 906             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 907                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 908             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 909
 910         expect_container = copy.deepcopy(stubs.expect_container_spec)
 911
 912         stubs.api.container_requests().create.assert_called_with(
 913             body=JsonDiffMatcher(expect_container))
 914         self.assertEqual(stubs.capture_stdout.getvalue(),
 915                          stubs.expect_container_request_uuid + '\n')
 916         self.assertEqual(exited, 0)
 917
 918     @stubs
 919     def test_submit_missing_input(self, stubs):
 920         exited = arvados_cwl.main(
 921             ["--submit", "--no-wait", "--api=containers", "--debug",
 922              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 923             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 924         self.assertEqual(exited, 0)
 925
 926         exited = arvados_cwl.main(
 927             ["--submit", "--no-wait", "--api=containers", "--debug",
 928              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 929             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 930         self.assertEqual(exited, 1)
 931
 932     @stubs
 933     def test_submit_container_project(self, stubs):
 934         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 935         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 936         exited = arvados_cwl.main(
 937             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 938                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 939             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 940
 941         expect_container = copy.deepcopy(stubs.expect_container_spec)
 942         expect_container["owner_uuid"] = project_uuid
 943         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 944                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 945                                        "--eval-timeout=20", "--thread-count=0",
 946                                        '--enable-reuse', "--collection-cache-size=256",
 947                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 948                                        '--on-error=continue',
 949                                        '--project-uuid='+project_uuid,
 950                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 951
 952         stubs.api.container_requests().create.assert_called_with(
 953             body=JsonDiffMatcher(expect_container))
 954         self.assertEqual(stubs.capture_stdout.getvalue(),
 955                          stubs.expect_container_request_uuid + '\n')
 956         self.assertEqual(exited, 0)
 957
 958     @stubs
 959     def test_submit_container_eval_timeout(self, stubs):
 960         exited = arvados_cwl.main(
 961             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 962                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 963             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 964
 965         expect_container = copy.deepcopy(stubs.expect_container_spec)
 966         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 967                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 968                                        '--eval-timeout=60.0', '--thread-count=0',
 969                                        '--enable-reuse', "--collection-cache-size=256",
 970                                        '--debug', '--on-error=continue',
 971                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 972
 973         stubs.api.container_requests().create.assert_called_with(
 974             body=JsonDiffMatcher(expect_container))
 975         self.assertEqual(stubs.capture_stdout.getvalue(),
 976                          stubs.expect_container_request_uuid + '\n')
 977         self.assertEqual(exited, 0)
 978
 979     @stubs
 980     def test_submit_container_collection_cache(self, stubs):
 981         exited = arvados_cwl.main(
 982             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 983                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 984             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 985
 986         expect_container = copy.deepcopy(stubs.expect_container_spec)
 987         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 988                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 989                                        '--eval-timeout=20', '--thread-count=0',
 990                                        '--enable-reuse', "--collection-cache-size=500",
 991                                        '--debug', '--on-error=continue',
 992                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 993         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 994
 995         stubs.api.container_requests().create.assert_called_with(
 996             body=JsonDiffMatcher(expect_container))
 997         self.assertEqual(stubs.capture_stdout.getvalue(),
 998                          stubs.expect_container_request_uuid + '\n')
 999         self.assertEqual(exited, 0)
1000
1001     @stubs
1002     def test_submit_container_thread_count(self, stubs):
1003         exited = arvados_cwl.main(
1004             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1005                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1006             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1007
1008         expect_container = copy.deepcopy(stubs.expect_container_spec)
1009         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1010                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1011                                        '--eval-timeout=20', '--thread-count=20',
1012                                        '--enable-reuse', "--collection-cache-size=256",
1013                                        '--debug', '--on-error=continue',
1014                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1015
1016         stubs.api.container_requests().create.assert_called_with(
1017             body=JsonDiffMatcher(expect_container))
1018         self.assertEqual(stubs.capture_stdout.getvalue(),
1019                          stubs.expect_container_request_uuid + '\n')
1020         self.assertEqual(exited, 0)
1021
1022     @stubs
1023     def test_submit_container_runner_image(self, stubs):
1024         exited = arvados_cwl.main(
1025             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1026                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1027             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1028
1029         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1030
1031         expect_container = copy.deepcopy(stubs.expect_container_spec)
1032         stubs.api.container_requests().create.assert_called_with(
1033             body=JsonDiffMatcher(expect_container))
1034         self.assertEqual(stubs.capture_stdout.getvalue(),
1035                          stubs.expect_container_request_uuid + '\n')
1036         self.assertEqual(exited, 0)
1037
1038     @stubs
1039     def test_submit_priority(self, stubs):
1040         exited = arvados_cwl.main(
1041             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1042                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1043             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1044
1045         stubs.expect_container_spec["priority"] = 669
1046
1047         expect_container = copy.deepcopy(stubs.expect_container_spec)
1048         stubs.api.container_requests().create.assert_called_with(
1049             body=JsonDiffMatcher(expect_container))
1050         self.assertEqual(stubs.capture_stdout.getvalue(),
1051                          stubs.expect_container_request_uuid + '\n')
1052         self.assertEqual(exited, 0)
1053
1054     @stubs('submit_wf_runner_resources.cwl')
1055     def test_submit_wf_runner_resources(self, stubs):
1056         exited = arvados_cwl.main(
1057             ["--submit", "--no-wait", "--api=containers", "--debug",
1058                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1059             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1060
1061         expect_container = copy.deepcopy(stubs.expect_container_spec)
1062         expect_container["runtime_constraints"] = {
1063             "API": True,
1064             "vcpus": 2,
1065             "ram": (2000+512) * 2**20
1066         }
1067         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1068             {
1069                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1070                 "coresMin": 2,
1071                 "ramMin": 2000,
1072                 "keep_cache": 512
1073             }
1074         ]
1075         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1076             "arv": "http://arvados.org/cwl#",
1077         }
1078         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1079                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1080                         '--eval-timeout=20', '--thread-count=0',
1081                         '--enable-reuse', "--collection-cache-size=512",
1082                                        '--output-name=Output from workflow submit_wf_runner_resources.cwl',
1083                                        '--debug', '--on-error=continue',
1084                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1085
1086         stubs.api.container_requests().create.assert_called_with(
1087             body=JsonDiffMatcher(expect_container))
1088         self.assertEqual(stubs.capture_stdout.getvalue(),
1089                          stubs.expect_container_request_uuid + '\n')
1090         self.assertEqual(exited, 0)
1091
1092     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1093     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1094     @mock.patch("arvados.api")
1095     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1096         arvados_cwl.arvdocker.arv_docker_clear_cache()
1097
1098         arvrunner = mock.MagicMock()
1099         arvrunner.project_uuid = ""
1100         api.return_value = mock.MagicMock()
1101         arvrunner.api = api.return_value
1102         arvrunner.runtimeContext.match_local_docker = False
1103         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1104                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1105                                                                         "link_class": "docker_image_repo+tag",
1106                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1107                                                                         "owner_uuid": "",
1108                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1109                                                             {"items": [{"created_at": "",
1110                                                                         "head_uuid": "",
1111                                                                         "link_class": "docker_image_hash",
1112                                                                         "name": "123456",
1113                                                                         "owner_uuid": "",
1114                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1115                                                             {"items": [{"created_at": "",
1116                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1117                                                                         "link_class": "docker_image_repo+tag",
1118                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1119                                                                         "owner_uuid": "",
1120                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1121                                                             {"items": [{"created_at": "",
1122                                                                         "head_uuid": "",
1123                                                                         "link_class": "docker_image_hash",
1124                                                                         "name": "123456",
1125                                                                         "owner_uuid": "",
1126                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1127         )
1128         find_one_image_hash.return_value = "123456"
1129
1130         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1131                                                                               "owner_uuid": "",
1132                                                                               "manifest_text": "",
1133                                                                               "properties": ""
1134                                                                               }], "items_available": 1, "offset": 0},
1135                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1136                                                                               "owner_uuid": "",
1137                                                                               "manifest_text": "",
1138                                                                               "properties": ""
1139                                                                           }], "items_available": 1, "offset": 0})
1140         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1141         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1142                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1143
1144         self.assertEqual("9999999999999999999999999999999b+99",
1145                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1146
1147
1148     @stubs
1149     def test_submit_secrets(self, stubs):
1150         exited = arvados_cwl.main(
1151             ["--submit", "--no-wait", "--api=containers", "--debug",
1152                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1153             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1154
1155         expect_container = {
1156             "command": [
1157                 "arvados-cwl-runner",
1158                 "--local",
1159                 "--api=containers",
1160                 "--no-log-timestamps",
1161                 "--disable-validate",
1162                 "--disable-color",
1163                 "--eval-timeout=20",
1164                 '--thread-count=0',
1165                 "--enable-reuse",
1166                 "--collection-cache-size=256",
1167                 '--output-name=Output from workflow secret_wf.cwl'
1168                 '--debug',
1169                 "--on-error=continue",
1170                 "/var/lib/cwl/workflow.json#main",
1171                 "/var/lib/cwl/cwl.input.json"
1172             ],
1173             "container_image": "999999999999999999999999999999d3+99",
1174             "cwd": "/var/spool/cwl",
1175             "mounts": {
1176                 "/var/lib/cwl/cwl.input.json": {
1177                     "content": {
1178                         "pw": {
1179                             "$include": "/secrets/s0"
1180                         }
1181                     },
1182                     "kind": "json"
1183                 },
1184                 "/var/lib/cwl/workflow.json": {
1185                     "content": {
1186                         "$graph": [
1187                             {
1188                                 "arguments": [
1189                                     "md5sum",
1190                                     "example.conf"
1191                                 ],
1192                                 "class": "CommandLineTool",
1193                                 "hints": [
1194                                     {
1195                                         "class": "http://commonwl.org/cwltool#Secrets",
1196                                         "secrets": [
1197                                             "#secret_job.cwl/pw"
1198                                         ]
1199                                     }
1200                                 ],
1201                                 "id": "#secret_job.cwl",
1202                                 "inputs": [
1203                                     {
1204                                         "id": "#secret_job.cwl/pw",
1205                                         "type": "string"
1206                                     }
1207                                 ],
1208                                 "outputs": [
1209                                     {
1210                                         "id": "#secret_job.cwl/out",
1211                                         "type": "File",
1212                                         "outputBinding": {
1213                                               "glob": "hashed_example.txt"
1214                                         }
1215                                     }
1216                                 ],
1217                                 "stdout": "hashed_example.txt",
1218                                 "requirements": [
1219                                     {
1220                                         "class": "InitialWorkDirRequirement",
1221                                         "listing": [
1222                                             {
1223                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1224                                                 "entryname": "example.conf"
1225                                             }
1226                                         ]
1227                                     }
1228                                 ]
1229                             },
1230                             {
1231                                 "class": "Workflow",
1232                                 "hints": [
1233                                     {
1234                                         "class": "DockerRequirement",
1235                                         "dockerPull": "debian:buster-slim",
1236                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1237                                     },
1238                                     {
1239                                         "class": "http://commonwl.org/cwltool#Secrets",
1240                                         "secrets": [
1241                                             "#main/pw"
1242                                         ]
1243                                     }
1244                                 ],
1245                                 "id": "#main",
1246                                 "inputs": [
1247                                     {
1248                                         "id": "#main/pw",
1249                                         "type": "string"
1250                                     }
1251                                 ],
1252                                 "outputs": [
1253                                     {
1254                                         "id": "#main/out",
1255                                         "outputSource": "#main/step1/out",
1256                                         "type": "File"
1257                                     }
1258                                 ],
1259                                 "steps": [
1260                                     {
1261                                         "id": "#main/step1",
1262                                         "in": [
1263                                             {
1264                                                 "id": "#main/step1/pw",
1265                                                 "source": "#main/pw"
1266                                             }
1267                                         ],
1268                                         "out": [
1269                                             "#main/step1/out"
1270                                         ],
1271                                         "run": "#secret_job.cwl"
1272                                     }
1273                                 ]
1274                             }
1275                         ],
1276                         "$namespaces": {
1277                             "cwltool": "http://commonwl.org/cwltool#"
1278                         },
1279                         "cwlVersion": "v1.0"
1280                     },
1281                     "kind": "json"
1282                 },
1283                 "/var/spool/cwl": {
1284                     "kind": "collection",
1285                     "writable": True
1286                 },
1287                 "stdout": {
1288                     "kind": "file",
1289                     "path": "/var/spool/cwl/cwl.output.json"
1290                 }
1291             },
1292             "name": "secret_wf.cwl",
1293             "output_name": "Output from workflow secret_wf.cwl",
1294             "output_path": "/var/spool/cwl",
1295             "priority": 500,
1296             "properties": {},
1297             "runtime_constraints": {
1298                 "API": True,
1299                 "ram": 1342177280,
1300                 "vcpus": 1
1301             },
1302             "secret_mounts": {
1303                 "/secrets/s0": {
1304                     "content": "blorp",
1305                     "kind": "text"
1306                 }
1307             },
1308             "state": "Committed",
1309             "use_existing": False
1310         }
1311
1312         stubs.api.container_requests().create.assert_called_with(
1313             body=JsonDiffMatcher(expect_container))
1314         self.assertEqual(stubs.capture_stdout.getvalue(),
1315                          stubs.expect_container_request_uuid + '\n')
1316         self.assertEqual(exited, 0)
1317
1318     @stubs
1319     def test_submit_request_uuid(self, stubs):
1320         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1321         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1322
1323         stubs.api.container_requests().update().execute.return_value = {
1324             "uuid": stubs.expect_container_request_uuid,
1325             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1326             "state": "Queued"
1327         }
1328
1329         exited = arvados_cwl.main(
1330             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1331                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1332             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1333
1334         stubs.api.container_requests().update.assert_called_with(
1335             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1336         self.assertEqual(stubs.capture_stdout.getvalue(),
1337                          stubs.expect_container_request_uuid + '\n')
1338         self.assertEqual(exited, 0)
1339
1340     @stubs
1341     def test_submit_container_cluster_id(self, stubs):
1342         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1343
1344         exited = arvados_cwl.main(
1345             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1346                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1347             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1348
1349         expect_container = copy.deepcopy(stubs.expect_container_spec)
1350
1351         stubs.api.container_requests().create.assert_called_with(
1352             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1353         self.assertEqual(stubs.capture_stdout.getvalue(),
1354                          stubs.expect_container_request_uuid + '\n')
1355         self.assertEqual(exited, 0)
1356
1357     @stubs
1358     def test_submit_validate_cluster_id(self, stubs):
1359         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1360         exited = arvados_cwl.main(
1361             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1362              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1363             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1364         self.assertEqual(exited, 1)
1365
1366     @stubs
1367     def test_submit_validate_project_uuid(self, stubs):
1368         # Fails with bad cluster prefix
1369         exited = arvados_cwl.main(
1370             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1371              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1372             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1373         self.assertEqual(exited, 1)
1374
1375         # Project lookup fails
1376         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1377         exited = arvados_cwl.main(
1378             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1379              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1380             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1381         self.assertEqual(exited, 1)
1382
1383         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1384         exited = arvados_cwl.main(
1385             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1386              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1387             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1388         self.assertEqual(exited, 0)
1389
1390
1391     @mock.patch("arvados.collection.CollectionReader")
1392     @stubs
1393     def test_submit_uuid_inputs(self, stubs, collectionReader):
1394         collectionReader().exists.return_value = True
1395         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1396         def list_side_effect(**kwargs):
1397             m = mock.MagicMock()
1398             if "count" in kwargs:
1399                 m.execute.return_value = {"items": [
1400                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1401                 ]}
1402             else:
1403                 m.execute.return_value = {"items": []}
1404             return m
1405         stubs.api.collections().list.side_effect = list_side_effect
1406
1407         exited = arvados_cwl.main(
1408             ["--submit", "--no-wait", "--api=containers", "--debug",
1409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1411
1412         expect_container = copy.deepcopy(stubs.expect_container_spec)
1413         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1414         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1415         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1416
1417         stubs.api.collections().list.assert_has_calls([
1418             mock.call(count='none',
1419                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1420                       select=['uuid', 'portable_data_hash'])])
1421         stubs.api.container_requests().create.assert_called_with(
1422             body=JsonDiffMatcher(expect_container))
1423         self.assertEqual(stubs.capture_stdout.getvalue(),
1424                          stubs.expect_container_request_uuid + '\n')
1425         self.assertEqual(exited, 0)
1426
1427     @stubs
1428     def test_submit_mismatched_uuid_inputs(self, stubs):
1429         def list_side_effect(**kwargs):
1430             m = mock.MagicMock()
1431             if "count" in kwargs:
1432                 m.execute.return_value = {"items": [
1433                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1434                 ]}
1435             else:
1436                 m.execute.return_value = {"items": []}
1437             return m
1438         stubs.api.collections().list.side_effect = list_side_effect
1439
1440         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1441             capture_stderr = StringIO()
1442             cwltool_logger = logging.getLogger('cwltool')
1443             stderr_logger = logging.StreamHandler(capture_stderr)
1444             cwltool_logger.addHandler(stderr_logger)
1445
1446             try:
1447                 exited = arvados_cwl.main(
1448                     ["--submit", "--no-wait", "--api=containers", "--debug",
1449                         "tests/wf/submit_wf.cwl", infile],
1450                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1451
1452                 self.assertEqual(exited, 1)
1453                 self.assertRegex(
1454                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1455                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1456             finally:
1457                 cwltool_logger.removeHandler(stderr_logger)
1458
1459     @mock.patch("arvados.collection.CollectionReader")
1460     @stubs
1461     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1462         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1463         capture_stderr = StringIO()
1464
1465         cwltool_logger = logging.getLogger('cwltool')
1466         stderr_logger = logging.StreamHandler(capture_stderr)
1467         cwltool_logger.addHandler(stderr_logger)
1468
1469         exited = arvados_cwl.main(
1470             ["--submit", "--no-wait", "--api=containers", "--debug",
1471                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1472             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1473
1474         try:
1475             self.assertEqual(exited, 1)
1476             self.assertRegex(
1477                 capture_stderr.getvalue(),
1478                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1479         finally:
1480             cwltool_logger.removeHandler(stderr_logger)
1481
1482     @stubs('submit_wf_process_properties.cwl')
1483     def test_submit_set_process_properties(self, stubs):
1484         exited = arvados_cwl.main(
1485             ["--submit", "--no-wait", "--api=containers", "--debug",
1486                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1487             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1488
1489         expect_container = copy.deepcopy(stubs.expect_container_spec)
1490
1491         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1492             {
1493                 "class": "http://arvados.org/cwl#ProcessProperties",
1494                 "processProperties": [
1495                     {"propertyName": "baz",
1496                      "propertyValue": "$(inputs.x.basename)"},
1497                     {"propertyName": "foo",
1498                      "propertyValue": "bar"},
1499                     {"propertyName": "quux",
1500                      "propertyValue": {
1501                          "q1": 1,
1502                          "q2": 2
1503                      }
1504                     }
1505                 ],
1506             }
1507         ]
1508         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1509             "arv": "http://arvados.org/cwl#"
1510         }
1511
1512         expect_container["properties"] = {
1513             "baz": "blorp.txt",
1514             "foo": "bar",
1515             "quux": {
1516                 "q1": 1,
1517                 "q2": 2
1518             }
1519         }
1520
1521         stubs.api.container_requests().create.assert_called_with(
1522             body=JsonDiffMatcher(expect_container))
1523         self.assertEqual(stubs.capture_stdout.getvalue(),
1524                          stubs.expect_container_request_uuid + '\n')
1525         self.assertEqual(exited, 0)
1526
1527
1528     @stubs
1529     def test_submit_enable_preemptible(self, stubs):
1530         exited = arvados_cwl.main(
1531             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1532                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1533             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1534
1535         expect_container = copy.deepcopy(stubs.expect_container_spec)
1536         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1537                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1538                         '--eval-timeout=20', '--thread-count=0',
1539                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1540                                        '--enable-preemptible',
1541                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1542
1543         stubs.api.container_requests().create.assert_called_with(
1544             body=JsonDiffMatcher(expect_container))
1545         self.assertEqual(stubs.capture_stdout.getvalue(),
1546                          stubs.expect_container_request_uuid + '\n')
1547         self.assertEqual(exited, 0)
1548
1549     @stubs
1550     def test_submit_disable_preemptible(self, stubs):
1551         exited = arvados_cwl.main(
1552             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1553                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1554             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1555
1556         expect_container = copy.deepcopy(stubs.expect_container_spec)
1557         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1558                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1559                         '--eval-timeout=20', '--thread-count=0',
1560                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1561                                        '--disable-preemptible',
1562                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1563
1564         stubs.api.container_requests().create.assert_called_with(
1565             body=JsonDiffMatcher(expect_container))
1566         self.assertEqual(stubs.capture_stdout.getvalue(),
1567                          stubs.expect_container_request_uuid + '\n')
1568         self.assertEqual(exited, 0)
1569
1570
1571 class TestCreateWorkflow(unittest.TestCase):
1572     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1573     expect_workflow = StripYAMLComments(
1574         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1575
1576     def setUp(self):
1577         cwltool.process._names = set()
1578         arvados_cwl.arvdocker.arv_docker_clear_cache()
1579
1580     def tearDown(self):
1581         root_logger = logging.getLogger('')
1582
1583         # Remove existing RuntimeStatusLoggingHandlers if they exist
1584         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1585         root_logger.handlers = handlers
1586
1587     @stubs
1588     def test_create(self, stubs):
1589         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1590         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1591
1592         exited = arvados_cwl.main(
1593             ["--create-workflow", "--debug",
1594              "--api=containers",
1595              "--project-uuid", project_uuid,
1596              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1597             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1598
1599         stubs.api.pipeline_templates().create.refute_called()
1600         stubs.api.container_requests().create.refute_called()
1601
1602         body = {
1603             "workflow": {
1604                 "owner_uuid": project_uuid,
1605                 "name": "submit_wf.cwl",
1606                 "description": "",
1607                 "definition": self.expect_workflow,
1608             }
1609         }
1610         stubs.api.workflows().create.assert_called_with(
1611             body=JsonDiffMatcher(body))
1612
1613         self.assertEqual(stubs.capture_stdout.getvalue(),
1614                          stubs.expect_workflow_uuid + '\n')
1615         self.assertEqual(exited, 0)
1616
1617     @stubs
1618     def test_create_name(self, stubs):
1619         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1620         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1621
1622         exited = arvados_cwl.main(
1623             ["--create-workflow", "--debug",
1624              "--api=containers",
1625              "--project-uuid", project_uuid,
1626              "--name", "testing 123",
1627              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1628             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1629
1630         stubs.api.pipeline_templates().create.refute_called()
1631         stubs.api.container_requests().create.refute_called()
1632
1633         body = {
1634             "workflow": {
1635                 "owner_uuid": project_uuid,
1636                 "name": "testing 123",
1637                 "description": "",
1638                 "definition": self.expect_workflow,
1639             }
1640         }
1641         stubs.api.workflows().create.assert_called_with(
1642             body=JsonDiffMatcher(body))
1643
1644         self.assertEqual(stubs.capture_stdout.getvalue(),
1645                          stubs.expect_workflow_uuid + '\n')
1646         self.assertEqual(exited, 0)
1647
1648
1649     @stubs
1650     def test_update(self, stubs):
1651         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1652         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1653
1654         exited = arvados_cwl.main(
1655             ["--update-workflow", self.existing_workflow_uuid,
1656              "--debug",
1657              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1658             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1659
1660         body = {
1661             "workflow": {
1662                 "name": "submit_wf.cwl",
1663                 "description": "",
1664                 "definition": self.expect_workflow,
1665                 "owner_uuid": project_uuid
1666             }
1667         }
1668         stubs.api.workflows().update.assert_called_with(
1669             uuid=self.existing_workflow_uuid,
1670             body=JsonDiffMatcher(body))
1671         self.assertEqual(stubs.capture_stdout.getvalue(),
1672                          self.existing_workflow_uuid + '\n')
1673         self.assertEqual(exited, 0)
1674
1675
1676     @stubs
1677     def test_update_name(self, stubs):
1678         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1679         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1680
1681         exited = arvados_cwl.main(
1682             ["--update-workflow", self.existing_workflow_uuid,
1683              "--debug", "--name", "testing 123",
1684              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1685             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1686
1687         body = {
1688             "workflow": {
1689                 "name": "testing 123",
1690                 "description": "",
1691                 "definition": self.expect_workflow,
1692                 "owner_uuid": project_uuid
1693             }
1694         }
1695         stubs.api.workflows().update.assert_called_with(
1696             uuid=self.existing_workflow_uuid,
1697             body=JsonDiffMatcher(body))
1698         self.assertEqual(stubs.capture_stdout.getvalue(),
1699                          self.existing_workflow_uuid + '\n')
1700         self.assertEqual(exited, 0)
1701
1702     @stubs
1703     def test_create_collection_per_tool(self, stubs):
1704         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1705         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1706
1707         exited = arvados_cwl.main(
1708             ["--create-workflow", "--debug",
1709              "--api=containers",
1710              "--project-uuid", project_uuid,
1711              "tests/collection_per_tool/collection_per_tool.cwl"],
1712             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1713
1714         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1715         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1716
1717         body = {
1718             "workflow": {
1719                 "owner_uuid": project_uuid,
1720                 "name": "collection_per_tool.cwl",
1721                 "description": "",
1722                 "definition": expect_workflow,
1723             }
1724         }
1725         stubs.api.workflows().create.assert_called_with(
1726             body=JsonDiffMatcher(body))
1727
1728         self.assertEqual(stubs.capture_stdout.getvalue(),
1729                          stubs.expect_workflow_uuid + '\n')
1730         self.assertEqual(exited, 0)
1731
1732     @stubs
1733     def test_create_with_imports(self, stubs):
1734         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1735         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1736
1737         exited = arvados_cwl.main(
1738             ["--create-workflow", "--debug",
1739              "--api=containers",
1740              "--project-uuid", project_uuid,
1741              "tests/wf/feddemo/feddemo.cwl"],
1742             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1743
1744         stubs.api.pipeline_templates().create.refute_called()
1745         stubs.api.container_requests().create.refute_called()
1746
1747         self.assertEqual(stubs.capture_stdout.getvalue(),
1748                          stubs.expect_workflow_uuid + '\n')
1749         self.assertEqual(exited, 0)
1750
1751     @stubs
1752     def test_create_with_no_input(self, stubs):
1753         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1754         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1755
1756         exited = arvados_cwl.main(
1757             ["--create-workflow", "--debug",
1758              "--api=containers",
1759              "--project-uuid", project_uuid,
1760              "tests/wf/revsort/revsort.cwl"],
1761             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1762
1763         stubs.api.pipeline_templates().create.refute_called()
1764         stubs.api.container_requests().create.refute_called()
1765
1766         self.assertEqual(stubs.capture_stdout.getvalue(),
1767                          stubs.expect_workflow_uuid + '\n')
1768         self.assertEqual(exited, 0)