sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml as yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfdetails=('submit_wf.cwl', None)):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62
  63             wfname = wfdetails[0]
  64             wfpath = wfdetails[1]
  65
  66             stubs = Stubs()
  67             stubs.events = events
  68             stubs.keepdocker = keepdocker
  69
  70             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  71                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  73
  74             determine_image_id.return_value = None
  75
  76             def putstub(p, **kwargs):
  77                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  78             keep_client1().put.side_effect = putstub
  79             keep_client1.put.side_effect = putstub
  80             keep_client2().put.side_effect = putstub
  81             keep_client2.put.side_effect = putstub
  82
  83             stubs.keep_client = keep_client2
  84             stubs.docker_images = {
  85                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  86                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  87                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  88                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  89             }
  90             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  91                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  92             stubs.keepdocker.side_effect = kd
  93
  94             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  95             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  96
  97             if sys.version_info[0] < 3:
  98                 stubs.capture_stdout = BytesIO()
  99             else:
 100                 stubs.capture_stdout = StringIO()
 101
 102             stubs.api = mock.MagicMock()
 103             stubs.api._rootDesc = get_rootDesc()
 104             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 105             stubs.api._rootDesc["revision"] = "20210628"
 106
 107             stubs.api.users().current().execute.return_value = {
 108                 "uuid": stubs.fake_user_uuid,
 109             }
 110             stubs.api.collections().list().execute.return_value = {"items": []}
 111             stubs.api.containers().current().execute.return_value = {
 112                 "uuid": stubs.fake_container_uuid,
 113             }
 114             stubs.api.config()["StorageClasses"].items.return_value = {
 115                 "default": {
 116                     "Default": True
 117                 }
 118             }.items()
 119
 120             class CollectionExecute(object):
 121                 def __init__(self, exe):
 122                     self.exe = exe
 123                 def execute(self, num_retries=None):
 124                     return self.exe
 125
 126             def collection_createstub(created_collections, body, ensure_unique_name=None):
 127                 mt = body["manifest_text"].encode('utf-8')
 128                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 129                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 130                 created_collections[uuid] = {
 131                     "uuid": uuid,
 132                     "portable_data_hash": pdh,
 133                     "manifest_text": mt.decode('utf-8')
 134                 }
 135                 return CollectionExecute(created_collections[uuid])
 136
 137             def collection_getstub(created_collections, uuid):
 138                 for v in viewvalues(created_collections):
 139                     if uuid in (v["uuid"], v["portable_data_hash"]):
 140                         return CollectionExecute(v)
 141
 142             created_collections = {
 143                 "99999999999999999999999999999998+99": {
 144                     "uuid": "",
 145                     "portable_data_hash": "99999999999999999999999999999998+99",
 146                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 147                 },
 148                 "99999999999999999999999999999997+99": {
 149                     "uuid": "",
 150                     "portable_data_hash": "99999999999999999999999999999997+99",
 151                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 152                 },
 153                 "99999999999999999999999999999994+99": {
 154                     "uuid": "",
 155                     "portable_data_hash": "99999999999999999999999999999994+99",
 156                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 157                 },
 158                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 159                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 160                     "portable_data_hash": "999999999999999999999999999999d3+99",
 161                     "manifest_text": ""
 162                 },
 163                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 164                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 165                     "portable_data_hash": "999999999999999999999999999999d4+99",
 166                     "manifest_text": ""
 167                 },
 168                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 169                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 170                     "portable_data_hash": "999999999999999999999999999999d5+99",
 171                     "manifest_text": ""
 172                 },
 173                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 174                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 175                     "portable_data_hash": "999999999999999999999999999999d6+99",
 176                     "manifest_text": ""
 177                 }
 178             }
 179             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 180             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 181
 182             stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 183             stubs.api.jobs().create().execute.return_value = {
 184                 "uuid": stubs.expect_job_uuid,
 185                 "state": "Queued",
 186             }
 187
 188             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 189             stubs.api.container_requests().create().execute.return_value = {
 190                 "uuid": stubs.expect_container_request_uuid,
 191                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 192                 "state": "Queued"
 193             }
 194
 195             stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 196             stubs.api.pipeline_templates().create().execute.return_value = {
 197                 "uuid": stubs.expect_pipeline_template_uuid,
 198             }
 199             stubs.expect_job_spec = {
 200                 'runtime_constraints': {
 201                     'docker_image': '999999999999999999999999999999d3+99',
 202                     'min_ram_mb_per_node': 1024
 203                 },
 204                 'script_parameters': {
 205                     'x': {
 206                         'basename': 'blorp.txt',
 207                         'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 208                         'class': 'File'
 209                     },
 210                     'y': {
 211                         'basename': '99999999999999999999999999999998+99',
 212                         'location': 'keep:99999999999999999999999999999998+99',
 213                         'class': 'Directory'
 214                     },
 215                     'z': {
 216                         'basename': 'anonymous',
 217                         "listing": [{
 218                             "basename": "renamed.txt",
 219                             "class": "File",
 220                             "location": "keep:99999999999999999999999999999998+99/file1.txt",
 221                             "size": 0
 222                         }],
 223                         'class': 'Directory'
 224                     },
 225                     'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 226                 },
 227                 'repository': 'arvados',
 228                 'script_version': 'master',
 229                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 230                 'script': 'cwl-runner'
 231             }
 232             stubs.pipeline_component = stubs.expect_job_spec.copy()
 233             stubs.expect_pipeline_instance = {
 234                 'name': 'submit_wf.cwl',
 235                 'state': 'RunningOnServer',
 236                 'owner_uuid': None,
 237                 "components": {
 238                     "cwl-runner": {
 239                         'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 240                         'script_parameters': {
 241                             'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 242                             'x': {"value": {
 243                                 'basename': 'blorp.txt',
 244                                 'class': 'File',
 245                                 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 246                                 "size": 16
 247                             }},
 248                             'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 249                                   'listing': [
 250                                       {
 251                                           'basename': 'renamed.txt',
 252                                           'class': 'File', 'location':
 253                                           'keep:99999999999999999999999999999998+99/file1.txt',
 254                                           'size': 0
 255                                       }
 256                                   ]}},
 257                             'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 258                             'arv:debug': True,
 259                             'arv:enable_reuse': True,
 260                             'arv:on_error': 'continue'
 261                         },
 262                         'repository': 'arvados',
 263                         'script_version': 'master',
 264                         'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 265                         'script': 'cwl-runner',
 266                         'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 267                     }
 268                 }
 269             }
 270             stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 271             stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 272             stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 273             stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 274             stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 275                 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 276                 "state": "Queued"
 277             }
 278             stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 279             stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 280
 281             cwd = os.getcwd()
 282             filepath = os.path.join(cwd, "tests/wf/submit_wf_packed.cwl")
 283             with open(filepath) as f:
 284                 expect_packed_workflow = yaml.round_trip_load(f)
 285
 286             if wfpath is None:
 287                 wfpath = wfname
 288
 289             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 290             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 291             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 292
 293             git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 294             expect_packed_workflow.update(git_info)
 295
 296             git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in git_info.items()}
 297
 298             if wfname == wfpath:
 299                 container_name = "%s (%s)" % (wfpath, git_props["arv:gitDescribe"])
 300             else:
 301                 container_name = wfname
 302
 303             stubs.expect_container_spec = {
 304                 'priority': 500,
 305                 'mounts': {
 306                     '/var/spool/cwl': {
 307                         'writable': True,
 308                         'kind': 'collection'
 309                     },
 310                     '/var/lib/cwl/workflow.json': {
 311                         'content': expect_packed_workflow,
 312                         'kind': 'json'
 313                     },
 314                     'stdout': {
 315                         'path': '/var/spool/cwl/cwl.output.json',
 316                         'kind': 'file'
 317                     },
 318                     '/var/lib/cwl/cwl.input.json': {
 319                         'kind': 'json',
 320                         'content': {
 321                             'y': {
 322                                 'basename': '99999999999999999999999999999998+99',
 323                                 'location': 'keep:99999999999999999999999999999998+99',
 324                                 'class': 'Directory'},
 325                             'x': {
 326                                 'basename': u'blorp.txt',
 327                                 'class': 'File',
 328                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 329                                 "size": 16
 330                             },
 331                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 332                                 {'basename': 'renamed.txt',
 333                                  'class': 'File',
 334                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 335                                  'size': 0
 336                                 }
 337                             ]}
 338                         },
 339                         'kind': 'json'
 340                     }
 341                 },
 342                 'secret_mounts': {},
 343                 'state': 'Committed',
 344                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 345                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 346                             '--eval-timeout=20', '--thread-count=0',
 347                             '--enable-reuse', "--collection-cache-size=256",
 348                             '--output-name=Output from workflow '+container_name,
 349                             '--debug', '--on-error=continue',
 350                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 351                 'name': container_name,
 352                 'container_image': '999999999999999999999999999999d3+99',
 353                 'output_name': 'Output from workflow %s' % (container_name),
 354                 'output_path': '/var/spool/cwl',
 355                 'cwd': '/var/spool/cwl',
 356                 'runtime_constraints': {
 357                     'API': True,
 358                     'vcpus': 1,
 359                     'ram': (1024+256)*1024*1024
 360                 },
 361                 'use_existing': False,
 362                 'properties': git_props,
 363                 'secret_mounts': {}
 364             }
 365
 366             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 367             stubs.api.workflows().create().execute.return_value = {
 368                 "uuid": stubs.expect_workflow_uuid,
 369             }
 370             def update_mock(**kwargs):
 371                 stubs.updated_uuid = kwargs.get('uuid')
 372                 return mock.DEFAULT
 373             stubs.api.workflows().update.side_effect = update_mock
 374             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 375                 "uuid": stubs.updated_uuid,
 376             }
 377
 378             return func(self, stubs, *args, **kwargs)
 379         return wrapped
 380     return outer_wrapper
 381
 382 class TestSubmit(unittest.TestCase):
 383
 384     def setUp(self):
 385         cwltool.process._names = set()
 386         arvados_cwl.arvdocker.arv_docker_clear_cache()
 387
 388     def tearDown(self):
 389         root_logger = logging.getLogger('')
 390
 391         # Remove existing RuntimeStatusLoggingHandlers if they exist
 392         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 393         root_logger.handlers = handlers
 394
 395     @mock.patch("time.sleep")
 396     @stubs
 397     def test_submit_invalid_runner_ram(self, stubs, tm):
 398         exited = arvados_cwl.main(
 399             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 400              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 401             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 402         self.assertEqual(exited, 1)
 403
 404
 405     @stubs
 406     def test_submit_container(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         stubs.api.collections().create.assert_has_calls([
 413             mock.call(body=JsonDiffMatcher({
 414                 'manifest_text':
 415                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 416                 'replication_desired': None,
 417                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 418             }), ensure_unique_name=False),
 419             mock.call(body=JsonDiffMatcher({
 420                 'manifest_text':
 421                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 422                 'replication_desired': None,
 423                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 424             }), ensure_unique_name=False),
 425             ])
 426
 427         expect_container = copy.deepcopy(stubs.expect_container_spec)
 428         stubs.api.container_requests().create.assert_called_with(
 429             body=JsonDiffMatcher(expect_container))
 430         self.assertEqual(stubs.capture_stdout.getvalue(),
 431                          stubs.expect_container_request_uuid + '\n')
 432         self.assertEqual(exited, 0)
 433
 434
 435     @stubs
 436     def test_submit_container_tool(self, stubs):
 437         # test for issue #16139
 438         exited = arvados_cwl.main(
 439             ["--submit", "--no-wait", "--api=containers", "--debug",
 440                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 441             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 442
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_container_request_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @stubs
 448     def test_submit_container_no_reuse(self, stubs):
 449         exited = arvados_cwl.main(
 450             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 451                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 452             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 453
 454         expect_container = copy.deepcopy(stubs.expect_container_spec)
 455         expect_container["command"] = [
 456             'arvados-cwl-runner', '--local', '--api=containers',
 457             '--no-log-timestamps', '--disable-validate', '--disable-color',
 458             '--eval-timeout=20', '--thread-count=0',
 459             '--disable-reuse', "--collection-cache-size=256",
 460             "--output-name=Output from workflow submit_wf.cwl",
 461             '--debug', '--on-error=continue',
 462             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 463         expect_container["use_existing"] = False
 464
 465         stubs.api.container_requests().create.assert_called_with(
 466             body=JsonDiffMatcher(expect_container))
 467         self.assertEqual(stubs.capture_stdout.getvalue(),
 468                          stubs.expect_container_request_uuid + '\n')
 469         self.assertEqual(exited, 0)
 470
 471     @stubs(('submit_wf_no_reuse.cwl', None))
 472     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 473         exited = arvados_cwl.main(
 474             ["--submit", "--no-wait", "--api=containers", "--debug",
 475              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 477         self.assertEqual(exited, 0)
 478
 479         expect_container = copy.deepcopy(stubs.expect_container_spec)
 480         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 481         expect_container["use_existing"] = False
 482         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 483             {
 484                 "class": "http://arvados.org/cwl#ReuseRequirement",
 485                 "enableReuse": False,
 486             },
 487         ]
 488         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 489             "arv": "http://arvados.org/cwl#",
 490             "cwltool": "http://commonwl.org/cwltool#"
 491         }
 492
 493         stubs.api.container_requests().create.assert_called_with(
 494             body=JsonDiffMatcher(expect_container))
 495         self.assertEqual(stubs.capture_stdout.getvalue(),
 496                          stubs.expect_container_request_uuid + '\n')
 497
 498
 499     @stubs
 500     def test_submit_container_on_error(self, stubs):
 501         exited = arvados_cwl.main(
 502             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 503                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505
 506         expect_container = copy.deepcopy(stubs.expect_container_spec)
 507         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 508                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 509                                        '--eval-timeout=20', '--thread-count=0',
 510                                        '--enable-reuse', "--collection-cache-size=256",
 511                                        "--output-name=Output from workflow submit_wf.cwl",
 512                                        '--debug', '--on-error=stop',
 513                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(stubs.capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519         self.assertEqual(exited, 0)
 520
 521     @stubs
 522     def test_submit_container_output_name(self, stubs):
 523         output_name = "test_output_name"
 524
 525         exited = arvados_cwl.main(
 526             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 527                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 528             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 529
 530         expect_container = copy.deepcopy(stubs.expect_container_spec)
 531         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 532                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 533                                        '--eval-timeout=20', '--thread-count=0',
 534                                        '--enable-reuse', "--collection-cache-size=256",
 535                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 536                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 537         expect_container["output_name"] = output_name
 538
 539         stubs.api.container_requests().create.assert_called_with(
 540             body=JsonDiffMatcher(expect_container))
 541         self.assertEqual(stubs.capture_stdout.getvalue(),
 542                          stubs.expect_container_request_uuid + '\n')
 543         self.assertEqual(exited, 0)
 544
 545     @stubs
 546     def test_submit_storage_classes(self, stubs):
 547         exited = arvados_cwl.main(
 548             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 549                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 550             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 551
 552         expect_container = copy.deepcopy(stubs.expect_container_spec)
 553         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 554                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 555                                        '--eval-timeout=20', '--thread-count=0',
 556                                        '--enable-reuse', "--collection-cache-size=256",
 557                                        '--output-name=Output from workflow submit_wf.cwl',
 558                                        "--debug",
 559                                        "--storage-classes=foo", '--on-error=continue',
 560                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 561
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(stubs.capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566         self.assertEqual(exited, 0)
 567
 568     @stubs
 569     def test_submit_multiple_storage_classes(self, stubs):
 570         exited = arvados_cwl.main(
 571             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 572                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 573             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 574
 575         expect_container = copy.deepcopy(stubs.expect_container_spec)
 576         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 577                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 578                                        '--eval-timeout=20', '--thread-count=0',
 579                                        '--enable-reuse', "--collection-cache-size=256",
 580                                        "--output-name=Output from workflow submit_wf.cwl",
 581                                        "--debug",
 582                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 583                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 584
 585         stubs.api.container_requests().create.assert_called_with(
 586             body=JsonDiffMatcher(expect_container))
 587         self.assertEqual(stubs.capture_stdout.getvalue(),
 588                          stubs.expect_container_request_uuid + '\n')
 589         self.assertEqual(exited, 0)
 590
 591     @mock.patch("cwltool.task_queue.TaskQueue")
 592     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 593     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 594     @stubs
 595     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 596         final_output_c = arvados.collection.Collection()
 597         make_output.return_value = ({},final_output_c)
 598
 599         def set_final_output(job_order, output_callback, runtimeContext):
 600             output_callback({"out": "zzzzz"}, "success")
 601             return []
 602         job.side_effect = set_final_output
 603
 604         exited = arvados_cwl.main(
 605             ["--debug", "--local", "--storage-classes=foo",
 606                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 607             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 608
 609         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 610         self.assertEqual(exited, 0)
 611
 612     @mock.patch("cwltool.task_queue.TaskQueue")
 613     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 614     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 615     @stubs
 616     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 617         final_output_c = arvados.collection.Collection()
 618         make_output.return_value = ({},final_output_c)
 619         stubs.api.config().get.return_value = {"default": {"Default": True}}
 620
 621         def set_final_output(job_order, output_callback, runtimeContext):
 622             output_callback({"out": "zzzzz"}, "success")
 623             return []
 624         job.side_effect = set_final_output
 625
 626         exited = arvados_cwl.main(
 627             ["--debug", "--local",
 628                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 629             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 630
 631         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 632         self.assertEqual(exited, 0)
 633
 634     @mock.patch("cwltool.task_queue.TaskQueue")
 635     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 636     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 637     @stubs
 638     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 639         final_output_c = arvados.collection.Collection()
 640         make_output.return_value = ({},final_output_c)
 641
 642         def set_final_output(job_order, output_callback, runtimeContext):
 643             output_callback({"out": "zzzzz"}, "success")
 644             return []
 645         job.side_effect = set_final_output
 646
 647         exited = arvados_cwl.main(
 648             ["--debug", "--local",
 649                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 650             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 651
 652         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 653         self.assertEqual(exited, 0)
 654
 655     @stubs
 656     def test_submit_container_output_ttl(self, stubs):
 657         exited = arvados_cwl.main(
 658             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 659                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 660             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 661
 662         expect_container = copy.deepcopy(stubs.expect_container_spec)
 663         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 664                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 665                                        '--eval-timeout=20', '--thread-count=0',
 666                                        '--enable-reuse', "--collection-cache-size=256",
 667                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 668                                        '--on-error=continue',
 669                                        "--intermediate-output-ttl=3600",
 670                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 671
 672         stubs.api.container_requests().create.assert_called_with(
 673             body=JsonDiffMatcher(expect_container))
 674         self.assertEqual(stubs.capture_stdout.getvalue(),
 675                          stubs.expect_container_request_uuid + '\n')
 676         self.assertEqual(exited, 0)
 677
 678     @stubs
 679     def test_submit_container_trash_intermediate(self, stubs):
 680         exited = arvados_cwl.main(
 681             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 682                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 683             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 684
 685
 686         expect_container = copy.deepcopy(stubs.expect_container_spec)
 687         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 688                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 689                                        '--eval-timeout=20', '--thread-count=0',
 690                                        '--enable-reuse', "--collection-cache-size=256",
 691                                        '--debug', '--on-error=continue',
 692                                        "--trash-intermediate",
 693                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 694
 695         stubs.api.container_requests().create.assert_called_with(
 696             body=JsonDiffMatcher(expect_container))
 697         self.assertEqual(stubs.capture_stdout.getvalue(),
 698                          stubs.expect_container_request_uuid + '\n')
 699         self.assertEqual(exited, 0)
 700
 701     @stubs
 702     def test_submit_container_output_tags(self, stubs):
 703         output_tags = "tag0,tag1,tag2"
 704
 705         exited = arvados_cwl.main(
 706             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 707                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 708             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 709
 710         expect_container = copy.deepcopy(stubs.expect_container_spec)
 711         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 712                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 713                                        '--eval-timeout=20', '--thread-count=0',
 714                                        '--enable-reuse', "--collection-cache-size=256",
 715                                        "--output-name=Output from workflow submit_wf.cwl",
 716                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 717                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 718
 719         stubs.api.container_requests().create.assert_called_with(
 720             body=JsonDiffMatcher(expect_container))
 721         self.assertEqual(stubs.capture_stdout.getvalue(),
 722                          stubs.expect_container_request_uuid + '\n')
 723         self.assertEqual(exited, 0)
 724
 725     @stubs
 726     def test_submit_container_runner_ram(self, stubs):
 727         exited = arvados_cwl.main(
 728             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 729                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 730             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 731
 732         expect_container = copy.deepcopy(stubs.expect_container_spec)
 733         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 734
 735         stubs.api.container_requests().create.assert_called_with(
 736             body=JsonDiffMatcher(expect_container))
 737         self.assertEqual(stubs.capture_stdout.getvalue(),
 738                          stubs.expect_container_request_uuid + '\n')
 739         self.assertEqual(exited, 0)
 740
 741     @mock.patch("arvados.collection.CollectionReader")
 742     @mock.patch("time.sleep")
 743     @stubs
 744     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 745         collectionReader().exists.return_value = True
 746         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 747         exited = arvados_cwl.main(
 748             ["--submit", "--no-wait", "--api=containers", "--debug",
 749              "tests/wf/submit_keepref_wf.cwl"],
 750             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 751         self.assertEqual(exited, 0)
 752
 753     @mock.patch("arvados.collection.CollectionReader")
 754     @mock.patch("time.sleep")
 755     @stubs
 756     def test_submit_keepref(self, stubs, tm, reader):
 757         with open("tests/wf/expect_arvworkflow.cwl") as f:
 758             reader().open().__enter__().read.return_value = f.read()
 759
 760         exited = arvados_cwl.main(
 761             ["--submit", "--no-wait", "--api=containers", "--debug",
 762              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 763             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 764
 765         expect_container = {
 766             'priority': 500,
 767             'mounts': {
 768                 '/var/spool/cwl': {
 769                     'writable': True,
 770                     'kind': 'collection'
 771                 },
 772                 'stdout': {
 773                     'path': '/var/spool/cwl/cwl.output.json',
 774                     'kind': 'file'
 775                 },
 776                 '/var/lib/cwl/workflow': {
 777                     'portable_data_hash': '99999999999999999999999999999994+99',
 778                     'kind': 'collection'
 779                 },
 780                 '/var/lib/cwl/cwl.input.json': {
 781                     'content': {
 782                         'x': 'XxX'
 783                     },
 784                     'kind': 'json'
 785                 }
 786             }, 'state': 'Committed',
 787             'output_path': '/var/spool/cwl',
 788             'name': 'expect_arvworkflow.cwl#main',
 789             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 790             'container_image': '999999999999999999999999999999d3+99',
 791             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 792                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 793                         '--eval-timeout=20', '--thread-count=0',
 794                         '--enable-reuse', "--collection-cache-size=256",
 795                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 796                         '--debug', '--on-error=continue',
 797                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 798             'cwd': '/var/spool/cwl',
 799             'runtime_constraints': {
 800                 'API': True,
 801                 'vcpus': 1,
 802                 'ram': 1342177280
 803             },
 804             'use_existing': False,
 805             'properties': {},
 806             'secret_mounts': {}
 807         }
 808
 809         stubs.api.container_requests().create.assert_called_with(
 810             body=JsonDiffMatcher(expect_container))
 811         self.assertEqual(stubs.capture_stdout.getvalue(),
 812                          stubs.expect_container_request_uuid + '\n')
 813         self.assertEqual(exited, 0)
 814
 815     @mock.patch("time.sleep")
 816     @stubs
 817     def test_submit_arvworkflow(self, stubs, tm):
 818         with open("tests/wf/expect_arvworkflow.cwl") as f:
 819             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 820
 821         exited = arvados_cwl.main(
 822             ["--submit", "--no-wait", "--api=containers", "--debug",
 823              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 824             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 825
 826         expect_container = {
 827             'priority': 500,
 828             'mounts': {
 829                 '/var/spool/cwl': {
 830                     'writable': True,
 831                     'kind': 'collection'
 832                 },
 833                 'stdout': {
 834                     'path': '/var/spool/cwl/cwl.output.json',
 835                     'kind': 'file'
 836                 },
 837                 '/var/lib/cwl/workflow.json': {
 838                     'kind': 'json',
 839                     'content': {
 840                         'cwlVersion': 'v1.0',
 841                         '$graph': [
 842                             {
 843                                 'id': '#main',
 844                                 'inputs': [
 845                                     {'type': 'string', 'id': '#main/x'}
 846                                 ],
 847                                 'steps': [
 848                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 849                                      'run': '#submit_tool.cwl',
 850                                      'id': '#main/step1',
 851                                      'out': []}
 852                                 ],
 853                                 'class': 'Workflow',
 854                                 'outputs': []
 855                             },
 856                             {
 857                                 'inputs': [
 858                                     {
 859                                         'inputBinding': {'position': 1},
 860                                         'type': 'string',
 861                                         'id': '#submit_tool.cwl/x'}
 862                                 ],
 863                                 'requirements': [
 864                                     {
 865                                         'dockerPull': 'debian:buster-slim',
 866                                         'class': 'DockerRequirement',
 867                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 868                                     }
 869                                 ],
 870                                 'id': '#submit_tool.cwl',
 871                                 'outputs': [],
 872                                 'baseCommand': 'cat',
 873                                 'class': 'CommandLineTool'
 874                             }
 875                         ]
 876                     }
 877                 },
 878                 '/var/lib/cwl/cwl.input.json': {
 879                     'content': {
 880                         'x': 'XxX'
 881                     },
 882                     'kind': 'json'
 883                 }
 884             }, 'state': 'Committed',
 885             'output_path': '/var/spool/cwl',
 886             'name': 'a test workflow',
 887             'container_image': "999999999999999999999999999999d3+99",
 888             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 889                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 890                         '--eval-timeout=20', '--thread-count=0',
 891                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 892                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 893             'cwd': '/var/spool/cwl',
 894             'runtime_constraints': {
 895                 'API': True,
 896                 'vcpus': 1,
 897                 'ram': 1342177280
 898             },
 899             'use_existing': False,
 900             'properties': {
 901                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 902             },
 903             'secret_mounts': {}
 904         }
 905
 906         stubs.api.container_requests().create.assert_called_with(
 907             body=JsonDiffMatcher(expect_container))
 908         self.assertEqual(stubs.capture_stdout.getvalue(),
 909                          stubs.expect_container_request_uuid + '\n')
 910         self.assertEqual(exited, 0)
 911
 912     @stubs(('hello container 123', 'submit_wf.cwl'))
 913     def test_submit_container_name(self, stubs):
 914         exited = arvados_cwl.main(
 915             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 916                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 917             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 918
 919         expect_container = copy.deepcopy(stubs.expect_container_spec)
 920
 921         stubs.api.container_requests().create.assert_called_with(
 922             body=JsonDiffMatcher(expect_container))
 923         self.assertEqual(stubs.capture_stdout.getvalue(),
 924                          stubs.expect_container_request_uuid + '\n')
 925         self.assertEqual(exited, 0)
 926
 927     @stubs
 928     def test_submit_missing_input(self, stubs):
 929         exited = arvados_cwl.main(
 930             ["--submit", "--no-wait", "--api=containers", "--debug",
 931              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 932             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 933         self.assertEqual(exited, 0)
 934
 935         exited = arvados_cwl.main(
 936             ["--submit", "--no-wait", "--api=containers", "--debug",
 937              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 938             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 939         self.assertEqual(exited, 1)
 940
 941     @stubs
 942     def test_submit_container_project(self, stubs):
 943         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 944         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 945         exited = arvados_cwl.main(
 946             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 947                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 948             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 949
 950         expect_container = copy.deepcopy(stubs.expect_container_spec)
 951         expect_container["owner_uuid"] = project_uuid
 952         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 953                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 954                                        "--eval-timeout=20", "--thread-count=0",
 955                                        '--enable-reuse', "--collection-cache-size=256",
 956                                        "--output-name=Output from workflow submit_wf.cwl", '--debug',
 957                                        '--on-error=continue',
 958                                        '--project-uuid='+project_uuid,
 959                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 960
 961         stubs.api.container_requests().create.assert_called_with(
 962             body=JsonDiffMatcher(expect_container))
 963         self.assertEqual(stubs.capture_stdout.getvalue(),
 964                          stubs.expect_container_request_uuid + '\n')
 965         self.assertEqual(exited, 0)
 966
 967     @stubs
 968     def test_submit_container_eval_timeout(self, stubs):
 969         exited = arvados_cwl.main(
 970             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 971                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 972             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 973
 974         expect_container = copy.deepcopy(stubs.expect_container_spec)
 975         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 976                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 977                                        '--eval-timeout=60.0', '--thread-count=0',
 978                                        '--enable-reuse', "--collection-cache-size=256",
 979                                        '--debug', '--on-error=continue',
 980                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 981
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     @stubs
 989     def test_submit_container_collection_cache(self, stubs):
 990         exited = arvados_cwl.main(
 991             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 992                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 993             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 994
 995         expect_container = copy.deepcopy(stubs.expect_container_spec)
 996         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 997                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 998                                        '--eval-timeout=20', '--thread-count=0',
 999                                        '--enable-reuse', "--collection-cache-size=500",
1000                                        '--debug', '--on-error=continue',
1001                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1002         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1003
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     @stubs
1011     def test_submit_container_thread_count(self, stubs):
1012         exited = arvados_cwl.main(
1013             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1014                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1015             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1016
1017         expect_container = copy.deepcopy(stubs.expect_container_spec)
1018         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1019                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1020                                        '--eval-timeout=20', '--thread-count=20',
1021                                        '--enable-reuse', "--collection-cache-size=256",
1022                                        '--debug', '--on-error=continue',
1023                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1024
1025         stubs.api.container_requests().create.assert_called_with(
1026             body=JsonDiffMatcher(expect_container))
1027         self.assertEqual(stubs.capture_stdout.getvalue(),
1028                          stubs.expect_container_request_uuid + '\n')
1029         self.assertEqual(exited, 0)
1030
1031     @stubs
1032     def test_submit_container_runner_image(self, stubs):
1033         exited = arvados_cwl.main(
1034             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1035                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1036             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1037
1038         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1039
1040         expect_container = copy.deepcopy(stubs.expect_container_spec)
1041         stubs.api.container_requests().create.assert_called_with(
1042             body=JsonDiffMatcher(expect_container))
1043         self.assertEqual(stubs.capture_stdout.getvalue(),
1044                          stubs.expect_container_request_uuid + '\n')
1045         self.assertEqual(exited, 0)
1046
1047     @stubs
1048     def test_submit_priority(self, stubs):
1049         exited = arvados_cwl.main(
1050             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1051                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1052             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1053
1054         stubs.expect_container_spec["priority"] = 669
1055
1056         expect_container = copy.deepcopy(stubs.expect_container_spec)
1057         stubs.api.container_requests().create.assert_called_with(
1058             body=JsonDiffMatcher(expect_container))
1059         self.assertEqual(stubs.capture_stdout.getvalue(),
1060                          stubs.expect_container_request_uuid + '\n')
1061         self.assertEqual(exited, 0)
1062
1063     @stubs(('submit_wf_runner_resources.cwl', None))
1064     def test_submit_wf_runner_resources(self, stubs):
1065         exited = arvados_cwl.main(
1066             ["--submit", "--no-wait", "--api=containers", "--debug",
1067                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1068             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1069
1070         expect_container = copy.deepcopy(stubs.expect_container_spec)
1071         expect_container["runtime_constraints"] = {
1072             "API": True,
1073             "vcpus": 2,
1074             "ram": (2000+512) * 2**20
1075         }
1076         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1077             {
1078                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1079                 "coresMin": 2,
1080                 "ramMin": 2000,
1081                 "keep_cache": 512
1082             }
1083         ]
1084         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1085             "arv": "http://arvados.org/cwl#",
1086         }
1087         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1088
1089         stubs.api.container_requests().create.assert_called_with(
1090             body=JsonDiffMatcher(expect_container))
1091         self.assertEqual(stubs.capture_stdout.getvalue(),
1092                          stubs.expect_container_request_uuid + '\n')
1093         self.assertEqual(exited, 0)
1094
1095     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1096     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1097     @mock.patch("arvados.api")
1098     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1099         arvados_cwl.arvdocker.arv_docker_clear_cache()
1100
1101         arvrunner = mock.MagicMock()
1102         arvrunner.project_uuid = ""
1103         api.return_value = mock.MagicMock()
1104         arvrunner.api = api.return_value
1105         arvrunner.runtimeContext.match_local_docker = False
1106         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1107                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1108                                                                         "link_class": "docker_image_repo+tag",
1109                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1110                                                                         "owner_uuid": "",
1111                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1112                                                             {"items": [{"created_at": "",
1113                                                                         "head_uuid": "",
1114                                                                         "link_class": "docker_image_hash",
1115                                                                         "name": "123456",
1116                                                                         "owner_uuid": "",
1117                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1118                                                             {"items": [{"created_at": "",
1119                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1120                                                                         "link_class": "docker_image_repo+tag",
1121                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1122                                                                         "owner_uuid": "",
1123                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1124                                                             {"items": [{"created_at": "",
1125                                                                         "head_uuid": "",
1126                                                                         "link_class": "docker_image_hash",
1127                                                                         "name": "123456",
1128                                                                         "owner_uuid": "",
1129                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1130         )
1131         find_one_image_hash.return_value = "123456"
1132
1133         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1134                                                                               "owner_uuid": "",
1135                                                                               "manifest_text": "",
1136                                                                               "properties": ""
1137                                                                               }], "items_available": 1, "offset": 0},
1138                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1139                                                                               "owner_uuid": "",
1140                                                                               "manifest_text": "",
1141                                                                               "properties": ""
1142                                                                           }], "items_available": 1, "offset": 0})
1143         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1144         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1145                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1146
1147         self.assertEqual("9999999999999999999999999999999b+99",
1148                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1149
1150
1151     @stubs
1152     def test_submit_secrets(self, stubs):
1153         exited = arvados_cwl.main(
1154             ["--submit", "--no-wait", "--api=containers", "--debug",
1155                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1156             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1157
1158         expect_container = {
1159             "command": [
1160                 "arvados-cwl-runner",
1161                 "--local",
1162                 "--api=containers",
1163                 "--no-log-timestamps",
1164                 "--disable-validate",
1165                 "--disable-color",
1166                 "--eval-timeout=20",
1167                 '--thread-count=0',
1168                 "--enable-reuse",
1169                 "--collection-cache-size=256",
1170                 '--output-name=Output from workflow secret_wf.cwl'
1171                 '--debug',
1172                 "--on-error=continue",
1173                 "/var/lib/cwl/workflow.json#main",
1174                 "/var/lib/cwl/cwl.input.json"
1175             ],
1176             "container_image": "999999999999999999999999999999d3+99",
1177             "cwd": "/var/spool/cwl",
1178             "mounts": {
1179                 "/var/lib/cwl/cwl.input.json": {
1180                     "content": {
1181                         "pw": {
1182                             "$include": "/secrets/s0"
1183                         }
1184                     },
1185                     "kind": "json"
1186                 },
1187                 "/var/lib/cwl/workflow.json": {
1188                     "content": {
1189                         "$graph": [
1190                             {
1191                                 "arguments": [
1192                                     "md5sum",
1193                                     "example.conf"
1194                                 ],
1195                                 "class": "CommandLineTool",
1196                                 "hints": [
1197                                     {
1198                                         "class": "http://commonwl.org/cwltool#Secrets",
1199                                         "secrets": [
1200                                             "#secret_job.cwl/pw"
1201                                         ]
1202                                     }
1203                                 ],
1204                                 "id": "#secret_job.cwl",
1205                                 "inputs": [
1206                                     {
1207                                         "id": "#secret_job.cwl/pw",
1208                                         "type": "string"
1209                                     }
1210                                 ],
1211                                 "outputs": [
1212                                     {
1213                                         "id": "#secret_job.cwl/out",
1214                                         "type": "File",
1215                                         "outputBinding": {
1216                                               "glob": "hashed_example.txt"
1217                                         }
1218                                     }
1219                                 ],
1220                                 "stdout": "hashed_example.txt",
1221                                 "requirements": [
1222                                     {
1223                                         "class": "InitialWorkDirRequirement",
1224                                         "listing": [
1225                                             {
1226                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1227                                                 "entryname": "example.conf"
1228                                             }
1229                                         ]
1230                                     }
1231                                 ]
1232                             },
1233                             {
1234                                 "class": "Workflow",
1235                                 "hints": [
1236                                     {
1237                                         "class": "DockerRequirement",
1238                                         "dockerPull": "debian:buster-slim",
1239                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1240                                     },
1241                                     {
1242                                         "class": "http://commonwl.org/cwltool#Secrets",
1243                                         "secrets": [
1244                                             "#main/pw"
1245                                         ]
1246                                     }
1247                                 ],
1248                                 "id": "#main",
1249                                 "inputs": [
1250                                     {
1251                                         "id": "#main/pw",
1252                                         "type": "string"
1253                                     }
1254                                 ],
1255                                 "outputs": [
1256                                     {
1257                                         "id": "#main/out",
1258                                         "outputSource": "#main/step1/out",
1259                                         "type": "File"
1260                                     }
1261                                 ],
1262                                 "steps": [
1263                                     {
1264                                         "id": "#main/step1",
1265                                         "in": [
1266                                             {
1267                                                 "id": "#main/step1/pw",
1268                                                 "source": "#main/pw"
1269                                             }
1270                                         ],
1271                                         "out": [
1272                                             "#main/step1/out"
1273                                         ],
1274                                         "run": "#secret_job.cwl"
1275                                     }
1276                                 ]
1277                             }
1278                         ],
1279                         "$namespaces": {
1280                             "cwltool": "http://commonwl.org/cwltool#"
1281                         },
1282                         "cwlVersion": "v1.0"
1283                     },
1284                     "kind": "json"
1285                 },
1286                 "/var/spool/cwl": {
1287                     "kind": "collection",
1288                     "writable": True
1289                 },
1290                 "stdout": {
1291                     "kind": "file",
1292                     "path": "/var/spool/cwl/cwl.output.json"
1293                 }
1294             },
1295             "name": "secret_wf.cwl",
1296             "output_name": "Output from workflow secret_wf.cwl",
1297             "output_path": "/var/spool/cwl",
1298             "priority": 500,
1299             "properties": {},
1300             "runtime_constraints": {
1301                 "API": True,
1302                 "ram": 1342177280,
1303                 "vcpus": 1
1304             },
1305             "secret_mounts": {
1306                 "/secrets/s0": {
1307                     "content": "blorp",
1308                     "kind": "text"
1309                 }
1310             },
1311             "state": "Committed",
1312             "use_existing": False
1313         }
1314
1315         stubs.api.container_requests().create.assert_called_with(
1316             body=JsonDiffMatcher(expect_container))
1317         self.assertEqual(stubs.capture_stdout.getvalue(),
1318                          stubs.expect_container_request_uuid + '\n')
1319         self.assertEqual(exited, 0)
1320
1321     @stubs
1322     def test_submit_request_uuid(self, stubs):
1323         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1324         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1325
1326         stubs.api.container_requests().update().execute.return_value = {
1327             "uuid": stubs.expect_container_request_uuid,
1328             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1329             "state": "Queued"
1330         }
1331
1332         exited = arvados_cwl.main(
1333             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1334                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1335             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1336
1337         stubs.api.container_requests().update.assert_called_with(
1338             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1339         self.assertEqual(stubs.capture_stdout.getvalue(),
1340                          stubs.expect_container_request_uuid + '\n')
1341         self.assertEqual(exited, 0)
1342
1343     @stubs
1344     def test_submit_container_cluster_id(self, stubs):
1345         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1346
1347         exited = arvados_cwl.main(
1348             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1349                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1350             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1351
1352         expect_container = copy.deepcopy(stubs.expect_container_spec)
1353
1354         stubs.api.container_requests().create.assert_called_with(
1355             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1356         self.assertEqual(stubs.capture_stdout.getvalue(),
1357                          stubs.expect_container_request_uuid + '\n')
1358         self.assertEqual(exited, 0)
1359
1360     @stubs
1361     def test_submit_validate_cluster_id(self, stubs):
1362         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1363         exited = arvados_cwl.main(
1364             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1365              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1366             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1367         self.assertEqual(exited, 1)
1368
1369     @stubs
1370     def test_submit_validate_project_uuid(self, stubs):
1371         # Fails with bad cluster prefix
1372         exited = arvados_cwl.main(
1373             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1374              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1375             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1376         self.assertEqual(exited, 1)
1377
1378         # Project lookup fails
1379         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1380         exited = arvados_cwl.main(
1381             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1382              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1383             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1384         self.assertEqual(exited, 1)
1385
1386         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1387         exited = arvados_cwl.main(
1388             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1390             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1391         self.assertEqual(exited, 0)
1392
1393
1394     @mock.patch("arvados.collection.CollectionReader")
1395     @stubs
1396     def test_submit_uuid_inputs(self, stubs, collectionReader):
1397         collectionReader().exists.return_value = True
1398         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1399         def list_side_effect(**kwargs):
1400             m = mock.MagicMock()
1401             if "count" in kwargs:
1402                 m.execute.return_value = {"items": [
1403                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1404                 ]}
1405             else:
1406                 m.execute.return_value = {"items": []}
1407             return m
1408         stubs.api.collections().list.side_effect = list_side_effect
1409
1410         exited = arvados_cwl.main(
1411             ["--submit", "--no-wait", "--api=containers", "--debug",
1412                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1413             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1414
1415         expect_container = copy.deepcopy(stubs.expect_container_spec)
1416         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1417         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1418         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1419
1420         stubs.api.collections().list.assert_has_calls([
1421             mock.call(count='none',
1422                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1423                       select=['uuid', 'portable_data_hash'])])
1424         stubs.api.container_requests().create.assert_called_with(
1425             body=JsonDiffMatcher(expect_container))
1426         self.assertEqual(stubs.capture_stdout.getvalue(),
1427                          stubs.expect_container_request_uuid + '\n')
1428         self.assertEqual(exited, 0)
1429
1430     @stubs
1431     def test_submit_mismatched_uuid_inputs(self, stubs):
1432         def list_side_effect(**kwargs):
1433             m = mock.MagicMock()
1434             if "count" in kwargs:
1435                 m.execute.return_value = {"items": [
1436                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1437                 ]}
1438             else:
1439                 m.execute.return_value = {"items": []}
1440             return m
1441         stubs.api.collections().list.side_effect = list_side_effect
1442
1443         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1444             capture_stderr = StringIO()
1445             cwltool_logger = logging.getLogger('cwltool')
1446             stderr_logger = logging.StreamHandler(capture_stderr)
1447             cwltool_logger.addHandler(stderr_logger)
1448
1449             try:
1450                 exited = arvados_cwl.main(
1451                     ["--submit", "--no-wait", "--api=containers", "--debug",
1452                         "tests/wf/submit_wf.cwl", infile],
1453                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1454
1455                 self.assertEqual(exited, 1)
1456                 self.assertRegex(
1457                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1458                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1459             finally:
1460                 cwltool_logger.removeHandler(stderr_logger)
1461
1462     @mock.patch("arvados.collection.CollectionReader")
1463     @stubs
1464     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1465         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1466         capture_stderr = StringIO()
1467
1468         cwltool_logger = logging.getLogger('cwltool')
1469         stderr_logger = logging.StreamHandler(capture_stderr)
1470         cwltool_logger.addHandler(stderr_logger)
1471
1472         exited = arvados_cwl.main(
1473             ["--submit", "--no-wait", "--api=containers", "--debug",
1474                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1475             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1476
1477         try:
1478             self.assertEqual(exited, 1)
1479             self.assertRegex(
1480                 capture_stderr.getvalue(),
1481                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1482         finally:
1483             cwltool_logger.removeHandler(stderr_logger)
1484
1485     @stubs(('submit_wf_process_properties.cwl', None))
1486     def test_submit_set_process_properties(self, stubs):
1487         exited = arvados_cwl.main(
1488             ["--submit", "--no-wait", "--api=containers", "--debug",
1489                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1490             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1491
1492         expect_container = copy.deepcopy(stubs.expect_container_spec)
1493
1494         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1495             {
1496                 "class": "http://arvados.org/cwl#ProcessProperties",
1497                 "processProperties": [
1498                     {"propertyName": "baz",
1499                      "propertyValue": "$(inputs.x.basename)"},
1500                     {"propertyName": "foo",
1501                      "propertyValue": "bar"},
1502                     {"propertyName": "quux",
1503                      "propertyValue": {
1504                          "q1": 1,
1505                          "q2": 2
1506                      }
1507                     }
1508                 ],
1509             }
1510         ]
1511         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1512             "arv": "http://arvados.org/cwl#"
1513         }
1514
1515         expect_container["properties"].update({
1516             "baz": "blorp.txt",
1517             "foo": "bar",
1518             "quux": {
1519                 "q1": 1,
1520                 "q2": 2
1521             }
1522         })
1523
1524         stubs.api.container_requests().create.assert_called_with(
1525             body=JsonDiffMatcher(expect_container))
1526         self.assertEqual(stubs.capture_stdout.getvalue(),
1527                          stubs.expect_container_request_uuid + '\n')
1528         self.assertEqual(exited, 0)
1529
1530
1531     @stubs
1532     def test_submit_enable_preemptible(self, stubs):
1533         exited = arvados_cwl.main(
1534             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1535                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1536             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1537
1538         expect_container = copy.deepcopy(stubs.expect_container_spec)
1539         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1540                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1541                         '--eval-timeout=20', '--thread-count=0',
1542                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1543                                        '--enable-preemptible',
1544                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1545
1546         stubs.api.container_requests().create.assert_called_with(
1547             body=JsonDiffMatcher(expect_container))
1548         self.assertEqual(stubs.capture_stdout.getvalue(),
1549                          stubs.expect_container_request_uuid + '\n')
1550         self.assertEqual(exited, 0)
1551
1552     @stubs
1553     def test_submit_disable_preemptible(self, stubs):
1554         exited = arvados_cwl.main(
1555             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1556                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1557             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1558
1559         expect_container = copy.deepcopy(stubs.expect_container_spec)
1560         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1561                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1562                         '--eval-timeout=20', '--thread-count=0',
1563                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1564                                        '--disable-preemptible',
1565                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1566
1567         stubs.api.container_requests().create.assert_called_with(
1568             body=JsonDiffMatcher(expect_container))
1569         self.assertEqual(stubs.capture_stdout.getvalue(),
1570                          stubs.expect_container_request_uuid + '\n')
1571         self.assertEqual(exited, 0)
1572
1573
1574 class TestCreateWorkflow(unittest.TestCase):
1575     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1576     expect_workflow = StripYAMLComments(
1577         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1578
1579     def setUp(self):
1580         cwltool.process._names = set()
1581         arvados_cwl.arvdocker.arv_docker_clear_cache()
1582
1583     def tearDown(self):
1584         root_logger = logging.getLogger('')
1585
1586         # Remove existing RuntimeStatusLoggingHandlers if they exist
1587         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1588         root_logger.handlers = handlers
1589
1590     @stubs
1591     def test_create(self, stubs):
1592         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1593         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1594
1595         exited = arvados_cwl.main(
1596             ["--create-workflow", "--debug",
1597              "--api=containers",
1598              "--project-uuid", project_uuid,
1599              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1600             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1601
1602         stubs.api.pipeline_templates().create.refute_called()
1603         stubs.api.container_requests().create.refute_called()
1604
1605         body = {
1606             "workflow": {
1607                 "owner_uuid": project_uuid,
1608                 "name": "submit_wf.cwl",
1609                 "description": "",
1610                 "definition": self.expect_workflow,
1611             }
1612         }
1613         stubs.api.workflows().create.assert_called_with(
1614             body=JsonDiffMatcher(body))
1615
1616         self.assertEqual(stubs.capture_stdout.getvalue(),
1617                          stubs.expect_workflow_uuid + '\n')
1618         self.assertEqual(exited, 0)
1619
1620     @stubs
1621     def test_create_name(self, stubs):
1622         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1623         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1624
1625         exited = arvados_cwl.main(
1626             ["--create-workflow", "--debug",
1627              "--api=containers",
1628              "--project-uuid", project_uuid,
1629              "--name", "testing 123",
1630              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1631             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1632
1633         stubs.api.pipeline_templates().create.refute_called()
1634         stubs.api.container_requests().create.refute_called()
1635
1636         body = {
1637             "workflow": {
1638                 "owner_uuid": project_uuid,
1639                 "name": "testing 123",
1640                 "description": "",
1641                 "definition": self.expect_workflow,
1642             }
1643         }
1644         stubs.api.workflows().create.assert_called_with(
1645             body=JsonDiffMatcher(body))
1646
1647         self.assertEqual(stubs.capture_stdout.getvalue(),
1648                          stubs.expect_workflow_uuid + '\n')
1649         self.assertEqual(exited, 0)
1650
1651
1652     @stubs
1653     def test_update(self, stubs):
1654         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1655         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1656
1657         exited = arvados_cwl.main(
1658             ["--update-workflow", self.existing_workflow_uuid,
1659              "--debug",
1660              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1661             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1662
1663         body = {
1664             "workflow": {
1665                 "name": "submit_wf.cwl",
1666                 "description": "",
1667                 "definition": self.expect_workflow,
1668                 "owner_uuid": project_uuid
1669             }
1670         }
1671         stubs.api.workflows().update.assert_called_with(
1672             uuid=self.existing_workflow_uuid,
1673             body=JsonDiffMatcher(body))
1674         self.assertEqual(stubs.capture_stdout.getvalue(),
1675                          self.existing_workflow_uuid + '\n')
1676         self.assertEqual(exited, 0)
1677
1678
1679     @stubs
1680     def test_update_name(self, stubs):
1681         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1682         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1683
1684         exited = arvados_cwl.main(
1685             ["--update-workflow", self.existing_workflow_uuid,
1686              "--debug", "--name", "testing 123",
1687              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1688             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1689
1690         body = {
1691             "workflow": {
1692                 "name": "testing 123",
1693                 "description": "",
1694                 "definition": self.expect_workflow,
1695                 "owner_uuid": project_uuid
1696             }
1697         }
1698         stubs.api.workflows().update.assert_called_with(
1699             uuid=self.existing_workflow_uuid,
1700             body=JsonDiffMatcher(body))
1701         self.assertEqual(stubs.capture_stdout.getvalue(),
1702                          self.existing_workflow_uuid + '\n')
1703         self.assertEqual(exited, 0)
1704
1705     @stubs
1706     def test_create_collection_per_tool(self, stubs):
1707         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1708         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1709
1710         exited = arvados_cwl.main(
1711             ["--create-workflow", "--debug",
1712              "--api=containers",
1713              "--project-uuid", project_uuid,
1714              "tests/collection_per_tool/collection_per_tool.cwl"],
1715             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1716
1717         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1718         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1719
1720         body = {
1721             "workflow": {
1722                 "owner_uuid": project_uuid,
1723                 "name": "collection_per_tool.cwl",
1724                 "description": "",
1725                 "definition": expect_workflow,
1726             }
1727         }
1728         stubs.api.workflows().create.assert_called_with(
1729             body=JsonDiffMatcher(body))
1730
1731         self.assertEqual(stubs.capture_stdout.getvalue(),
1732                          stubs.expect_workflow_uuid + '\n')
1733         self.assertEqual(exited, 0)
1734
1735     @stubs
1736     def test_create_with_imports(self, stubs):
1737         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1738         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1739
1740         exited = arvados_cwl.main(
1741             ["--create-workflow", "--debug",
1742              "--api=containers",
1743              "--project-uuid", project_uuid,
1744              "tests/wf/feddemo/feddemo.cwl"],
1745             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1746
1747         stubs.api.pipeline_templates().create.refute_called()
1748         stubs.api.container_requests().create.refute_called()
1749
1750         self.assertEqual(stubs.capture_stdout.getvalue(),
1751                          stubs.expect_workflow_uuid + '\n')
1752         self.assertEqual(exited, 0)
1753
1754     @stubs
1755     def test_create_with_no_input(self, stubs):
1756         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1757         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1758
1759         exited = arvados_cwl.main(
1760             ["--create-workflow", "--debug",
1761              "--api=containers",
1762              "--project-uuid", project_uuid,
1763              "tests/wf/revsort/revsort.cwl"],
1764             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1765
1766         stubs.api.pipeline_templates().create.refute_called()
1767         stubs.api.container_requests().create.refute_called()
1768
1769         self.assertEqual(stubs.capture_stdout.getvalue(),
1770                          stubs.expect_workflow_uuid + '\n')
1771         self.assertEqual(exited, 0)