sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=4',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343
 344     @stubs
 345     def test_error_when_multiple_storage_classes_specified(self, stubs):
 346         storage_classes = "foo,bar"
 347         exited = arvados_cwl.main(
 348                 ["--debug", "--storage-classes", storage_classes,
 349                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 350                 sys.stdin, sys.stderr, api_client=stubs.api)
 351         self.assertEqual(exited, 1)
 352
 353     @mock.patch("time.sleep")
 354     @stubs
 355     def test_submit_invalid_runner_ram(self, stubs, tm):
 356         exited = arvados_cwl.main(
 357             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 359             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 360         self.assertEqual(exited, 1)
 361
 362
 363     @stubs
 364     def test_submit_container(self, stubs):
 365         exited = arvados_cwl.main(
 366             ["--submit", "--no-wait", "--api=containers", "--debug",
 367                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 368             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 369
 370         stubs.api.collections().create.assert_has_calls([
 371             mock.call(body=JsonDiffMatcher({
 372                 'manifest_text':
 373                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 374                 'replication_desired': None,
 375                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 376             }), ensure_unique_name=False),
 377             mock.call(body=JsonDiffMatcher({
 378                 'manifest_text':
 379                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 380                 'replication_desired': None,
 381                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 382             }), ensure_unique_name=False),
 383             ])
 384
 385         expect_container = copy.deepcopy(stubs.expect_container_spec)
 386         stubs.api.container_requests().create.assert_called_with(
 387             body=JsonDiffMatcher(expect_container))
 388         self.assertEqual(stubs.capture_stdout.getvalue(),
 389                          stubs.expect_container_request_uuid + '\n')
 390         self.assertEqual(exited, 0)
 391
 392
 393     @stubs
 394     def test_submit_container_tool(self, stubs):
 395         # test for issue #16139
 396         exited = arvados_cwl.main(
 397             ["--submit", "--no-wait", "--api=containers", "--debug",
 398                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 399             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 400
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_submit_container_no_reuse(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = [
 414             'arvados-cwl-runner', '--local', '--api=containers',
 415             '--no-log-timestamps', '--disable-validate', '--disable-color',
 416             '--eval-timeout=20', '--thread-count=4',
 417             '--disable-reuse', "--collection-cache-size=256",
 418             '--debug', '--on-error=continue',
 419             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420         expect_container["use_existing"] = False
 421
 422         stubs.api.container_requests().create.assert_called_with(
 423             body=JsonDiffMatcher(expect_container))
 424         self.assertEqual(stubs.capture_stdout.getvalue(),
 425                          stubs.expect_container_request_uuid + '\n')
 426         self.assertEqual(exited, 0)
 427
 428     @stubs
 429     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 430         exited = arvados_cwl.main(
 431             ["--submit", "--no-wait", "--api=containers", "--debug",
 432              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 433             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 434         self.assertEqual(exited, 0)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = [
 438             'arvados-cwl-runner', '--local', '--api=containers',
 439             '--no-log-timestamps', '--disable-validate', '--disable-color',
 440             '--eval-timeout=20', '--thread-count=4',
 441             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 442             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["use_existing"] = False
 444         expect_container["name"] = "submit_wf_no_reuse.cwl"
 445         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 446             {
 447                 "class": "http://arvados.org/cwl#ReuseRequirement",
 448                 "enableReuse": False,
 449             },
 450         ]
 451         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 452             "arv": "http://arvados.org/cwl#",
 453             "cwltool": "http://commonwl.org/cwltool#"
 454         }
 455
 456         stubs.api.container_requests().create.assert_called_with(
 457             body=JsonDiffMatcher(expect_container))
 458         self.assertEqual(stubs.capture_stdout.getvalue(),
 459                          stubs.expect_container_request_uuid + '\n')
 460
 461
 462     @stubs
 463     def test_submit_container_on_error(self, stubs):
 464         exited = arvados_cwl.main(
 465             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 466                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468
 469         expect_container = copy.deepcopy(stubs.expect_container_spec)
 470         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 471                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 472                                        '--eval-timeout=20', '--thread-count=4',
 473                                        '--enable-reuse', "--collection-cache-size=256",
 474                                        '--debug', '--on-error=stop',
 475                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 476
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @stubs
 484     def test_submit_container_output_name(self, stubs):
 485         output_name = "test_output_name"
 486
 487         exited = arvados_cwl.main(
 488             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 489                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 490             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 491
 492         expect_container = copy.deepcopy(stubs.expect_container_spec)
 493         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 494                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 495                                        '--eval-timeout=20', '--thread-count=4',
 496                                        '--enable-reuse', "--collection-cache-size=256",
 497                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 498                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 499         expect_container["output_name"] = output_name
 500
 501         stubs.api.container_requests().create.assert_called_with(
 502             body=JsonDiffMatcher(expect_container))
 503         self.assertEqual(stubs.capture_stdout.getvalue(),
 504                          stubs.expect_container_request_uuid + '\n')
 505         self.assertEqual(exited, 0)
 506
 507     @stubs
 508     def test_submit_storage_classes(self, stubs):
 509         exited = arvados_cwl.main(
 510             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 511                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 512             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 513
 514         expect_container = copy.deepcopy(stubs.expect_container_spec)
 515         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 516                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 517                                        '--eval-timeout=20', '--thread-count=4',
 518                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 519                                        "--storage-classes=foo", '--on-error=continue',
 520                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 521
 522         stubs.api.container_requests().create.assert_called_with(
 523             body=JsonDiffMatcher(expect_container))
 524         self.assertEqual(stubs.capture_stdout.getvalue(),
 525                          stubs.expect_container_request_uuid + '\n')
 526         self.assertEqual(exited, 0)
 527
 528     @mock.patch("cwltool.task_queue.TaskQueue")
 529     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 530     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 531     @stubs
 532     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 533         final_output_c = arvados.collection.Collection()
 534         make_output.return_value = ({},final_output_c)
 535
 536         def set_final_output(job_order, output_callback, runtimeContext):
 537             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 538             return []
 539         job.side_effect = set_final_output
 540
 541         exited = arvados_cwl.main(
 542             ["--debug", "--local", "--storage-classes=foo",
 543                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 544             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 545
 546         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 547         self.assertEqual(exited, 0)
 548
 549     @mock.patch("cwltool.task_queue.TaskQueue")
 550     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 551     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 552     @stubs
 553     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 554         final_output_c = arvados.collection.Collection()
 555         make_output.return_value = ({},final_output_c)
 556
 557         def set_final_output(job_order, output_callback, runtimeContext):
 558             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 559             return []
 560         job.side_effect = set_final_output
 561
 562         exited = arvados_cwl.main(
 563             ["--debug", "--local",
 564                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 565             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 566
 567         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 568         self.assertEqual(exited, 0)
 569
 570     @stubs
 571     def test_submit_container_output_ttl(self, stubs):
 572         exited = arvados_cwl.main(
 573             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         expect_container = copy.deepcopy(stubs.expect_container_spec)
 578         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 579                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 580                                        '--eval-timeout=20', '--thread-count=4',
 581                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 582                                        '--on-error=continue',
 583                                        "--intermediate-output-ttl=3600",
 584                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 585
 586         stubs.api.container_requests().create.assert_called_with(
 587             body=JsonDiffMatcher(expect_container))
 588         self.assertEqual(stubs.capture_stdout.getvalue(),
 589                          stubs.expect_container_request_uuid + '\n')
 590         self.assertEqual(exited, 0)
 591
 592     @stubs
 593     def test_submit_container_trash_intermediate(self, stubs):
 594         exited = arvados_cwl.main(
 595             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 596                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598
 599
 600         expect_container = copy.deepcopy(stubs.expect_container_spec)
 601         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 602                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 603                                        '--eval-timeout=20', '--thread-count=4',
 604                                        '--enable-reuse', "--collection-cache-size=256",
 605                                        '--debug', '--on-error=continue',
 606                                        "--trash-intermediate",
 607                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 608
 609         stubs.api.container_requests().create.assert_called_with(
 610             body=JsonDiffMatcher(expect_container))
 611         self.assertEqual(stubs.capture_stdout.getvalue(),
 612                          stubs.expect_container_request_uuid + '\n')
 613         self.assertEqual(exited, 0)
 614
 615     @stubs
 616     def test_submit_container_output_tags(self, stubs):
 617         output_tags = "tag0,tag1,tag2"
 618
 619         exited = arvados_cwl.main(
 620             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 621                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 622             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 623
 624         expect_container = copy.deepcopy(stubs.expect_container_spec)
 625         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 626                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 627                                        '--eval-timeout=20', '--thread-count=4',
 628                                        '--enable-reuse', "--collection-cache-size=256",
 629                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 630                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 631
 632         stubs.api.container_requests().create.assert_called_with(
 633             body=JsonDiffMatcher(expect_container))
 634         self.assertEqual(stubs.capture_stdout.getvalue(),
 635                          stubs.expect_container_request_uuid + '\n')
 636         self.assertEqual(exited, 0)
 637
 638     @stubs
 639     def test_submit_container_runner_ram(self, stubs):
 640         exited = arvados_cwl.main(
 641             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 642                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 643             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 644
 645         expect_container = copy.deepcopy(stubs.expect_container_spec)
 646         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 647
 648         stubs.api.container_requests().create.assert_called_with(
 649             body=JsonDiffMatcher(expect_container))
 650         self.assertEqual(stubs.capture_stdout.getvalue(),
 651                          stubs.expect_container_request_uuid + '\n')
 652         self.assertEqual(exited, 0)
 653
 654     @mock.patch("arvados.collection.CollectionReader")
 655     @mock.patch("time.sleep")
 656     @stubs
 657     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 658         collectionReader().exists.return_value = True
 659         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 660         exited = arvados_cwl.main(
 661             ["--submit", "--no-wait", "--api=containers", "--debug",
 662              "tests/wf/submit_keepref_wf.cwl"],
 663             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 664         self.assertEqual(exited, 0)
 665
 666     @mock.patch("arvados.collection.CollectionReader")
 667     @mock.patch("time.sleep")
 668     @stubs
 669     def test_submit_keepref(self, stubs, tm, reader):
 670         with open("tests/wf/expect_arvworkflow.cwl") as f:
 671             reader().open().__enter__().read.return_value = f.read()
 672
 673         exited = arvados_cwl.main(
 674             ["--submit", "--no-wait", "--api=containers", "--debug",
 675              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 676             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 677
 678         expect_container = {
 679             'priority': 500,
 680             'mounts': {
 681                 '/var/spool/cwl': {
 682                     'writable': True,
 683                     'kind': 'collection'
 684                 },
 685                 'stdout': {
 686                     'path': '/var/spool/cwl/cwl.output.json',
 687                     'kind': 'file'
 688                 },
 689                 '/var/lib/cwl/workflow': {
 690                     'portable_data_hash': '99999999999999999999999999999994+99',
 691                     'kind': 'collection'
 692                 },
 693                 '/var/lib/cwl/cwl.input.json': {
 694                     'content': {
 695                         'x': 'XxX'
 696                     },
 697                     'kind': 'json'
 698                 }
 699             }, 'state': 'Committed',
 700             'output_path': '/var/spool/cwl',
 701             'name': 'expect_arvworkflow.cwl#main',
 702             'container_image': '999999999999999999999999999999d3+99',
 703             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 704                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 705                         '--eval-timeout=20', '--thread-count=4',
 706                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 707                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 708             'cwd': '/var/spool/cwl',
 709             'runtime_constraints': {
 710                 'API': True,
 711                 'vcpus': 1,
 712                 'ram': 1342177280
 713             },
 714             'use_existing': False,
 715             'properties': {},
 716             'secret_mounts': {}
 717         }
 718
 719         stubs.api.container_requests().create.assert_called_with(
 720             body=JsonDiffMatcher(expect_container))
 721         self.assertEqual(stubs.capture_stdout.getvalue(),
 722                          stubs.expect_container_request_uuid + '\n')
 723         self.assertEqual(exited, 0)
 724
 725     @mock.patch("time.sleep")
 726     @stubs
 727     def test_submit_arvworkflow(self, stubs, tm):
 728         with open("tests/wf/expect_arvworkflow.cwl") as f:
 729             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 730
 731         exited = arvados_cwl.main(
 732             ["--submit", "--no-wait", "--api=containers", "--debug",
 733              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 734             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 735
 736         expect_container = {
 737             'priority': 500,
 738             'mounts': {
 739                 '/var/spool/cwl': {
 740                     'writable': True,
 741                     'kind': 'collection'
 742                 },
 743                 'stdout': {
 744                     'path': '/var/spool/cwl/cwl.output.json',
 745                     'kind': 'file'
 746                 },
 747                 '/var/lib/cwl/workflow.json': {
 748                     'kind': 'json',
 749                     'content': {
 750                         'cwlVersion': 'v1.0',
 751                         '$graph': [
 752                             {
 753                                 'id': '#main',
 754                                 'inputs': [
 755                                     {'type': 'string', 'id': '#main/x'}
 756                                 ],
 757                                 'steps': [
 758                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 759                                      'run': '#submit_tool.cwl',
 760                                      'id': '#main/step1',
 761                                      'out': []}
 762                                 ],
 763                                 'class': 'Workflow',
 764                                 'outputs': []
 765                             },
 766                             {
 767                                 'inputs': [
 768                                     {
 769                                         'inputBinding': {'position': 1},
 770                                         'type': 'string',
 771                                         'id': '#submit_tool.cwl/x'}
 772                                 ],
 773                                 'requirements': [
 774                                     {
 775                                         'dockerPull': 'debian:buster-slim',
 776                                         'class': 'DockerRequirement',
 777                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 778                                     }
 779                                 ],
 780                                 'id': '#submit_tool.cwl',
 781                                 'outputs': [],
 782                                 'baseCommand': 'cat',
 783                                 'class': 'CommandLineTool'
 784                             }
 785                         ]
 786                     }
 787                 },
 788                 '/var/lib/cwl/cwl.input.json': {
 789                     'content': {
 790                         'x': 'XxX'
 791                     },
 792                     'kind': 'json'
 793                 }
 794             }, 'state': 'Committed',
 795             'output_path': '/var/spool/cwl',
 796             'name': 'a test workflow',
 797             'container_image': "999999999999999999999999999999d3+99",
 798             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 799                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 800                         '--eval-timeout=20', '--thread-count=4',
 801                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 802                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 803             'cwd': '/var/spool/cwl',
 804             'runtime_constraints': {
 805                 'API': True,
 806                 'vcpus': 1,
 807                 'ram': 1342177280
 808             },
 809             'use_existing': False,
 810             'properties': {
 811                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 812             },
 813             'secret_mounts': {}
 814         }
 815
 816         stubs.api.container_requests().create.assert_called_with(
 817             body=JsonDiffMatcher(expect_container))
 818         self.assertEqual(stubs.capture_stdout.getvalue(),
 819                          stubs.expect_container_request_uuid + '\n')
 820         self.assertEqual(exited, 0)
 821
 822     @stubs
 823     def test_submit_container_name(self, stubs):
 824         exited = arvados_cwl.main(
 825             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 826                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 827             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 828
 829         expect_container = copy.deepcopy(stubs.expect_container_spec)
 830         expect_container["name"] = "hello container 123"
 831
 832         stubs.api.container_requests().create.assert_called_with(
 833             body=JsonDiffMatcher(expect_container))
 834         self.assertEqual(stubs.capture_stdout.getvalue(),
 835                          stubs.expect_container_request_uuid + '\n')
 836         self.assertEqual(exited, 0)
 837
 838     @stubs
 839     def test_submit_missing_input(self, stubs):
 840         exited = arvados_cwl.main(
 841             ["--submit", "--no-wait", "--api=containers", "--debug",
 842              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 843             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 844         self.assertEqual(exited, 0)
 845
 846         exited = arvados_cwl.main(
 847             ["--submit", "--no-wait", "--api=containers", "--debug",
 848              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 849             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 850         self.assertEqual(exited, 1)
 851
 852     @stubs
 853     def test_submit_container_project(self, stubs):
 854         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 855         exited = arvados_cwl.main(
 856             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 857                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 858             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 859
 860         expect_container = copy.deepcopy(stubs.expect_container_spec)
 861         expect_container["owner_uuid"] = project_uuid
 862         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 863                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 864                                        "--eval-timeout=20", "--thread-count=4",
 865                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 866                                        '--on-error=continue',
 867                                        '--project-uuid='+project_uuid,
 868                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 869
 870         stubs.api.container_requests().create.assert_called_with(
 871             body=JsonDiffMatcher(expect_container))
 872         self.assertEqual(stubs.capture_stdout.getvalue(),
 873                          stubs.expect_container_request_uuid + '\n')
 874         self.assertEqual(exited, 0)
 875
 876     @stubs
 877     def test_submit_container_eval_timeout(self, stubs):
 878         exited = arvados_cwl.main(
 879             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 880                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 881             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 882
 883         expect_container = copy.deepcopy(stubs.expect_container_spec)
 884         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 885                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 886                                        '--eval-timeout=60.0', '--thread-count=4',
 887                                        '--enable-reuse', "--collection-cache-size=256",
 888                                        '--debug', '--on-error=continue',
 889                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 890
 891         stubs.api.container_requests().create.assert_called_with(
 892             body=JsonDiffMatcher(expect_container))
 893         self.assertEqual(stubs.capture_stdout.getvalue(),
 894                          stubs.expect_container_request_uuid + '\n')
 895         self.assertEqual(exited, 0)
 896
 897     @stubs
 898     def test_submit_container_collection_cache(self, stubs):
 899         exited = arvados_cwl.main(
 900             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 901                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 902             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 903
 904         expect_container = copy.deepcopy(stubs.expect_container_spec)
 905         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 906                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 907                                        '--eval-timeout=20', '--thread-count=4',
 908                                        '--enable-reuse', "--collection-cache-size=500",
 909                                        '--debug', '--on-error=continue',
 910                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 911         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(stubs.capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917         self.assertEqual(exited, 0)
 918
 919     @stubs
 920     def test_submit_container_thread_count(self, stubs):
 921         exited = arvados_cwl.main(
 922             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 923                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 924             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 925
 926         expect_container = copy.deepcopy(stubs.expect_container_spec)
 927         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 928                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 929                                        '--eval-timeout=20', '--thread-count=20',
 930                                        '--enable-reuse', "--collection-cache-size=256",
 931                                        '--debug', '--on-error=continue',
 932                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 933
 934         stubs.api.container_requests().create.assert_called_with(
 935             body=JsonDiffMatcher(expect_container))
 936         self.assertEqual(stubs.capture_stdout.getvalue(),
 937                          stubs.expect_container_request_uuid + '\n')
 938         self.assertEqual(exited, 0)
 939
 940     @stubs
 941     def test_submit_container_runner_image(self, stubs):
 942         exited = arvados_cwl.main(
 943             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 944                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 945             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 946
 947         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 948
 949         expect_container = copy.deepcopy(stubs.expect_container_spec)
 950         stubs.api.container_requests().create.assert_called_with(
 951             body=JsonDiffMatcher(expect_container))
 952         self.assertEqual(stubs.capture_stdout.getvalue(),
 953                          stubs.expect_container_request_uuid + '\n')
 954         self.assertEqual(exited, 0)
 955
 956     @stubs
 957     def test_submit_priority(self, stubs):
 958         exited = arvados_cwl.main(
 959             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 960                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 961             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 962
 963         stubs.expect_container_spec["priority"] = 669
 964
 965         expect_container = copy.deepcopy(stubs.expect_container_spec)
 966         stubs.api.container_requests().create.assert_called_with(
 967             body=JsonDiffMatcher(expect_container))
 968         self.assertEqual(stubs.capture_stdout.getvalue(),
 969                          stubs.expect_container_request_uuid + '\n')
 970         self.assertEqual(exited, 0)
 971
 972     @stubs
 973     def test_submit_wf_runner_resources(self, stubs):
 974         exited = arvados_cwl.main(
 975             ["--submit", "--no-wait", "--api=containers", "--debug",
 976                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 977             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 978
 979         expect_container = copy.deepcopy(stubs.expect_container_spec)
 980         expect_container["runtime_constraints"] = {
 981             "API": True,
 982             "vcpus": 2,
 983             "ram": (2000+512) * 2**20
 984         }
 985         expect_container["name"] = "submit_wf_runner_resources.cwl"
 986         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 987             {
 988                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 989                 "coresMin": 2,
 990                 "ramMin": 2000,
 991                 "keep_cache": 512
 992             }
 993         ]
 994         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 995             "arv": "http://arvados.org/cwl#",
 996         }
 997         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 998                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 999                         '--eval-timeout=20', '--thread-count=4',
1000                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1001                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1002
1003         stubs.api.container_requests().create.assert_called_with(
1004             body=JsonDiffMatcher(expect_container))
1005         self.assertEqual(stubs.capture_stdout.getvalue(),
1006                          stubs.expect_container_request_uuid + '\n')
1007         self.assertEqual(exited, 0)
1008
1009     def tearDown(self):
1010         arvados_cwl.arvdocker.arv_docker_clear_cache()
1011
1012     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1013     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1014     @mock.patch("arvados.api")
1015     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1016         arvados_cwl.arvdocker.arv_docker_clear_cache()
1017
1018         arvrunner = mock.MagicMock()
1019         arvrunner.project_uuid = ""
1020         api.return_value = mock.MagicMock()
1021         arvrunner.api = api.return_value
1022         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1023                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1024                                                                         "link_class": "docker_image_repo+tag",
1025                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1026                                                                         "owner_uuid": "",
1027                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1028                                                             {"items": [{"created_at": "",
1029                                                                         "head_uuid": "",
1030                                                                         "link_class": "docker_image_hash",
1031                                                                         "name": "123456",
1032                                                                         "owner_uuid": "",
1033                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1034         )
1035         find_one_image_hash.return_value = "123456"
1036
1037         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1038                                                                               "owner_uuid": "",
1039                                                                               "manifest_text": "",
1040                                                                               "properties": ""
1041                                                                           }], "items_available": 1, "offset": 0},)
1042         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1043         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1044                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1045         self.assertEqual("9999999999999999999999999999999b+99",
1046                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1047
1048
1049     @stubs
1050     def test_submit_secrets(self, stubs):
1051         exited = arvados_cwl.main(
1052             ["--submit", "--no-wait", "--api=containers", "--debug",
1053                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1054             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1055
1056         expect_container = {
1057             "command": [
1058                 "arvados-cwl-runner",
1059                 "--local",
1060                 "--api=containers",
1061                 "--no-log-timestamps",
1062                 "--disable-validate",
1063                 "--disable-color",
1064                 "--eval-timeout=20",
1065                 '--thread-count=4',
1066                 "--enable-reuse",
1067                 "--collection-cache-size=256",
1068                 '--debug',
1069                 "--on-error=continue",
1070                 "/var/lib/cwl/workflow.json#main",
1071                 "/var/lib/cwl/cwl.input.json"
1072             ],
1073             "container_image": "999999999999999999999999999999d3+99",
1074             "cwd": "/var/spool/cwl",
1075             "mounts": {
1076                 "/var/lib/cwl/cwl.input.json": {
1077                     "content": {
1078                         "pw": {
1079                             "$include": "/secrets/s0"
1080                         }
1081                     },
1082                     "kind": "json"
1083                 },
1084                 "/var/lib/cwl/workflow.json": {
1085                     "content": {
1086                         "$graph": [
1087                             {
1088                                 "$namespaces": {
1089                                     "cwltool": "http://commonwl.org/cwltool#"
1090                                 },
1091                                 "arguments": [
1092                                     "md5sum",
1093                                     "example.conf"
1094                                 ],
1095                                 "class": "CommandLineTool",
1096                                 "hints": [
1097                                     {
1098                                         "class": "http://commonwl.org/cwltool#Secrets",
1099                                         "secrets": [
1100                                             "#secret_job.cwl/pw"
1101                                         ]
1102                                     }
1103                                 ],
1104                                 "id": "#secret_job.cwl",
1105                                 "inputs": [
1106                                     {
1107                                         "id": "#secret_job.cwl/pw",
1108                                         "type": "string"
1109                                     }
1110                                 ],
1111                                 "outputs": [
1112                                     {
1113                                         "id": "#secret_job.cwl/out",
1114                                         "type": "File",
1115                                         "outputBinding": {
1116                                               "glob": "hashed_example.txt"
1117                                         }
1118                                     }
1119                                 ],
1120                                 "stdout": "hashed_example.txt",
1121                                 "requirements": [
1122                                     {
1123                                         "class": "InitialWorkDirRequirement",
1124                                         "listing": [
1125                                             {
1126                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1127                                                 "entryname": "example.conf"
1128                                             }
1129                                         ]
1130                                     }
1131                                 ]
1132                             },
1133                             {
1134                                 "class": "Workflow",
1135                                 "hints": [
1136                                     {
1137                                         "class": "DockerRequirement",
1138                                         "dockerPull": "debian:buster-slim",
1139                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1140                                     },
1141                                     {
1142                                         "class": "http://commonwl.org/cwltool#Secrets",
1143                                         "secrets": [
1144                                             "#main/pw"
1145                                         ]
1146                                     }
1147                                 ],
1148                                 "id": "#main",
1149                                 "inputs": [
1150                                     {
1151                                         "id": "#main/pw",
1152                                         "type": "string"
1153                                     }
1154                                 ],
1155                                 "outputs": [
1156                                     {
1157                                         "id": "#main/out",
1158                                         "outputSource": "#main/step1/out",
1159                                         "type": "File"
1160                                     }
1161                                 ],
1162                                 "steps": [
1163                                     {
1164                                         "id": "#main/step1",
1165                                         "in": [
1166                                             {
1167                                                 "id": "#main/step1/pw",
1168                                                 "source": "#main/pw"
1169                                             }
1170                                         ],
1171                                         "out": [
1172                                             "#main/step1/out"
1173                                         ],
1174                                         "run": "#secret_job.cwl"
1175                                     }
1176                                 ]
1177                             }
1178                         ],
1179                         "cwlVersion": "v1.0"
1180                     },
1181                     "kind": "json"
1182                 },
1183                 "/var/spool/cwl": {
1184                     "kind": "collection",
1185                     "writable": True
1186                 },
1187                 "stdout": {
1188                     "kind": "file",
1189                     "path": "/var/spool/cwl/cwl.output.json"
1190                 }
1191             },
1192             "name": "secret_wf.cwl",
1193             "output_path": "/var/spool/cwl",
1194             "priority": 500,
1195             "properties": {},
1196             "runtime_constraints": {
1197                 "API": True,
1198                 "ram": 1342177280,
1199                 "vcpus": 1
1200             },
1201             "secret_mounts": {
1202                 "/secrets/s0": {
1203                     "content": "blorp",
1204                     "kind": "text"
1205                 }
1206             },
1207             "state": "Committed",
1208             "use_existing": False
1209         }
1210
1211         stubs.api.container_requests().create.assert_called_with(
1212             body=JsonDiffMatcher(expect_container))
1213         self.assertEqual(stubs.capture_stdout.getvalue(),
1214                          stubs.expect_container_request_uuid + '\n')
1215         self.assertEqual(exited, 0)
1216
1217     @stubs
1218     def test_submit_request_uuid(self, stubs):
1219         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1220         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1221
1222         stubs.api.container_requests().update().execute.return_value = {
1223             "uuid": stubs.expect_container_request_uuid,
1224             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1225             "state": "Queued"
1226         }
1227
1228         exited = arvados_cwl.main(
1229             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1230                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1231             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1232
1233         stubs.api.container_requests().update.assert_called_with(
1234             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1235         self.assertEqual(stubs.capture_stdout.getvalue(),
1236                          stubs.expect_container_request_uuid + '\n')
1237         self.assertEqual(exited, 0)
1238
1239     @stubs
1240     def test_submit_container_cluster_id(self, stubs):
1241         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1242
1243         exited = arvados_cwl.main(
1244             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1245                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1246             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1247
1248         expect_container = copy.deepcopy(stubs.expect_container_spec)
1249
1250         stubs.api.container_requests().create.assert_called_with(
1251             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1252         self.assertEqual(stubs.capture_stdout.getvalue(),
1253                          stubs.expect_container_request_uuid + '\n')
1254         self.assertEqual(exited, 0)
1255
1256     @stubs
1257     def test_submit_validate_cluster_id(self, stubs):
1258         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1259         exited = arvados_cwl.main(
1260             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1261              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1262             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1263         self.assertEqual(exited, 1)
1264
1265     @mock.patch("arvados.collection.CollectionReader")
1266     @stubs
1267     def test_submit_uuid_inputs(self, stubs, collectionReader):
1268         collectionReader().exists.return_value = True
1269         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1270         def list_side_effect(**kwargs):
1271             m = mock.MagicMock()
1272             if "count" in kwargs:
1273                 m.execute.return_value = {"items": [
1274                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1275                 ]}
1276             else:
1277                 m.execute.return_value = {"items": []}
1278             return m
1279         stubs.api.collections().list.side_effect = list_side_effect
1280
1281         exited = arvados_cwl.main(
1282             ["--submit", "--no-wait", "--api=containers", "--debug",
1283                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1284             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1285
1286         expect_container = copy.deepcopy(stubs.expect_container_spec)
1287         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1288         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1289         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1290
1291         stubs.api.collections().list.assert_has_calls([
1292             mock.call(count='none',
1293                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1294                       select=['uuid', 'portable_data_hash'])])
1295         stubs.api.container_requests().create.assert_called_with(
1296             body=JsonDiffMatcher(expect_container))
1297         self.assertEqual(stubs.capture_stdout.getvalue(),
1298                          stubs.expect_container_request_uuid + '\n')
1299         self.assertEqual(exited, 0)
1300
1301     @stubs
1302     def test_submit_mismatched_uuid_inputs(self, stubs):
1303         def list_side_effect(**kwargs):
1304             m = mock.MagicMock()
1305             if "count" in kwargs:
1306                 m.execute.return_value = {"items": [
1307                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1308                 ]}
1309             else:
1310                 m.execute.return_value = {"items": []}
1311             return m
1312         stubs.api.collections().list.side_effect = list_side_effect
1313
1314         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1315             capture_stderr = StringIO()
1316             cwltool_logger = logging.getLogger('cwltool')
1317             stderr_logger = logging.StreamHandler(capture_stderr)
1318             cwltool_logger.addHandler(stderr_logger)
1319
1320             try:
1321                 exited = arvados_cwl.main(
1322                     ["--submit", "--no-wait", "--api=containers", "--debug",
1323                         "tests/wf/submit_wf.cwl", infile],
1324                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1325
1326                 self.assertEqual(exited, 1)
1327                 self.assertRegex(
1328                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1329                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1330             finally:
1331                 cwltool_logger.removeHandler(stderr_logger)
1332
1333     @mock.patch("arvados.collection.CollectionReader")
1334     @stubs
1335     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1336         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1337         capture_stderr = StringIO()
1338
1339         cwltool_logger = logging.getLogger('cwltool')
1340         stderr_logger = logging.StreamHandler(capture_stderr)
1341         cwltool_logger.addHandler(stderr_logger)
1342
1343         exited = arvados_cwl.main(
1344             ["--submit", "--no-wait", "--api=containers", "--debug",
1345                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1346             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1347
1348         try:
1349             self.assertEqual(exited, 1)
1350             self.assertRegex(
1351                 capture_stderr.getvalue(),
1352                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1353         finally:
1354             cwltool_logger.removeHandler(stderr_logger)
1355
1356
1357 class TestCreateWorkflow(unittest.TestCase):
1358     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1359     expect_workflow = StripYAMLComments(
1360         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1361
1362     @stubs
1363     def test_create(self, stubs):
1364         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1365
1366         exited = arvados_cwl.main(
1367             ["--create-workflow", "--debug",
1368              "--api=containers",
1369              "--project-uuid", project_uuid,
1370              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1371             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1372
1373         stubs.api.pipeline_templates().create.refute_called()
1374         stubs.api.container_requests().create.refute_called()
1375
1376         body = {
1377             "workflow": {
1378                 "owner_uuid": project_uuid,
1379                 "name": "submit_wf.cwl",
1380                 "description": "",
1381                 "definition": self.expect_workflow,
1382             }
1383         }
1384         stubs.api.workflows().create.assert_called_with(
1385             body=JsonDiffMatcher(body))
1386
1387         self.assertEqual(stubs.capture_stdout.getvalue(),
1388                          stubs.expect_workflow_uuid + '\n')
1389         self.assertEqual(exited, 0)
1390
1391     @stubs
1392     def test_create_name(self, stubs):
1393         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1394
1395         exited = arvados_cwl.main(
1396             ["--create-workflow", "--debug",
1397              "--api=containers",
1398              "--project-uuid", project_uuid,
1399              "--name", "testing 123",
1400              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1401             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1402
1403         stubs.api.pipeline_templates().create.refute_called()
1404         stubs.api.container_requests().create.refute_called()
1405
1406         body = {
1407             "workflow": {
1408                 "owner_uuid": project_uuid,
1409                 "name": "testing 123",
1410                 "description": "",
1411                 "definition": self.expect_workflow,
1412             }
1413         }
1414         stubs.api.workflows().create.assert_called_with(
1415             body=JsonDiffMatcher(body))
1416
1417         self.assertEqual(stubs.capture_stdout.getvalue(),
1418                          stubs.expect_workflow_uuid + '\n')
1419         self.assertEqual(exited, 0)
1420
1421
1422     @stubs
1423     def test_update(self, stubs):
1424         exited = arvados_cwl.main(
1425             ["--update-workflow", self.existing_workflow_uuid,
1426              "--debug",
1427              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1428             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1429
1430         body = {
1431             "workflow": {
1432                 "name": "submit_wf.cwl",
1433                 "description": "",
1434                 "definition": self.expect_workflow,
1435             }
1436         }
1437         stubs.api.workflows().update.assert_called_with(
1438             uuid=self.existing_workflow_uuid,
1439             body=JsonDiffMatcher(body))
1440         self.assertEqual(stubs.capture_stdout.getvalue(),
1441                          self.existing_workflow_uuid + '\n')
1442         self.assertEqual(exited, 0)
1443
1444     @stubs
1445     def test_update_name(self, stubs):
1446         exited = arvados_cwl.main(
1447             ["--update-workflow", self.existing_workflow_uuid,
1448              "--debug", "--name", "testing 123",
1449              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1450             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1451
1452         body = {
1453             "workflow": {
1454                 "name": "testing 123",
1455                 "description": "",
1456                 "definition": self.expect_workflow,
1457             }
1458         }
1459         stubs.api.workflows().update.assert_called_with(
1460             uuid=self.existing_workflow_uuid,
1461             body=JsonDiffMatcher(body))
1462         self.assertEqual(stubs.capture_stdout.getvalue(),
1463                          self.existing_workflow_uuid + '\n')
1464         self.assertEqual(exited, 0)
1465
1466     @stubs
1467     def test_create_collection_per_tool(self, stubs):
1468         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1469
1470         exited = arvados_cwl.main(
1471             ["--create-workflow", "--debug",
1472              "--api=containers",
1473              "--project-uuid", project_uuid,
1474              "tests/collection_per_tool/collection_per_tool.cwl"],
1475             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1476
1477         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1478         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1479
1480         body = {
1481             "workflow": {
1482                 "owner_uuid": project_uuid,
1483                 "name": "collection_per_tool.cwl",
1484                 "description": "",
1485                 "definition": expect_workflow,
1486             }
1487         }
1488         stubs.api.workflows().create.assert_called_with(
1489             body=JsonDiffMatcher(body))
1490
1491         self.assertEqual(stubs.capture_stdout.getvalue(),
1492                          stubs.expect_workflow_uuid + '\n')
1493         self.assertEqual(exited, 0)
1494
1495     @stubs
1496     def test_create_with_imports(self, stubs):
1497         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1498
1499         exited = arvados_cwl.main(
1500             ["--create-workflow", "--debug",
1501              "--api=containers",
1502              "--project-uuid", project_uuid,
1503              "tests/wf/feddemo/feddemo.cwl"],
1504             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1505
1506         stubs.api.pipeline_templates().create.refute_called()
1507         stubs.api.container_requests().create.refute_called()
1508
1509         self.assertEqual(stubs.capture_stdout.getvalue(),
1510                          stubs.expect_workflow_uuid + '\n')
1511         self.assertEqual(exited, 0)
1512
1513     @stubs
1514     def test_create_with_no_input(self, stubs):
1515         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1516
1517         exited = arvados_cwl.main(
1518             ["--create-workflow", "--debug",
1519              "--api=containers",
1520              "--project-uuid", project_uuid,
1521              "tests/wf/revsort/revsort.cwl"],
1522             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1523
1524         stubs.api.pipeline_templates().create.refute_called()
1525         stubs.api.container_requests().create.refute_called()
1526
1527         self.assertEqual(stubs.capture_stdout.getvalue(),
1528                          stubs.expect_workflow_uuid + '\n')
1529         self.assertEqual(exited, 0)