sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=4',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343         arvados_cwl.arvdocker.arv_docker_clear_cache()
 344
 345     @stubs
 346     def test_error_when_multiple_storage_classes_specified(self, stubs):
 347         storage_classes = "foo,bar"
 348         exited = arvados_cwl.main(
 349                 ["--debug", "--storage-classes", storage_classes,
 350                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351                 sys.stdin, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 1)
 353
 354     @mock.patch("time.sleep")
 355     @stubs
 356     def test_submit_invalid_runner_ram(self, stubs, tm):
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 361         self.assertEqual(exited, 1)
 362
 363
 364     @stubs
 365     def test_submit_container(self, stubs):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--api=containers", "--debug",
 368                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 370
 371         stubs.api.collections().create.assert_has_calls([
 372             mock.call(body=JsonDiffMatcher({
 373                 'manifest_text':
 374                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 375                 'replication_desired': None,
 376                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 377             }), ensure_unique_name=False),
 378             mock.call(body=JsonDiffMatcher({
 379                 'manifest_text':
 380                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 381                 'replication_desired': None,
 382                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 383             }), ensure_unique_name=False),
 384             ])
 385
 386         expect_container = copy.deepcopy(stubs.expect_container_spec)
 387         stubs.api.container_requests().create.assert_called_with(
 388             body=JsonDiffMatcher(expect_container))
 389         self.assertEqual(stubs.capture_stdout.getvalue(),
 390                          stubs.expect_container_request_uuid + '\n')
 391         self.assertEqual(exited, 0)
 392
 393
 394     @stubs
 395     def test_submit_container_tool(self, stubs):
 396         # test for issue #16139
 397         exited = arvados_cwl.main(
 398             ["--submit", "--no-wait", "--api=containers", "--debug",
 399                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 400             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 401
 402         self.assertEqual(stubs.capture_stdout.getvalue(),
 403                          stubs.expect_container_request_uuid + '\n')
 404         self.assertEqual(exited, 0)
 405
 406     @stubs
 407     def test_submit_container_no_reuse(self, stubs):
 408         exited = arvados_cwl.main(
 409             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 410                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         expect_container["command"] = [
 415             'arvados-cwl-runner', '--local', '--api=containers',
 416             '--no-log-timestamps', '--disable-validate', '--disable-color',
 417             '--eval-timeout=20', '--thread-count=4',
 418             '--disable-reuse', "--collection-cache-size=256",
 419             '--debug', '--on-error=continue',
 420             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 421         expect_container["use_existing"] = False
 422
 423         stubs.api.container_requests().create.assert_called_with(
 424             body=JsonDiffMatcher(expect_container))
 425         self.assertEqual(stubs.capture_stdout.getvalue(),
 426                          stubs.expect_container_request_uuid + '\n')
 427         self.assertEqual(exited, 0)
 428
 429     @stubs
 430     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug",
 433              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435         self.assertEqual(exited, 0)
 436
 437         expect_container = copy.deepcopy(stubs.expect_container_spec)
 438         expect_container["command"] = [
 439             'arvados-cwl-runner', '--local', '--api=containers',
 440             '--no-log-timestamps', '--disable-validate', '--disable-color',
 441             '--eval-timeout=20', '--thread-count=4',
 442             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 443             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 444         expect_container["use_existing"] = False
 445         expect_container["name"] = "submit_wf_no_reuse.cwl"
 446         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 447             {
 448                 "class": "http://arvados.org/cwl#ReuseRequirement",
 449                 "enableReuse": False,
 450             },
 451         ]
 452         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 453             "arv": "http://arvados.org/cwl#",
 454             "cwltool": "http://commonwl.org/cwltool#"
 455         }
 456
 457         stubs.api.container_requests().create.assert_called_with(
 458             body=JsonDiffMatcher(expect_container))
 459         self.assertEqual(stubs.capture_stdout.getvalue(),
 460                          stubs.expect_container_request_uuid + '\n')
 461
 462
 463     @stubs
 464     def test_submit_container_on_error(self, stubs):
 465         exited = arvados_cwl.main(
 466             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 467                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 468             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 469
 470         expect_container = copy.deepcopy(stubs.expect_container_spec)
 471         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 472                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 473                                        '--eval-timeout=20', '--thread-count=4',
 474                                        '--enable-reuse', "--collection-cache-size=256",
 475                                        '--debug', '--on-error=stop',
 476                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 477
 478         stubs.api.container_requests().create.assert_called_with(
 479             body=JsonDiffMatcher(expect_container))
 480         self.assertEqual(stubs.capture_stdout.getvalue(),
 481                          stubs.expect_container_request_uuid + '\n')
 482         self.assertEqual(exited, 0)
 483
 484     @stubs
 485     def test_submit_container_output_name(self, stubs):
 486         output_name = "test_output_name"
 487
 488         exited = arvados_cwl.main(
 489             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 490                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 491             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 492
 493         expect_container = copy.deepcopy(stubs.expect_container_spec)
 494         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 495                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 496                                        '--eval-timeout=20', '--thread-count=4',
 497                                        '--enable-reuse', "--collection-cache-size=256",
 498                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 499                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 500         expect_container["output_name"] = output_name
 501
 502         stubs.api.container_requests().create.assert_called_with(
 503             body=JsonDiffMatcher(expect_container))
 504         self.assertEqual(stubs.capture_stdout.getvalue(),
 505                          stubs.expect_container_request_uuid + '\n')
 506         self.assertEqual(exited, 0)
 507
 508     @stubs
 509     def test_submit_storage_classes(self, stubs):
 510         exited = arvados_cwl.main(
 511             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 512                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514
 515         expect_container = copy.deepcopy(stubs.expect_container_spec)
 516         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 517                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 518                                        '--eval-timeout=20', '--thread-count=4',
 519                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 520                                        "--storage-classes=foo", '--on-error=continue',
 521                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 522
 523         stubs.api.container_requests().create.assert_called_with(
 524             body=JsonDiffMatcher(expect_container))
 525         self.assertEqual(stubs.capture_stdout.getvalue(),
 526                          stubs.expect_container_request_uuid + '\n')
 527         self.assertEqual(exited, 0)
 528
 529     @mock.patch("cwltool.task_queue.TaskQueue")
 530     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 531     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 532     @stubs
 533     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 534         final_output_c = arvados.collection.Collection()
 535         make_output.return_value = ({},final_output_c)
 536
 537         def set_final_output(job_order, output_callback, runtimeContext):
 538             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 539             return []
 540         job.side_effect = set_final_output
 541
 542         exited = arvados_cwl.main(
 543             ["--debug", "--local", "--storage-classes=foo",
 544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 545             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 546
 547         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 548         self.assertEqual(exited, 0)
 549
 550     @mock.patch("cwltool.task_queue.TaskQueue")
 551     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 552     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 553     @stubs
 554     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 555         final_output_c = arvados.collection.Collection()
 556         make_output.return_value = ({},final_output_c)
 557
 558         def set_final_output(job_order, output_callback, runtimeContext):
 559             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 560             return []
 561         job.side_effect = set_final_output
 562
 563         exited = arvados_cwl.main(
 564             ["--debug", "--local",
 565                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 566             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 567
 568         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 569         self.assertEqual(exited, 0)
 570
 571     @stubs
 572     def test_submit_container_output_ttl(self, stubs):
 573         exited = arvados_cwl.main(
 574             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 575                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 576             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 577
 578         expect_container = copy.deepcopy(stubs.expect_container_spec)
 579         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 580                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 581                                        '--eval-timeout=20', '--thread-count=4',
 582                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 583                                        '--on-error=continue',
 584                                        "--intermediate-output-ttl=3600",
 585                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586
 587         stubs.api.container_requests().create.assert_called_with(
 588             body=JsonDiffMatcher(expect_container))
 589         self.assertEqual(stubs.capture_stdout.getvalue(),
 590                          stubs.expect_container_request_uuid + '\n')
 591         self.assertEqual(exited, 0)
 592
 593     @stubs
 594     def test_submit_container_trash_intermediate(self, stubs):
 595         exited = arvados_cwl.main(
 596             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 597                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599
 600
 601         expect_container = copy.deepcopy(stubs.expect_container_spec)
 602         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 603                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 604                                        '--eval-timeout=20', '--thread-count=4',
 605                                        '--enable-reuse', "--collection-cache-size=256",
 606                                        '--debug', '--on-error=continue',
 607                                        "--trash-intermediate",
 608                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 609
 610         stubs.api.container_requests().create.assert_called_with(
 611             body=JsonDiffMatcher(expect_container))
 612         self.assertEqual(stubs.capture_stdout.getvalue(),
 613                          stubs.expect_container_request_uuid + '\n')
 614         self.assertEqual(exited, 0)
 615
 616     @stubs
 617     def test_submit_container_output_tags(self, stubs):
 618         output_tags = "tag0,tag1,tag2"
 619
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 627                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 628                                        '--eval-timeout=20', '--thread-count=4',
 629                                        '--enable-reuse', "--collection-cache-size=256",
 630                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 631                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 632
 633         stubs.api.container_requests().create.assert_called_with(
 634             body=JsonDiffMatcher(expect_container))
 635         self.assertEqual(stubs.capture_stdout.getvalue(),
 636                          stubs.expect_container_request_uuid + '\n')
 637         self.assertEqual(exited, 0)
 638
 639     @stubs
 640     def test_submit_container_runner_ram(self, stubs):
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 643                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 645
 646         expect_container = copy.deepcopy(stubs.expect_container_spec)
 647         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(stubs.capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653         self.assertEqual(exited, 0)
 654
 655     @mock.patch("arvados.collection.CollectionReader")
 656     @mock.patch("time.sleep")
 657     @stubs
 658     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 659         collectionReader().exists.return_value = True
 660         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug",
 663              "tests/wf/submit_keepref_wf.cwl"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 665         self.assertEqual(exited, 0)
 666
 667     @mock.patch("arvados.collection.CollectionReader")
 668     @mock.patch("time.sleep")
 669     @stubs
 670     def test_submit_keepref(self, stubs, tm, reader):
 671         with open("tests/wf/expect_arvworkflow.cwl") as f:
 672             reader().open().__enter__().read.return_value = f.read()
 673
 674         exited = arvados_cwl.main(
 675             ["--submit", "--no-wait", "--api=containers", "--debug",
 676              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 677             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 678
 679         expect_container = {
 680             'priority': 500,
 681             'mounts': {
 682                 '/var/spool/cwl': {
 683                     'writable': True,
 684                     'kind': 'collection'
 685                 },
 686                 'stdout': {
 687                     'path': '/var/spool/cwl/cwl.output.json',
 688                     'kind': 'file'
 689                 },
 690                 '/var/lib/cwl/workflow': {
 691                     'portable_data_hash': '99999999999999999999999999999994+99',
 692                     'kind': 'collection'
 693                 },
 694                 '/var/lib/cwl/cwl.input.json': {
 695                     'content': {
 696                         'x': 'XxX'
 697                     },
 698                     'kind': 'json'
 699                 }
 700             }, 'state': 'Committed',
 701             'output_path': '/var/spool/cwl',
 702             'name': 'expect_arvworkflow.cwl#main',
 703             'container_image': '999999999999999999999999999999d3+99',
 704             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 705                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 706                         '--eval-timeout=20', '--thread-count=4',
 707                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 708                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 709             'cwd': '/var/spool/cwl',
 710             'runtime_constraints': {
 711                 'API': True,
 712                 'vcpus': 1,
 713                 'ram': 1342177280
 714             },
 715             'use_existing': False,
 716             'properties': {},
 717             'secret_mounts': {}
 718         }
 719
 720         stubs.api.container_requests().create.assert_called_with(
 721             body=JsonDiffMatcher(expect_container))
 722         self.assertEqual(stubs.capture_stdout.getvalue(),
 723                          stubs.expect_container_request_uuid + '\n')
 724         self.assertEqual(exited, 0)
 725
 726     @mock.patch("time.sleep")
 727     @stubs
 728     def test_submit_arvworkflow(self, stubs, tm):
 729         with open("tests/wf/expect_arvworkflow.cwl") as f:
 730             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 731
 732         exited = arvados_cwl.main(
 733             ["--submit", "--no-wait", "--api=containers", "--debug",
 734              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 735             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 736
 737         expect_container = {
 738             'priority': 500,
 739             'mounts': {
 740                 '/var/spool/cwl': {
 741                     'writable': True,
 742                     'kind': 'collection'
 743                 },
 744                 'stdout': {
 745                     'path': '/var/spool/cwl/cwl.output.json',
 746                     'kind': 'file'
 747                 },
 748                 '/var/lib/cwl/workflow.json': {
 749                     'kind': 'json',
 750                     'content': {
 751                         'cwlVersion': 'v1.0',
 752                         '$graph': [
 753                             {
 754                                 'id': '#main',
 755                                 'inputs': [
 756                                     {'type': 'string', 'id': '#main/x'}
 757                                 ],
 758                                 'steps': [
 759                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 760                                      'run': '#submit_tool.cwl',
 761                                      'id': '#main/step1',
 762                                      'out': []}
 763                                 ],
 764                                 'class': 'Workflow',
 765                                 'outputs': []
 766                             },
 767                             {
 768                                 'inputs': [
 769                                     {
 770                                         'inputBinding': {'position': 1},
 771                                         'type': 'string',
 772                                         'id': '#submit_tool.cwl/x'}
 773                                 ],
 774                                 'requirements': [
 775                                     {
 776                                         'dockerPull': 'debian:buster-slim',
 777                                         'class': 'DockerRequirement',
 778                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 779                                     }
 780                                 ],
 781                                 'id': '#submit_tool.cwl',
 782                                 'outputs': [],
 783                                 'baseCommand': 'cat',
 784                                 'class': 'CommandLineTool'
 785                             }
 786                         ]
 787                     }
 788                 },
 789                 '/var/lib/cwl/cwl.input.json': {
 790                     'content': {
 791                         'x': 'XxX'
 792                     },
 793                     'kind': 'json'
 794                 }
 795             }, 'state': 'Committed',
 796             'output_path': '/var/spool/cwl',
 797             'name': 'a test workflow',
 798             'container_image': "999999999999999999999999999999d3+99",
 799             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 800                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 801                         '--eval-timeout=20', '--thread-count=4',
 802                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 803                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 804             'cwd': '/var/spool/cwl',
 805             'runtime_constraints': {
 806                 'API': True,
 807                 'vcpus': 1,
 808                 'ram': 1342177280
 809             },
 810             'use_existing': False,
 811             'properties': {
 812                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 813             },
 814             'secret_mounts': {}
 815         }
 816
 817         stubs.api.container_requests().create.assert_called_with(
 818             body=JsonDiffMatcher(expect_container))
 819         self.assertEqual(stubs.capture_stdout.getvalue(),
 820                          stubs.expect_container_request_uuid + '\n')
 821         self.assertEqual(exited, 0)
 822
 823     @stubs
 824     def test_submit_container_name(self, stubs):
 825         exited = arvados_cwl.main(
 826             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 827                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 828             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 829
 830         expect_container = copy.deepcopy(stubs.expect_container_spec)
 831         expect_container["name"] = "hello container 123"
 832
 833         stubs.api.container_requests().create.assert_called_with(
 834             body=JsonDiffMatcher(expect_container))
 835         self.assertEqual(stubs.capture_stdout.getvalue(),
 836                          stubs.expect_container_request_uuid + '\n')
 837         self.assertEqual(exited, 0)
 838
 839     @stubs
 840     def test_submit_missing_input(self, stubs):
 841         exited = arvados_cwl.main(
 842             ["--submit", "--no-wait", "--api=containers", "--debug",
 843              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 844             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 845         self.assertEqual(exited, 0)
 846
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug",
 849              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851         self.assertEqual(exited, 1)
 852
 853     @stubs
 854     def test_submit_container_project(self, stubs):
 855         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 856         exited = arvados_cwl.main(
 857             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 858                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 859             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 860
 861         expect_container = copy.deepcopy(stubs.expect_container_spec)
 862         expect_container["owner_uuid"] = project_uuid
 863         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 864                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 865                                        "--eval-timeout=20", "--thread-count=4",
 866                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 867                                        '--on-error=continue',
 868                                        '--project-uuid='+project_uuid,
 869                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 870
 871         stubs.api.container_requests().create.assert_called_with(
 872             body=JsonDiffMatcher(expect_container))
 873         self.assertEqual(stubs.capture_stdout.getvalue(),
 874                          stubs.expect_container_request_uuid + '\n')
 875         self.assertEqual(exited, 0)
 876
 877     @stubs
 878     def test_submit_container_eval_timeout(self, stubs):
 879         exited = arvados_cwl.main(
 880             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 881                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 882             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 883
 884         expect_container = copy.deepcopy(stubs.expect_container_spec)
 885         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 886                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 887                                        '--eval-timeout=60.0', '--thread-count=4',
 888                                        '--enable-reuse', "--collection-cache-size=256",
 889                                        '--debug', '--on-error=continue',
 890                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 891
 892         stubs.api.container_requests().create.assert_called_with(
 893             body=JsonDiffMatcher(expect_container))
 894         self.assertEqual(stubs.capture_stdout.getvalue(),
 895                          stubs.expect_container_request_uuid + '\n')
 896         self.assertEqual(exited, 0)
 897
 898     @stubs
 899     def test_submit_container_collection_cache(self, stubs):
 900         exited = arvados_cwl.main(
 901             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 902                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904
 905         expect_container = copy.deepcopy(stubs.expect_container_spec)
 906         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 907                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 908                                        '--eval-timeout=20', '--thread-count=4',
 909                                        '--enable-reuse', "--collection-cache-size=500",
 910                                        '--debug', '--on-error=continue',
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 913
 914         stubs.api.container_requests().create.assert_called_with(
 915             body=JsonDiffMatcher(expect_container))
 916         self.assertEqual(stubs.capture_stdout.getvalue(),
 917                          stubs.expect_container_request_uuid + '\n')
 918         self.assertEqual(exited, 0)
 919
 920     @stubs
 921     def test_submit_container_thread_count(self, stubs):
 922         exited = arvados_cwl.main(
 923             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 924                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 925             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 926
 927         expect_container = copy.deepcopy(stubs.expect_container_spec)
 928         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 929                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 930                                        '--eval-timeout=20', '--thread-count=20',
 931                                        '--enable-reuse', "--collection-cache-size=256",
 932                                        '--debug', '--on-error=continue',
 933                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 934
 935         stubs.api.container_requests().create.assert_called_with(
 936             body=JsonDiffMatcher(expect_container))
 937         self.assertEqual(stubs.capture_stdout.getvalue(),
 938                          stubs.expect_container_request_uuid + '\n')
 939         self.assertEqual(exited, 0)
 940
 941     @stubs
 942     def test_submit_container_runner_image(self, stubs):
 943         exited = arvados_cwl.main(
 944             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 945                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 946             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 947
 948         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 949
 950         expect_container = copy.deepcopy(stubs.expect_container_spec)
 951         stubs.api.container_requests().create.assert_called_with(
 952             body=JsonDiffMatcher(expect_container))
 953         self.assertEqual(stubs.capture_stdout.getvalue(),
 954                          stubs.expect_container_request_uuid + '\n')
 955         self.assertEqual(exited, 0)
 956
 957     @stubs
 958     def test_submit_priority(self, stubs):
 959         exited = arvados_cwl.main(
 960             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 961                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 962             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 963
 964         stubs.expect_container_spec["priority"] = 669
 965
 966         expect_container = copy.deepcopy(stubs.expect_container_spec)
 967         stubs.api.container_requests().create.assert_called_with(
 968             body=JsonDiffMatcher(expect_container))
 969         self.assertEqual(stubs.capture_stdout.getvalue(),
 970                          stubs.expect_container_request_uuid + '\n')
 971         self.assertEqual(exited, 0)
 972
 973     @stubs
 974     def test_submit_wf_runner_resources(self, stubs):
 975         exited = arvados_cwl.main(
 976             ["--submit", "--no-wait", "--api=containers", "--debug",
 977                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 978             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 979
 980         expect_container = copy.deepcopy(stubs.expect_container_spec)
 981         expect_container["runtime_constraints"] = {
 982             "API": True,
 983             "vcpus": 2,
 984             "ram": (2000+512) * 2**20
 985         }
 986         expect_container["name"] = "submit_wf_runner_resources.cwl"
 987         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 988             {
 989                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 990                 "coresMin": 2,
 991                 "ramMin": 2000,
 992                 "keep_cache": 512
 993             }
 994         ]
 995         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 996             "arv": "http://arvados.org/cwl#",
 997         }
 998         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 999                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1000                         '--eval-timeout=20', '--thread-count=4',
1001                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1002                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1003
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     def tearDown(self):
1011         arvados_cwl.arvdocker.arv_docker_clear_cache()
1012
1013     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1014     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1015     @mock.patch("arvados.api")
1016     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1017         arvados_cwl.arvdocker.arv_docker_clear_cache()
1018
1019         arvrunner = mock.MagicMock()
1020         arvrunner.project_uuid = ""
1021         api.return_value = mock.MagicMock()
1022         arvrunner.api = api.return_value
1023         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1024                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1025                                                                         "link_class": "docker_image_repo+tag",
1026                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1027                                                                         "owner_uuid": "",
1028                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1029                                                             {"items": [{"created_at": "",
1030                                                                         "head_uuid": "",
1031                                                                         "link_class": "docker_image_hash",
1032                                                                         "name": "123456",
1033                                                                         "owner_uuid": "",
1034                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1035         )
1036         find_one_image_hash.return_value = "123456"
1037
1038         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1039                                                                               "owner_uuid": "",
1040                                                                               "manifest_text": "",
1041                                                                               "properties": ""
1042                                                                           }], "items_available": 1, "offset": 0},)
1043         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1044         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1045                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1046         self.assertEqual("9999999999999999999999999999999b+99",
1047                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1048
1049
1050     @stubs
1051     def test_submit_secrets(self, stubs):
1052         exited = arvados_cwl.main(
1053             ["--submit", "--no-wait", "--api=containers", "--debug",
1054                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1055             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1056
1057         expect_container = {
1058             "command": [
1059                 "arvados-cwl-runner",
1060                 "--local",
1061                 "--api=containers",
1062                 "--no-log-timestamps",
1063                 "--disable-validate",
1064                 "--disable-color",
1065                 "--eval-timeout=20",
1066                 '--thread-count=4',
1067                 "--enable-reuse",
1068                 "--collection-cache-size=256",
1069                 '--debug',
1070                 "--on-error=continue",
1071                 "/var/lib/cwl/workflow.json#main",
1072                 "/var/lib/cwl/cwl.input.json"
1073             ],
1074             "container_image": "999999999999999999999999999999d3+99",
1075             "cwd": "/var/spool/cwl",
1076             "mounts": {
1077                 "/var/lib/cwl/cwl.input.json": {
1078                     "content": {
1079                         "pw": {
1080                             "$include": "/secrets/s0"
1081                         }
1082                     },
1083                     "kind": "json"
1084                 },
1085                 "/var/lib/cwl/workflow.json": {
1086                     "content": {
1087                         "$graph": [
1088                             {
1089                                 "$namespaces": {
1090                                     "cwltool": "http://commonwl.org/cwltool#"
1091                                 },
1092                                 "arguments": [
1093                                     "md5sum",
1094                                     "example.conf"
1095                                 ],
1096                                 "class": "CommandLineTool",
1097                                 "hints": [
1098                                     {
1099                                         "class": "http://commonwl.org/cwltool#Secrets",
1100                                         "secrets": [
1101                                             "#secret_job.cwl/pw"
1102                                         ]
1103                                     }
1104                                 ],
1105                                 "id": "#secret_job.cwl",
1106                                 "inputs": [
1107                                     {
1108                                         "id": "#secret_job.cwl/pw",
1109                                         "type": "string"
1110                                     }
1111                                 ],
1112                                 "outputs": [
1113                                     {
1114                                         "id": "#secret_job.cwl/out",
1115                                         "type": "File",
1116                                         "outputBinding": {
1117                                               "glob": "hashed_example.txt"
1118                                         }
1119                                     }
1120                                 ],
1121                                 "stdout": "hashed_example.txt",
1122                                 "requirements": [
1123                                     {
1124                                         "class": "InitialWorkDirRequirement",
1125                                         "listing": [
1126                                             {
1127                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1128                                                 "entryname": "example.conf"
1129                                             }
1130                                         ]
1131                                     }
1132                                 ]
1133                             },
1134                             {
1135                                 "class": "Workflow",
1136                                 "hints": [
1137                                     {
1138                                         "class": "DockerRequirement",
1139                                         "dockerPull": "debian:buster-slim",
1140                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1141                                     },
1142                                     {
1143                                         "class": "http://commonwl.org/cwltool#Secrets",
1144                                         "secrets": [
1145                                             "#main/pw"
1146                                         ]
1147                                     }
1148                                 ],
1149                                 "id": "#main",
1150                                 "inputs": [
1151                                     {
1152                                         "id": "#main/pw",
1153                                         "type": "string"
1154                                     }
1155                                 ],
1156                                 "outputs": [
1157                                     {
1158                                         "id": "#main/out",
1159                                         "outputSource": "#main/step1/out",
1160                                         "type": "File"
1161                                     }
1162                                 ],
1163                                 "steps": [
1164                                     {
1165                                         "id": "#main/step1",
1166                                         "in": [
1167                                             {
1168                                                 "id": "#main/step1/pw",
1169                                                 "source": "#main/pw"
1170                                             }
1171                                         ],
1172                                         "out": [
1173                                             "#main/step1/out"
1174                                         ],
1175                                         "run": "#secret_job.cwl"
1176                                     }
1177                                 ]
1178                             }
1179                         ],
1180                         "cwlVersion": "v1.0"
1181                     },
1182                     "kind": "json"
1183                 },
1184                 "/var/spool/cwl": {
1185                     "kind": "collection",
1186                     "writable": True
1187                 },
1188                 "stdout": {
1189                     "kind": "file",
1190                     "path": "/var/spool/cwl/cwl.output.json"
1191                 }
1192             },
1193             "name": "secret_wf.cwl",
1194             "output_path": "/var/spool/cwl",
1195             "priority": 500,
1196             "properties": {},
1197             "runtime_constraints": {
1198                 "API": True,
1199                 "ram": 1342177280,
1200                 "vcpus": 1
1201             },
1202             "secret_mounts": {
1203                 "/secrets/s0": {
1204                     "content": "blorp",
1205                     "kind": "text"
1206                 }
1207             },
1208             "state": "Committed",
1209             "use_existing": False
1210         }
1211
1212         stubs.api.container_requests().create.assert_called_with(
1213             body=JsonDiffMatcher(expect_container))
1214         self.assertEqual(stubs.capture_stdout.getvalue(),
1215                          stubs.expect_container_request_uuid + '\n')
1216         self.assertEqual(exited, 0)
1217
1218     @stubs
1219     def test_submit_request_uuid(self, stubs):
1220         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1221         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1222
1223         stubs.api.container_requests().update().execute.return_value = {
1224             "uuid": stubs.expect_container_request_uuid,
1225             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1226             "state": "Queued"
1227         }
1228
1229         exited = arvados_cwl.main(
1230             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1231                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1232             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1233
1234         stubs.api.container_requests().update.assert_called_with(
1235             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1236         self.assertEqual(stubs.capture_stdout.getvalue(),
1237                          stubs.expect_container_request_uuid + '\n')
1238         self.assertEqual(exited, 0)
1239
1240     @stubs
1241     def test_submit_container_cluster_id(self, stubs):
1242         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1243
1244         exited = arvados_cwl.main(
1245             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1246                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1247             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1248
1249         expect_container = copy.deepcopy(stubs.expect_container_spec)
1250
1251         stubs.api.container_requests().create.assert_called_with(
1252             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1253         self.assertEqual(stubs.capture_stdout.getvalue(),
1254                          stubs.expect_container_request_uuid + '\n')
1255         self.assertEqual(exited, 0)
1256
1257     @stubs
1258     def test_submit_validate_cluster_id(self, stubs):
1259         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1260         exited = arvados_cwl.main(
1261             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1262              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1263             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1264         self.assertEqual(exited, 1)
1265
1266     @mock.patch("arvados.collection.CollectionReader")
1267     @stubs
1268     def test_submit_uuid_inputs(self, stubs, collectionReader):
1269         collectionReader().exists.return_value = True
1270         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1271         def list_side_effect(**kwargs):
1272             m = mock.MagicMock()
1273             if "count" in kwargs:
1274                 m.execute.return_value = {"items": [
1275                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1276                 ]}
1277             else:
1278                 m.execute.return_value = {"items": []}
1279             return m
1280         stubs.api.collections().list.side_effect = list_side_effect
1281
1282         exited = arvados_cwl.main(
1283             ["--submit", "--no-wait", "--api=containers", "--debug",
1284                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1285             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1286
1287         expect_container = copy.deepcopy(stubs.expect_container_spec)
1288         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1289         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1290         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1291
1292         stubs.api.collections().list.assert_has_calls([
1293             mock.call(count='none',
1294                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1295                       select=['uuid', 'portable_data_hash'])])
1296         stubs.api.container_requests().create.assert_called_with(
1297             body=JsonDiffMatcher(expect_container))
1298         self.assertEqual(stubs.capture_stdout.getvalue(),
1299                          stubs.expect_container_request_uuid + '\n')
1300         self.assertEqual(exited, 0)
1301
1302     @stubs
1303     def test_submit_mismatched_uuid_inputs(self, stubs):
1304         def list_side_effect(**kwargs):
1305             m = mock.MagicMock()
1306             if "count" in kwargs:
1307                 m.execute.return_value = {"items": [
1308                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1309                 ]}
1310             else:
1311                 m.execute.return_value = {"items": []}
1312             return m
1313         stubs.api.collections().list.side_effect = list_side_effect
1314
1315         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1316             capture_stderr = StringIO()
1317             cwltool_logger = logging.getLogger('cwltool')
1318             stderr_logger = logging.StreamHandler(capture_stderr)
1319             cwltool_logger.addHandler(stderr_logger)
1320
1321             try:
1322                 exited = arvados_cwl.main(
1323                     ["--submit", "--no-wait", "--api=containers", "--debug",
1324                         "tests/wf/submit_wf.cwl", infile],
1325                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1326
1327                 self.assertEqual(exited, 1)
1328                 self.assertRegex(
1329                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1330                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1331             finally:
1332                 cwltool_logger.removeHandler(stderr_logger)
1333
1334     @mock.patch("arvados.collection.CollectionReader")
1335     @stubs
1336     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1337         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1338         capture_stderr = StringIO()
1339
1340         cwltool_logger = logging.getLogger('cwltool')
1341         stderr_logger = logging.StreamHandler(capture_stderr)
1342         cwltool_logger.addHandler(stderr_logger)
1343
1344         exited = arvados_cwl.main(
1345             ["--submit", "--no-wait", "--api=containers", "--debug",
1346                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1347             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1348
1349         try:
1350             self.assertEqual(exited, 1)
1351             self.assertRegex(
1352                 capture_stderr.getvalue(),
1353                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1354         finally:
1355             cwltool_logger.removeHandler(stderr_logger)
1356
1357
1358 class TestCreateWorkflow(unittest.TestCase):
1359     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1360     expect_workflow = StripYAMLComments(
1361         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1362
1363     def setUp(self):
1364         cwltool.process._names = set()
1365         arvados_cwl.arvdocker.arv_docker_clear_cache()
1366
1367     @stubs
1368     def test_create(self, stubs):
1369         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1370
1371         exited = arvados_cwl.main(
1372             ["--create-workflow", "--debug",
1373              "--api=containers",
1374              "--project-uuid", project_uuid,
1375              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1376             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1377
1378         stubs.api.pipeline_templates().create.refute_called()
1379         stubs.api.container_requests().create.refute_called()
1380
1381         body = {
1382             "workflow": {
1383                 "owner_uuid": project_uuid,
1384                 "name": "submit_wf.cwl",
1385                 "description": "",
1386                 "definition": self.expect_workflow,
1387             }
1388         }
1389         stubs.api.workflows().create.assert_called_with(
1390             body=JsonDiffMatcher(body))
1391
1392         self.assertEqual(stubs.capture_stdout.getvalue(),
1393                          stubs.expect_workflow_uuid + '\n')
1394         self.assertEqual(exited, 0)
1395
1396     @stubs
1397     def test_create_name(self, stubs):
1398         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1399
1400         exited = arvados_cwl.main(
1401             ["--create-workflow", "--debug",
1402              "--api=containers",
1403              "--project-uuid", project_uuid,
1404              "--name", "testing 123",
1405              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1406             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1407
1408         stubs.api.pipeline_templates().create.refute_called()
1409         stubs.api.container_requests().create.refute_called()
1410
1411         body = {
1412             "workflow": {
1413                 "owner_uuid": project_uuid,
1414                 "name": "testing 123",
1415                 "description": "",
1416                 "definition": self.expect_workflow,
1417             }
1418         }
1419         stubs.api.workflows().create.assert_called_with(
1420             body=JsonDiffMatcher(body))
1421
1422         self.assertEqual(stubs.capture_stdout.getvalue(),
1423                          stubs.expect_workflow_uuid + '\n')
1424         self.assertEqual(exited, 0)
1425
1426
1427     @stubs
1428     def test_update(self, stubs):
1429         exited = arvados_cwl.main(
1430             ["--update-workflow", self.existing_workflow_uuid,
1431              "--debug",
1432              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1433             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1434
1435         body = {
1436             "workflow": {
1437                 "name": "submit_wf.cwl",
1438                 "description": "",
1439                 "definition": self.expect_workflow,
1440             }
1441         }
1442         stubs.api.workflows().update.assert_called_with(
1443             uuid=self.existing_workflow_uuid,
1444             body=JsonDiffMatcher(body))
1445         self.assertEqual(stubs.capture_stdout.getvalue(),
1446                          self.existing_workflow_uuid + '\n')
1447         self.assertEqual(exited, 0)
1448
1449     @stubs
1450     def test_update_name(self, stubs):
1451         exited = arvados_cwl.main(
1452             ["--update-workflow", self.existing_workflow_uuid,
1453              "--debug", "--name", "testing 123",
1454              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1455             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1456
1457         body = {
1458             "workflow": {
1459                 "name": "testing 123",
1460                 "description": "",
1461                 "definition": self.expect_workflow,
1462             }
1463         }
1464         stubs.api.workflows().update.assert_called_with(
1465             uuid=self.existing_workflow_uuid,
1466             body=JsonDiffMatcher(body))
1467         self.assertEqual(stubs.capture_stdout.getvalue(),
1468                          self.existing_workflow_uuid + '\n')
1469         self.assertEqual(exited, 0)
1470
1471     @stubs
1472     def test_create_collection_per_tool(self, stubs):
1473         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1474
1475         exited = arvados_cwl.main(
1476             ["--create-workflow", "--debug",
1477              "--api=containers",
1478              "--project-uuid", project_uuid,
1479              "tests/collection_per_tool/collection_per_tool.cwl"],
1480             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1481
1482         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1483         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1484
1485         body = {
1486             "workflow": {
1487                 "owner_uuid": project_uuid,
1488                 "name": "collection_per_tool.cwl",
1489                 "description": "",
1490                 "definition": expect_workflow,
1491             }
1492         }
1493         stubs.api.workflows().create.assert_called_with(
1494             body=JsonDiffMatcher(body))
1495
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_workflow_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500     @stubs
1501     def test_create_with_imports(self, stubs):
1502         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1503
1504         exited = arvados_cwl.main(
1505             ["--create-workflow", "--debug",
1506              "--api=containers",
1507              "--project-uuid", project_uuid,
1508              "tests/wf/feddemo/feddemo.cwl"],
1509             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1510
1511         stubs.api.pipeline_templates().create.refute_called()
1512         stubs.api.container_requests().create.refute_called()
1513
1514         self.assertEqual(stubs.capture_stdout.getvalue(),
1515                          stubs.expect_workflow_uuid + '\n')
1516         self.assertEqual(exited, 0)
1517
1518     @stubs
1519     def test_create_with_no_input(self, stubs):
1520         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1521
1522         exited = arvados_cwl.main(
1523             ["--create-workflow", "--debug",
1524              "--api=containers",
1525              "--project-uuid", project_uuid,
1526              "tests/wf/revsort/revsort.cwl"],
1527             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1528
1529         stubs.api.pipeline_templates().create.refute_called()
1530         stubs.api.container_requests().create.refute_called()
1531
1532         self.assertEqual(stubs.capture_stdout.getvalue(),
1533                          stubs.expect_workflow_uuid + '\n')
1534         self.assertEqual(exited, 0)