sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=0',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343         arvados_cwl.arvdocker.arv_docker_clear_cache()
 344
 345     @stubs
 346     def test_error_when_multiple_storage_classes_specified(self, stubs):
 347         storage_classes = "foo,bar"
 348         exited = arvados_cwl.main(
 349                 ["--debug", "--storage-classes", storage_classes,
 350                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351                 sys.stdin, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 1)
 353
 354     @mock.patch("time.sleep")
 355     @stubs
 356     def test_submit_invalid_runner_ram(self, stubs, tm):
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 361         self.assertEqual(exited, 1)
 362
 363
 364     @stubs
 365     def test_submit_container(self, stubs):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--api=containers", "--debug",
 368                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 370
 371         stubs.api.collections().create.assert_has_calls([
 372             mock.call(body=JsonDiffMatcher({
 373                 'manifest_text':
 374                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 375                 'replication_desired': None,
 376                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 377             }), ensure_unique_name=False),
 378             mock.call(body=JsonDiffMatcher({
 379                 'manifest_text':
 380                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 381                 'replication_desired': None,
 382                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 383             }), ensure_unique_name=False),
 384             ])
 385
 386         expect_container = copy.deepcopy(stubs.expect_container_spec)
 387         stubs.api.container_requests().create.assert_called_with(
 388             body=JsonDiffMatcher(expect_container))
 389         self.assertEqual(stubs.capture_stdout.getvalue(),
 390                          stubs.expect_container_request_uuid + '\n')
 391         self.assertEqual(exited, 0)
 392
 393
 394     @stubs
 395     def test_submit_container_tool(self, stubs):
 396         # test for issue #16139
 397         exited = arvados_cwl.main(
 398             ["--submit", "--no-wait", "--api=containers", "--debug",
 399                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 400             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 401
 402         self.assertEqual(stubs.capture_stdout.getvalue(),
 403                          stubs.expect_container_request_uuid + '\n')
 404         self.assertEqual(exited, 0)
 405
 406     @stubs
 407     def test_submit_container_no_reuse(self, stubs):
 408         exited = arvados_cwl.main(
 409             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 410                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         expect_container["command"] = [
 415             'arvados-cwl-runner', '--local', '--api=containers',
 416             '--no-log-timestamps', '--disable-validate', '--disable-color',
 417             '--eval-timeout=20', '--thread-count=0',
 418             '--disable-reuse', "--collection-cache-size=256",
 419             '--debug', '--on-error=continue',
 420             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 421         expect_container["use_existing"] = False
 422
 423         stubs.api.container_requests().create.assert_called_with(
 424             body=JsonDiffMatcher(expect_container))
 425         self.assertEqual(stubs.capture_stdout.getvalue(),
 426                          stubs.expect_container_request_uuid + '\n')
 427         self.assertEqual(exited, 0)
 428
 429     @stubs
 430     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug",
 433              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435         self.assertEqual(exited, 0)
 436
 437         expect_container = copy.deepcopy(stubs.expect_container_spec)
 438         expect_container["command"] = [
 439             'arvados-cwl-runner', '--local', '--api=containers',
 440             '--no-log-timestamps', '--disable-validate', '--disable-color',
 441             '--eval-timeout=20', '--thread-count=0',
 442             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 443             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 444         expect_container["use_existing"] = False
 445         expect_container["name"] = "submit_wf_no_reuse.cwl"
 446         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 447             {
 448                 "class": "http://arvados.org/cwl#ReuseRequirement",
 449                 "enableReuse": False,
 450             },
 451         ]
 452         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 453             "arv": "http://arvados.org/cwl#",
 454             "cwltool": "http://commonwl.org/cwltool#"
 455         }
 456
 457         stubs.api.container_requests().create.assert_called_with(
 458             body=JsonDiffMatcher(expect_container))
 459         self.assertEqual(stubs.capture_stdout.getvalue(),
 460                          stubs.expect_container_request_uuid + '\n')
 461
 462
 463     @stubs
 464     def test_submit_container_on_error(self, stubs):
 465         exited = arvados_cwl.main(
 466             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 467                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 468             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 469
 470         expect_container = copy.deepcopy(stubs.expect_container_spec)
 471         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 472                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 473                                        '--eval-timeout=20', '--thread-count=0',
 474                                        '--enable-reuse', "--collection-cache-size=256",
 475                                        '--debug', '--on-error=stop',
 476                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 477
 478         stubs.api.container_requests().create.assert_called_with(
 479             body=JsonDiffMatcher(expect_container))
 480         self.assertEqual(stubs.capture_stdout.getvalue(),
 481                          stubs.expect_container_request_uuid + '\n')
 482         self.assertEqual(exited, 0)
 483
 484     @stubs
 485     def test_submit_container_output_name(self, stubs):
 486         output_name = "test_output_name"
 487
 488         exited = arvados_cwl.main(
 489             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 490                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 491             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 492
 493         expect_container = copy.deepcopy(stubs.expect_container_spec)
 494         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 495                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 496                                        '--eval-timeout=20', '--thread-count=0',
 497                                        '--enable-reuse', "--collection-cache-size=256",
 498                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 499                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 500         expect_container["output_name"] = output_name
 501
 502         stubs.api.container_requests().create.assert_called_with(
 503             body=JsonDiffMatcher(expect_container))
 504         self.assertEqual(stubs.capture_stdout.getvalue(),
 505                          stubs.expect_container_request_uuid + '\n')
 506         self.assertEqual(exited, 0)
 507
 508     @stubs
 509     def test_submit_storage_classes(self, stubs):
 510         exited = arvados_cwl.main(
 511             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 512                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514
 515         expect_container = copy.deepcopy(stubs.expect_container_spec)
 516         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 517                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 518                                        '--eval-timeout=20', '--thread-count=0',
 519                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 520                                        "--storage-classes=foo", '--on-error=continue',
 521                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 522
 523         stubs.api.container_requests().create.assert_called_with(
 524             body=JsonDiffMatcher(expect_container))
 525         self.assertEqual(stubs.capture_stdout.getvalue(),
 526                          stubs.expect_container_request_uuid + '\n')
 527         self.assertEqual(exited, 0)
 528
 529     @mock.patch("cwltool.task_queue.TaskQueue")
 530     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 531     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 532     @stubs
 533     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 534         final_output_c = arvados.collection.Collection()
 535         make_output.return_value = ({},final_output_c)
 536
 537         def set_final_output(job_order, output_callback, runtimeContext):
 538             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 539             return []
 540         job.side_effect = set_final_output
 541
 542         exited = arvados_cwl.main(
 543             ["--debug", "--local", "--storage-classes=foo",
 544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 545             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 546
 547         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 548         self.assertEqual(exited, 0)
 549
 550     @mock.patch("cwltool.task_queue.TaskQueue")
 551     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 552     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 553     @stubs
 554     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 555         final_output_c = arvados.collection.Collection()
 556         make_output.return_value = ({},final_output_c)
 557
 558         def set_final_output(job_order, output_callback, runtimeContext):
 559             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 560             return []
 561         job.side_effect = set_final_output
 562
 563         exited = arvados_cwl.main(
 564             ["--debug", "--local",
 565                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 566             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 567
 568         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 569         self.assertEqual(exited, 0)
 570
 571     @stubs
 572     def test_submit_container_output_ttl(self, stubs):
 573         exited = arvados_cwl.main(
 574             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 575                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 576             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 577
 578         expect_container = copy.deepcopy(stubs.expect_container_spec)
 579         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 580                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 581                                        '--eval-timeout=20', '--thread-count=0',
 582                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 583                                        '--on-error=continue',
 584                                        "--intermediate-output-ttl=3600",
 585                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586
 587         stubs.api.container_requests().create.assert_called_with(
 588             body=JsonDiffMatcher(expect_container))
 589         self.assertEqual(stubs.capture_stdout.getvalue(),
 590                          stubs.expect_container_request_uuid + '\n')
 591         self.assertEqual(exited, 0)
 592
 593     @stubs
 594     def test_submit_container_trash_intermediate(self, stubs):
 595         exited = arvados_cwl.main(
 596             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 597                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599
 600
 601         expect_container = copy.deepcopy(stubs.expect_container_spec)
 602         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 603                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 604                                        '--eval-timeout=20', '--thread-count=0',
 605                                        '--enable-reuse', "--collection-cache-size=256",
 606                                        '--debug', '--on-error=continue',
 607                                        "--trash-intermediate",
 608                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 609
 610         stubs.api.container_requests().create.assert_called_with(
 611             body=JsonDiffMatcher(expect_container))
 612         self.assertEqual(stubs.capture_stdout.getvalue(),
 613                          stubs.expect_container_request_uuid + '\n')
 614         self.assertEqual(exited, 0)
 615
 616     @stubs
 617     def test_submit_container_output_tags(self, stubs):
 618         output_tags = "tag0,tag1,tag2"
 619
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 627                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 628                                        '--eval-timeout=20', '--thread-count=0',
 629                                        '--enable-reuse', "--collection-cache-size=256",
 630                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 631                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 632
 633         stubs.api.container_requests().create.assert_called_with(
 634             body=JsonDiffMatcher(expect_container))
 635         self.assertEqual(stubs.capture_stdout.getvalue(),
 636                          stubs.expect_container_request_uuid + '\n')
 637         self.assertEqual(exited, 0)
 638
 639     @stubs
 640     def test_submit_container_runner_ram(self, stubs):
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 643                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 645
 646         expect_container = copy.deepcopy(stubs.expect_container_spec)
 647         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(stubs.capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653         self.assertEqual(exited, 0)
 654
 655     @mock.patch("arvados.collection.CollectionReader")
 656     @mock.patch("time.sleep")
 657     @stubs
 658     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 659         collectionReader().exists.return_value = True
 660         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug",
 663              "tests/wf/submit_keepref_wf.cwl"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 665         self.assertEqual(exited, 0)
 666
 667     @mock.patch("arvados.collection.CollectionReader")
 668     @mock.patch("time.sleep")
 669     @stubs
 670     def test_submit_keepref(self, stubs, tm, reader):
 671         with open("tests/wf/expect_arvworkflow.cwl") as f:
 672             reader().open().__enter__().read.return_value = f.read()
 673
 674         exited = arvados_cwl.main(
 675             ["--submit", "--no-wait", "--api=containers", "--debug",
 676              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 677             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 678
 679         expect_container = {
 680             'priority': 500,
 681             'mounts': {
 682                 '/var/spool/cwl': {
 683                     'writable': True,
 684                     'kind': 'collection'
 685                 },
 686                 'stdout': {
 687                     'path': '/var/spool/cwl/cwl.output.json',
 688                     'kind': 'file'
 689                 },
 690                 '/var/lib/cwl/workflow': {
 691                     'portable_data_hash': '99999999999999999999999999999994+99',
 692                     'kind': 'collection'
 693                 },
 694                 '/var/lib/cwl/cwl.input.json': {
 695                     'content': {
 696                         'x': 'XxX'
 697                     },
 698                     'kind': 'json'
 699                 }
 700             }, 'state': 'Committed',
 701             'output_path': '/var/spool/cwl',
 702             'name': 'expect_arvworkflow.cwl#main',
 703             'container_image': '999999999999999999999999999999d3+99',
 704             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 705                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 706                         '--eval-timeout=20', '--thread-count=0',
 707                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 708                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 709             'cwd': '/var/spool/cwl',
 710             'runtime_constraints': {
 711                 'API': True,
 712                 'vcpus': 1,
 713                 'ram': 1342177280
 714             },
 715             'use_existing': False,
 716             'properties': {},
 717             'secret_mounts': {}
 718         }
 719
 720         stubs.api.container_requests().create.assert_called_with(
 721             body=JsonDiffMatcher(expect_container))
 722         self.assertEqual(stubs.capture_stdout.getvalue(),
 723                          stubs.expect_container_request_uuid + '\n')
 724         self.assertEqual(exited, 0)
 725
 726     @mock.patch("time.sleep")
 727     @stubs
 728     def test_submit_arvworkflow(self, stubs, tm):
 729         with open("tests/wf/expect_arvworkflow.cwl") as f:
 730             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 731
 732         exited = arvados_cwl.main(
 733             ["--submit", "--no-wait", "--api=containers", "--debug",
 734              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 735             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 736
 737         expect_container = {
 738             'priority': 500,
 739             'mounts': {
 740                 '/var/spool/cwl': {
 741                     'writable': True,
 742                     'kind': 'collection'
 743                 },
 744                 'stdout': {
 745                     'path': '/var/spool/cwl/cwl.output.json',
 746                     'kind': 'file'
 747                 },
 748                 '/var/lib/cwl/workflow.json': {
 749                     'kind': 'json',
 750                     'content': {
 751                         'cwlVersion': 'v1.0',
 752                         '$graph': [
 753                             {
 754                                 'id': '#main',
 755                                 'inputs': [
 756                                     {'type': 'string', 'id': '#main/x'}
 757                                 ],
 758                                 'steps': [
 759                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 760                                      'run': '#submit_tool.cwl',
 761                                      'id': '#main/step1',
 762                                      'out': []}
 763                                 ],
 764                                 'class': 'Workflow',
 765                                 'outputs': []
 766                             },
 767                             {
 768                                 'inputs': [
 769                                     {
 770                                         'inputBinding': {'position': 1},
 771                                         'type': 'string',
 772                                         'id': '#submit_tool.cwl/x'}
 773                                 ],
 774                                 'requirements': [
 775                                     {
 776                                         'dockerPull': 'debian:buster-slim',
 777                                         'class': 'DockerRequirement',
 778                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 779                                     }
 780                                 ],
 781                                 'id': '#submit_tool.cwl',
 782                                 'outputs': [],
 783                                 'baseCommand': 'cat',
 784                                 'class': 'CommandLineTool'
 785                             }
 786                         ]
 787                     }
 788                 },
 789                 '/var/lib/cwl/cwl.input.json': {
 790                     'content': {
 791                         'x': 'XxX'
 792                     },
 793                     'kind': 'json'
 794                 }
 795             }, 'state': 'Committed',
 796             'output_path': '/var/spool/cwl',
 797             'name': 'a test workflow',
 798             'container_image': "999999999999999999999999999999d3+99",
 799             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 800                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 801                         '--eval-timeout=20', '--thread-count=0',
 802                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 803                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 804             'cwd': '/var/spool/cwl',
 805             'runtime_constraints': {
 806                 'API': True,
 807                 'vcpus': 1,
 808                 'ram': 1342177280
 809             },
 810             'use_existing': False,
 811             'properties': {
 812                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 813             },
 814             'secret_mounts': {}
 815         }
 816
 817         stubs.api.container_requests().create.assert_called_with(
 818             body=JsonDiffMatcher(expect_container))
 819         self.assertEqual(stubs.capture_stdout.getvalue(),
 820                          stubs.expect_container_request_uuid + '\n')
 821         self.assertEqual(exited, 0)
 822
 823     @stubs
 824     def test_submit_container_name(self, stubs):
 825         exited = arvados_cwl.main(
 826             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 827                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 828             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 829
 830         expect_container = copy.deepcopy(stubs.expect_container_spec)
 831         expect_container["name"] = "hello container 123"
 832
 833         stubs.api.container_requests().create.assert_called_with(
 834             body=JsonDiffMatcher(expect_container))
 835         self.assertEqual(stubs.capture_stdout.getvalue(),
 836                          stubs.expect_container_request_uuid + '\n')
 837         self.assertEqual(exited, 0)
 838
 839     @stubs
 840     def test_submit_missing_input(self, stubs):
 841         exited = arvados_cwl.main(
 842             ["--submit", "--no-wait", "--api=containers", "--debug",
 843              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 844             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 845         self.assertEqual(exited, 0)
 846
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug",
 849              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851         self.assertEqual(exited, 1)
 852
 853     @stubs
 854     def test_submit_container_project(self, stubs):
 855         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 856         exited = arvados_cwl.main(
 857             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 858                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 859             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 860
 861         expect_container = copy.deepcopy(stubs.expect_container_spec)
 862         expect_container["owner_uuid"] = project_uuid
 863         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 864                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 865                                        "--eval-timeout=20", "--thread-count=0",
 866                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 867                                        '--on-error=continue',
 868                                        '--project-uuid='+project_uuid,
 869                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 870
 871         stubs.api.container_requests().create.assert_called_with(
 872             body=JsonDiffMatcher(expect_container))
 873         self.assertEqual(stubs.capture_stdout.getvalue(),
 874                          stubs.expect_container_request_uuid + '\n')
 875         self.assertEqual(exited, 0)
 876
 877     @stubs
 878     def test_submit_container_eval_timeout(self, stubs):
 879         exited = arvados_cwl.main(
 880             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 881                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 882             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 883
 884         expect_container = copy.deepcopy(stubs.expect_container_spec)
 885         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 886                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 887                                        '--eval-timeout=60.0', '--thread-count=0',
 888                                        '--enable-reuse', "--collection-cache-size=256",
 889                                        '--debug', '--on-error=continue',
 890                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 891
 892         stubs.api.container_requests().create.assert_called_with(
 893             body=JsonDiffMatcher(expect_container))
 894         self.assertEqual(stubs.capture_stdout.getvalue(),
 895                          stubs.expect_container_request_uuid + '\n')
 896         self.assertEqual(exited, 0)
 897
 898     @stubs
 899     def test_submit_container_collection_cache(self, stubs):
 900         exited = arvados_cwl.main(
 901             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 902                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904
 905         expect_container = copy.deepcopy(stubs.expect_container_spec)
 906         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 907                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 908                                        '--eval-timeout=20', '--thread-count=0',
 909                                        '--enable-reuse', "--collection-cache-size=500",
 910                                        '--debug', '--on-error=continue',
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 913
 914         stubs.api.container_requests().create.assert_called_with(
 915             body=JsonDiffMatcher(expect_container))
 916         self.assertEqual(stubs.capture_stdout.getvalue(),
 917                          stubs.expect_container_request_uuid + '\n')
 918         self.assertEqual(exited, 0)
 919
 920     @stubs
 921     def test_submit_container_thread_count(self, stubs):
 922         exited = arvados_cwl.main(
 923             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 924                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 925             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 926
 927         expect_container = copy.deepcopy(stubs.expect_container_spec)
 928         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 929                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 930                                        '--eval-timeout=20', '--thread-count=20',
 931                                        '--enable-reuse', "--collection-cache-size=256",
 932                                        '--debug', '--on-error=continue',
 933                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 934
 935         stubs.api.container_requests().create.assert_called_with(
 936             body=JsonDiffMatcher(expect_container))
 937         self.assertEqual(stubs.capture_stdout.getvalue(),
 938                          stubs.expect_container_request_uuid + '\n')
 939         self.assertEqual(exited, 0)
 940
 941     @stubs
 942     def test_submit_container_runner_image(self, stubs):
 943         exited = arvados_cwl.main(
 944             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 945                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 946             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 947
 948         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 949
 950         expect_container = copy.deepcopy(stubs.expect_container_spec)
 951         stubs.api.container_requests().create.assert_called_with(
 952             body=JsonDiffMatcher(expect_container))
 953         self.assertEqual(stubs.capture_stdout.getvalue(),
 954                          stubs.expect_container_request_uuid + '\n')
 955         self.assertEqual(exited, 0)
 956
 957     @stubs
 958     def test_submit_priority(self, stubs):
 959         exited = arvados_cwl.main(
 960             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 961                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 962             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 963
 964         stubs.expect_container_spec["priority"] = 669
 965
 966         expect_container = copy.deepcopy(stubs.expect_container_spec)
 967         stubs.api.container_requests().create.assert_called_with(
 968             body=JsonDiffMatcher(expect_container))
 969         self.assertEqual(stubs.capture_stdout.getvalue(),
 970                          stubs.expect_container_request_uuid + '\n')
 971         self.assertEqual(exited, 0)
 972
 973     @stubs
 974     def test_submit_wf_runner_resources(self, stubs):
 975         exited = arvados_cwl.main(
 976             ["--submit", "--no-wait", "--api=containers", "--debug",
 977                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 978             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 979
 980         expect_container = copy.deepcopy(stubs.expect_container_spec)
 981         expect_container["runtime_constraints"] = {
 982             "API": True,
 983             "vcpus": 2,
 984             "ram": (2000+512) * 2**20
 985         }
 986         expect_container["name"] = "submit_wf_runner_resources.cwl"
 987         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 988             {
 989                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 990                 "coresMin": 2,
 991                 "ramMin": 2000,
 992                 "keep_cache": 512
 993             }
 994         ]
 995         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 996             "arv": "http://arvados.org/cwl#",
 997         }
 998         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 999                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1000                         '--eval-timeout=20', '--thread-count=0',
1001                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1002                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1003
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     def tearDown(self):
1011         arvados_cwl.arvdocker.arv_docker_clear_cache()
1012
1013     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1014     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1015     @mock.patch("arvados.api")
1016     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1017         arvados_cwl.arvdocker.arv_docker_clear_cache()
1018
1019         arvrunner = mock.MagicMock()
1020         arvrunner.project_uuid = ""
1021         api.return_value = mock.MagicMock()
1022         arvrunner.api = api.return_value
1023         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1024                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1025                                                                         "link_class": "docker_image_repo+tag",
1026                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1027                                                                         "owner_uuid": "",
1028                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1029                                                             {"items": [{"created_at": "",
1030                                                                         "head_uuid": "",
1031                                                                         "link_class": "docker_image_hash",
1032                                                                         "name": "123456",
1033                                                                         "owner_uuid": "",
1034                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1035         )
1036         find_one_image_hash.return_value = "123456"
1037
1038         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1039                                                                               "owner_uuid": "",
1040                                                                               "manifest_text": "",
1041                                                                               "properties": ""
1042                                                                           }], "items_available": 1, "offset": 0},)
1043         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1044         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1045                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1046         self.assertEqual("9999999999999999999999999999999b+99",
1047                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1048
1049
1050     @stubs
1051     def test_submit_secrets(self, stubs):
1052         exited = arvados_cwl.main(
1053             ["--submit", "--no-wait", "--api=containers", "--debug",
1054                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1055             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1056
1057         expect_container = {
1058             "command": [
1059                 "arvados-cwl-runner",
1060                 "--local",
1061                 "--api=containers",
1062                 "--no-log-timestamps",
1063                 "--disable-validate",
1064                 "--disable-color",
1065                 "--eval-timeout=20",
1066                 '--thread-count=0',
1067                 "--enable-reuse",
1068                 "--collection-cache-size=256",
1069                 '--debug',
1070                 "--on-error=continue",
1071                 "/var/lib/cwl/workflow.json#main",
1072                 "/var/lib/cwl/cwl.input.json"
1073             ],
1074             "container_image": "999999999999999999999999999999d3+99",
1075             "cwd": "/var/spool/cwl",
1076             "mounts": {
1077                 "/var/lib/cwl/cwl.input.json": {
1078                     "content": {
1079                         "pw": {
1080                             "$include": "/secrets/s0"
1081                         }
1082                     },
1083                     "kind": "json"
1084                 },
1085                 "/var/lib/cwl/workflow.json": {
1086                     "content": {
1087                         "$graph": [
1088                             {
1089                                 "$namespaces": {
1090                                     "cwltool": "http://commonwl.org/cwltool#"
1091                                 },
1092                                 "arguments": [
1093                                     "md5sum",
1094                                     "example.conf"
1095                                 ],
1096                                 "class": "CommandLineTool",
1097                                 "hints": [
1098                                     {
1099                                         "class": "http://commonwl.org/cwltool#Secrets",
1100                                         "secrets": [
1101                                             "#secret_job.cwl/pw"
1102                                         ]
1103                                     }
1104                                 ],
1105                                 "id": "#secret_job.cwl",
1106                                 "inputs": [
1107                                     {
1108                                         "id": "#secret_job.cwl/pw",
1109                                         "type": "string"
1110                                     }
1111                                 ],
1112                                 "outputs": [
1113                                     {
1114                                         "id": "#secret_job.cwl/out",
1115                                         "type": "File",
1116                                         "outputBinding": {
1117                                               "glob": "hashed_example.txt"
1118                                         }
1119                                     }
1120                                 ],
1121                                 "stdout": "hashed_example.txt",
1122                                 "requirements": [
1123                                     {
1124                                         "class": "InitialWorkDirRequirement",
1125                                         "listing": [
1126                                             {
1127                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1128                                                 "entryname": "example.conf"
1129                                             }
1130                                         ]
1131                                     }
1132                                 ]
1133                             },
1134                             {
1135                                 "class": "Workflow",
1136                                 "hints": [
1137                                     {
1138                                         "class": "DockerRequirement",
1139                                         "dockerPull": "debian:buster-slim",
1140                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1141                                     },
1142                                     {
1143                                         "class": "http://commonwl.org/cwltool#Secrets",
1144                                         "secrets": [
1145                                             "#main/pw"
1146                                         ]
1147                                     }
1148                                 ],
1149                                 "id": "#main",
1150                                 "inputs": [
1151                                     {
1152                                         "id": "#main/pw",
1153                                         "type": "string"
1154                                     }
1155                                 ],
1156                                 "outputs": [
1157                                     {
1158                                         "id": "#main/out",
1159                                         "outputSource": "#main/step1/out",
1160                                         "type": "File"
1161                                     }
1162                                 ],
1163                                 "steps": [
1164                                     {
1165                                         "id": "#main/step1",
1166                                         "in": [
1167                                             {
1168                                                 "id": "#main/step1/pw",
1169                                                 "source": "#main/pw"
1170                                             }
1171                                         ],
1172                                         "out": [
1173                                             "#main/step1/out"
1174                                         ],
1175                                         "run": "#secret_job.cwl"
1176                                     }
1177                                 ]
1178                             }
1179                         ],
1180                         "cwlVersion": "v1.0"
1181                     },
1182                     "kind": "json"
1183                 },
1184                 "/var/spool/cwl": {
1185                     "kind": "collection",
1186                     "writable": True
1187                 },
1188                 "stdout": {
1189                     "kind": "file",
1190                     "path": "/var/spool/cwl/cwl.output.json"
1191                 }
1192             },
1193             "name": "secret_wf.cwl",
1194             "output_path": "/var/spool/cwl",
1195             "priority": 500,
1196             "properties": {},
1197             "runtime_constraints": {
1198                 "API": True,
1199                 "ram": 1342177280,
1200                 "vcpus": 1
1201             },
1202             "secret_mounts": {
1203                 "/secrets/s0": {
1204                     "content": "blorp",
1205                     "kind": "text"
1206                 }
1207             },
1208             "state": "Committed",
1209             "use_existing": False
1210         }
1211
1212         stubs.api.container_requests().create.assert_called_with(
1213             body=JsonDiffMatcher(expect_container))
1214         self.assertEqual(stubs.capture_stdout.getvalue(),
1215                          stubs.expect_container_request_uuid + '\n')
1216         self.assertEqual(exited, 0)
1217
1218     @stubs
1219     def test_submit_request_uuid(self, stubs):
1220         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1221         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1222
1223         stubs.api.container_requests().update().execute.return_value = {
1224             "uuid": stubs.expect_container_request_uuid,
1225             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1226             "state": "Queued"
1227         }
1228
1229         exited = arvados_cwl.main(
1230             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1231                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1232             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1233
1234         stubs.api.container_requests().update.assert_called_with(
1235             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1236         self.assertEqual(stubs.capture_stdout.getvalue(),
1237                          stubs.expect_container_request_uuid + '\n')
1238         self.assertEqual(exited, 0)
1239
1240     @stubs
1241     def test_submit_container_cluster_id(self, stubs):
1242         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1243
1244         exited = arvados_cwl.main(
1245             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1246                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1247             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1248
1249         expect_container = copy.deepcopy(stubs.expect_container_spec)
1250
1251         stubs.api.container_requests().create.assert_called_with(
1252             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1253         self.assertEqual(stubs.capture_stdout.getvalue(),
1254                          stubs.expect_container_request_uuid + '\n')
1255         self.assertEqual(exited, 0)
1256
1257     @stubs
1258     def test_submit_validate_cluster_id(self, stubs):
1259         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1260         exited = arvados_cwl.main(
1261             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1262              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1263             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1264         self.assertEqual(exited, 1)
1265
1266     @stubs
1267     def test_submit_validate_project_uuid(self, stubs):
1268         exited = arvados_cwl.main(
1269             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1270              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1271             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1272         self.assertEqual(exited, 1)
1273
1274         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1275         exited = arvados_cwl.main(
1276             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1277              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1278             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1279         self.assertEqual(exited, 1)
1280
1281     @mock.patch("arvados.collection.CollectionReader")
1282     @stubs
1283     def test_submit_uuid_inputs(self, stubs, collectionReader):
1284         collectionReader().exists.return_value = True
1285         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1286         def list_side_effect(**kwargs):
1287             m = mock.MagicMock()
1288             if "count" in kwargs:
1289                 m.execute.return_value = {"items": [
1290                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1291                 ]}
1292             else:
1293                 m.execute.return_value = {"items": []}
1294             return m
1295         stubs.api.collections().list.side_effect = list_side_effect
1296
1297         exited = arvados_cwl.main(
1298             ["--submit", "--no-wait", "--api=containers", "--debug",
1299                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1300             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1301
1302         expect_container = copy.deepcopy(stubs.expect_container_spec)
1303         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1304         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1305         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1306
1307         stubs.api.collections().list.assert_has_calls([
1308             mock.call(count='none',
1309                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1310                       select=['uuid', 'portable_data_hash'])])
1311         stubs.api.container_requests().create.assert_called_with(
1312             body=JsonDiffMatcher(expect_container))
1313         self.assertEqual(stubs.capture_stdout.getvalue(),
1314                          stubs.expect_container_request_uuid + '\n')
1315         self.assertEqual(exited, 0)
1316
1317     @stubs
1318     def test_submit_mismatched_uuid_inputs(self, stubs):
1319         def list_side_effect(**kwargs):
1320             m = mock.MagicMock()
1321             if "count" in kwargs:
1322                 m.execute.return_value = {"items": [
1323                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1324                 ]}
1325             else:
1326                 m.execute.return_value = {"items": []}
1327             return m
1328         stubs.api.collections().list.side_effect = list_side_effect
1329
1330         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1331             capture_stderr = StringIO()
1332             cwltool_logger = logging.getLogger('cwltool')
1333             stderr_logger = logging.StreamHandler(capture_stderr)
1334             cwltool_logger.addHandler(stderr_logger)
1335
1336             try:
1337                 exited = arvados_cwl.main(
1338                     ["--submit", "--no-wait", "--api=containers", "--debug",
1339                         "tests/wf/submit_wf.cwl", infile],
1340                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1341
1342                 self.assertEqual(exited, 1)
1343                 self.assertRegex(
1344                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1345                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1346             finally:
1347                 cwltool_logger.removeHandler(stderr_logger)
1348
1349     @mock.patch("arvados.collection.CollectionReader")
1350     @stubs
1351     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1352         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1353         capture_stderr = StringIO()
1354
1355         cwltool_logger = logging.getLogger('cwltool')
1356         stderr_logger = logging.StreamHandler(capture_stderr)
1357         cwltool_logger.addHandler(stderr_logger)
1358
1359         exited = arvados_cwl.main(
1360             ["--submit", "--no-wait", "--api=containers", "--debug",
1361                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1362             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1363
1364         try:
1365             self.assertEqual(exited, 1)
1366             self.assertRegex(
1367                 capture_stderr.getvalue(),
1368                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1369         finally:
1370             cwltool_logger.removeHandler(stderr_logger)
1371
1372
1373 class TestCreateWorkflow(unittest.TestCase):
1374     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1375     expect_workflow = StripYAMLComments(
1376         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1377
1378     def setUp(self):
1379         cwltool.process._names = set()
1380         arvados_cwl.arvdocker.arv_docker_clear_cache()
1381
1382     @stubs
1383     def test_create(self, stubs):
1384         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1385
1386         exited = arvados_cwl.main(
1387             ["--create-workflow", "--debug",
1388              "--api=containers",
1389              "--project-uuid", project_uuid,
1390              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1391             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1392
1393         stubs.api.pipeline_templates().create.refute_called()
1394         stubs.api.container_requests().create.refute_called()
1395
1396         body = {
1397             "workflow": {
1398                 "owner_uuid": project_uuid,
1399                 "name": "submit_wf.cwl",
1400                 "description": "",
1401                 "definition": self.expect_workflow,
1402             }
1403         }
1404         stubs.api.workflows().create.assert_called_with(
1405             body=JsonDiffMatcher(body))
1406
1407         self.assertEqual(stubs.capture_stdout.getvalue(),
1408                          stubs.expect_workflow_uuid + '\n')
1409         self.assertEqual(exited, 0)
1410
1411     @stubs
1412     def test_create_name(self, stubs):
1413         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1414
1415         exited = arvados_cwl.main(
1416             ["--create-workflow", "--debug",
1417              "--api=containers",
1418              "--project-uuid", project_uuid,
1419              "--name", "testing 123",
1420              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1421             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1422
1423         stubs.api.pipeline_templates().create.refute_called()
1424         stubs.api.container_requests().create.refute_called()
1425
1426         body = {
1427             "workflow": {
1428                 "owner_uuid": project_uuid,
1429                 "name": "testing 123",
1430                 "description": "",
1431                 "definition": self.expect_workflow,
1432             }
1433         }
1434         stubs.api.workflows().create.assert_called_with(
1435             body=JsonDiffMatcher(body))
1436
1437         self.assertEqual(stubs.capture_stdout.getvalue(),
1438                          stubs.expect_workflow_uuid + '\n')
1439         self.assertEqual(exited, 0)
1440
1441
1442     @stubs
1443     def test_update(self, stubs):
1444         exited = arvados_cwl.main(
1445             ["--update-workflow", self.existing_workflow_uuid,
1446              "--debug",
1447              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1448             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1449
1450         body = {
1451             "workflow": {
1452                 "name": "submit_wf.cwl",
1453                 "description": "",
1454                 "definition": self.expect_workflow,
1455             }
1456         }
1457         stubs.api.workflows().update.assert_called_with(
1458             uuid=self.existing_workflow_uuid,
1459             body=JsonDiffMatcher(body))
1460         self.assertEqual(stubs.capture_stdout.getvalue(),
1461                          self.existing_workflow_uuid + '\n')
1462         self.assertEqual(exited, 0)
1463
1464     @stubs
1465     def test_update_name(self, stubs):
1466         exited = arvados_cwl.main(
1467             ["--update-workflow", self.existing_workflow_uuid,
1468              "--debug", "--name", "testing 123",
1469              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1470             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1471
1472         body = {
1473             "workflow": {
1474                 "name": "testing 123",
1475                 "description": "",
1476                 "definition": self.expect_workflow,
1477             }
1478         }
1479         stubs.api.workflows().update.assert_called_with(
1480             uuid=self.existing_workflow_uuid,
1481             body=JsonDiffMatcher(body))
1482         self.assertEqual(stubs.capture_stdout.getvalue(),
1483                          self.existing_workflow_uuid + '\n')
1484         self.assertEqual(exited, 0)
1485
1486     @stubs
1487     def test_create_collection_per_tool(self, stubs):
1488         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1489
1490         exited = arvados_cwl.main(
1491             ["--create-workflow", "--debug",
1492              "--api=containers",
1493              "--project-uuid", project_uuid,
1494              "tests/collection_per_tool/collection_per_tool.cwl"],
1495             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1496
1497         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1498         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1499
1500         body = {
1501             "workflow": {
1502                 "owner_uuid": project_uuid,
1503                 "name": "collection_per_tool.cwl",
1504                 "description": "",
1505                 "definition": expect_workflow,
1506             }
1507         }
1508         stubs.api.workflows().create.assert_called_with(
1509             body=JsonDiffMatcher(body))
1510
1511         self.assertEqual(stubs.capture_stdout.getvalue(),
1512                          stubs.expect_workflow_uuid + '\n')
1513         self.assertEqual(exited, 0)
1514
1515     @stubs
1516     def test_create_with_imports(self, stubs):
1517         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1518
1519         exited = arvados_cwl.main(
1520             ["--create-workflow", "--debug",
1521              "--api=containers",
1522              "--project-uuid", project_uuid,
1523              "tests/wf/feddemo/feddemo.cwl"],
1524             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1525
1526         stubs.api.pipeline_templates().create.refute_called()
1527         stubs.api.container_requests().create.refute_called()
1528
1529         self.assertEqual(stubs.capture_stdout.getvalue(),
1530                          stubs.expect_workflow_uuid + '\n')
1531         self.assertEqual(exited, 0)
1532
1533     @stubs
1534     def test_create_with_no_input(self, stubs):
1535         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1536
1537         exited = arvados_cwl.main(
1538             ["--create-workflow", "--debug",
1539              "--api=containers",
1540              "--project-uuid", project_uuid,
1541              "tests/wf/revsort/revsort.cwl"],
1542             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1543
1544         stubs.api.pipeline_templates().create.refute_called()
1545         stubs.api.container_requests().create.refute_called()
1546
1547         self.assertEqual(stubs.capture_stdout.getvalue(),
1548                          stubs.expect_workflow_uuid + '\n')
1549         self.assertEqual(exited, 0)