sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate',
 306                         '--eval-timeout=20', '--thread-count=1',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343
 344     @stubs
 345     def test_error_when_multiple_storage_classes_specified(self, stubs):
 346         storage_classes = "foo,bar"
 347         exited = arvados_cwl.main(
 348                 ["--debug", "--storage-classes", storage_classes,
 349                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 350                 sys.stdin, sys.stderr, api_client=stubs.api)
 351         self.assertEqual(exited, 1)
 352
 353     @mock.patch("time.sleep")
 354     @stubs
 355     def test_submit_invalid_runner_ram(self, stubs, tm):
 356         exited = arvados_cwl.main(
 357             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 359             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 360         self.assertEqual(exited, 1)
 361
 362
 363     @stubs
 364     def test_submit_container(self, stubs):
 365         exited = arvados_cwl.main(
 366             ["--submit", "--no-wait", "--api=containers", "--debug",
 367                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 368             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 369
 370         stubs.api.collections().create.assert_has_calls([
 371             mock.call(body=JsonDiffMatcher({
 372                 'manifest_text':
 373                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 374                 'replication_desired': None,
 375                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 376             }), ensure_unique_name=False),
 377             mock.call(body=JsonDiffMatcher({
 378                 'manifest_text':
 379                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 380                 'replication_desired': None,
 381                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 382             }), ensure_unique_name=False),
 383             ])
 384
 385         expect_container = copy.deepcopy(stubs.expect_container_spec)
 386         stubs.api.container_requests().create.assert_called_with(
 387             body=JsonDiffMatcher(expect_container))
 388         self.assertEqual(stubs.capture_stdout.getvalue(),
 389                          stubs.expect_container_request_uuid + '\n')
 390         self.assertEqual(exited, 0)
 391
 392
 393     @stubs
 394     def test_submit_container_tool(self, stubs):
 395         # test for issue #16139
 396         exited = arvados_cwl.main(
 397             ["--submit", "--no-wait", "--api=containers", "--debug",
 398                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 399             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 400
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_submit_container_no_reuse(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = [
 414             'arvados-cwl-runner', '--local', '--api=containers',
 415             '--no-log-timestamps', '--disable-validate',
 416             '--eval-timeout=20', '--thread-count=1',
 417             '--disable-reuse', "--collection-cache-size=256",
 418             '--debug', '--on-error=continue',
 419             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420         expect_container["use_existing"] = False
 421
 422         stubs.api.container_requests().create.assert_called_with(
 423             body=JsonDiffMatcher(expect_container))
 424         self.assertEqual(stubs.capture_stdout.getvalue(),
 425                          stubs.expect_container_request_uuid + '\n')
 426         self.assertEqual(exited, 0)
 427
 428     @stubs
 429     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 430         exited = arvados_cwl.main(
 431             ["--submit", "--no-wait", "--api=containers", "--debug",
 432              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 433             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 434         self.assertEqual(exited, 0)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = [
 438             'arvados-cwl-runner', '--local', '--api=containers',
 439             '--no-log-timestamps', '--disable-validate',
 440             '--eval-timeout=20', '--thread-count=1',
 441             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 442             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["use_existing"] = False
 444         expect_container["name"] = "submit_wf_no_reuse.cwl"
 445         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 446             {
 447                 "class": "http://arvados.org/cwl#ReuseRequirement",
 448                 "enableReuse": False,
 449             },
 450         ]
 451         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 452             "arv": "http://arvados.org/cwl#",
 453             "cwltool": "http://commonwl.org/cwltool#"
 454         }
 455
 456         stubs.api.container_requests().create.assert_called_with(
 457             body=JsonDiffMatcher(expect_container))
 458         self.assertEqual(stubs.capture_stdout.getvalue(),
 459                          stubs.expect_container_request_uuid + '\n')
 460
 461
 462     @stubs
 463     def test_submit_container_on_error(self, stubs):
 464         exited = arvados_cwl.main(
 465             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 466                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468
 469         expect_container = copy.deepcopy(stubs.expect_container_spec)
 470         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 471                                        '--no-log-timestamps', '--disable-validate',
 472                                        '--eval-timeout=20', '--thread-count=1',
 473                                        '--enable-reuse', "--collection-cache-size=256",
 474                                        '--debug', '--on-error=stop',
 475                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 476
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @stubs
 484     def test_submit_container_output_name(self, stubs):
 485         output_name = "test_output_name"
 486
 487         exited = arvados_cwl.main(
 488             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 489                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 490             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 491
 492         expect_container = copy.deepcopy(stubs.expect_container_spec)
 493         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 494                                        '--no-log-timestamps', '--disable-validate',
 495                                        '--eval-timeout=20', '--thread-count=1',
 496                                        '--enable-reuse', "--collection-cache-size=256",
 497                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 498                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 499         expect_container["output_name"] = output_name
 500
 501         stubs.api.container_requests().create.assert_called_with(
 502             body=JsonDiffMatcher(expect_container))
 503         self.assertEqual(stubs.capture_stdout.getvalue(),
 504                          stubs.expect_container_request_uuid + '\n')
 505         self.assertEqual(exited, 0)
 506
 507     @stubs
 508     def test_submit_storage_classes(self, stubs):
 509         exited = arvados_cwl.main(
 510             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 511                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 512             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 513
 514         expect_container = copy.deepcopy(stubs.expect_container_spec)
 515         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 516                                        '--no-log-timestamps', '--disable-validate',
 517                                        '--eval-timeout=20', '--thread-count=1',
 518                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 519                                        "--storage-classes=foo", '--on-error=continue',
 520                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 521
 522         stubs.api.container_requests().create.assert_called_with(
 523             body=JsonDiffMatcher(expect_container))
 524         self.assertEqual(stubs.capture_stdout.getvalue(),
 525                          stubs.expect_container_request_uuid + '\n')
 526         self.assertEqual(exited, 0)
 527
 528     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 529     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 530     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 531     @stubs
 532     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 533         final_output_c = arvados.collection.Collection()
 534         make_output.return_value = ({},final_output_c)
 535
 536         def set_final_output(job_order, output_callback, runtimeContext):
 537             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 538             return []
 539         job.side_effect = set_final_output
 540
 541         exited = arvados_cwl.main(
 542             ["--debug", "--local", "--storage-classes=foo",
 543                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 544             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 545
 546         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 547         self.assertEqual(exited, 0)
 548
 549     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 550     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 551     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 552     @stubs
 553     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 554         final_output_c = arvados.collection.Collection()
 555         make_output.return_value = ({},final_output_c)
 556
 557         def set_final_output(job_order, output_callback, runtimeContext):
 558             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 559             return []
 560         job.side_effect = set_final_output
 561
 562         exited = arvados_cwl.main(
 563             ["--debug", "--local",
 564                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 565             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 566
 567         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 568         self.assertEqual(exited, 0)
 569
 570     @stubs
 571     def test_submit_container_output_ttl(self, stubs):
 572         exited = arvados_cwl.main(
 573             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         expect_container = copy.deepcopy(stubs.expect_container_spec)
 578         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 579                                        '--no-log-timestamps', '--disable-validate',
 580                                        '--eval-timeout=20', '--thread-count=1',
 581                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 582                                        '--on-error=continue',
 583                                        "--intermediate-output-ttl=3600",
 584                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 585
 586         stubs.api.container_requests().create.assert_called_with(
 587             body=JsonDiffMatcher(expect_container))
 588         self.assertEqual(stubs.capture_stdout.getvalue(),
 589                          stubs.expect_container_request_uuid + '\n')
 590         self.assertEqual(exited, 0)
 591
 592     @stubs
 593     def test_submit_container_trash_intermediate(self, stubs):
 594         exited = arvados_cwl.main(
 595             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 596                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598
 599
 600         expect_container = copy.deepcopy(stubs.expect_container_spec)
 601         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 602                                        '--no-log-timestamps', '--disable-validate',
 603                                        '--eval-timeout=20', '--thread-count=1',
 604                                        '--enable-reuse', "--collection-cache-size=256",
 605                                        '--debug', '--on-error=continue',
 606                                        "--trash-intermediate",
 607                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 608
 609         stubs.api.container_requests().create.assert_called_with(
 610             body=JsonDiffMatcher(expect_container))
 611         self.assertEqual(stubs.capture_stdout.getvalue(),
 612                          stubs.expect_container_request_uuid + '\n')
 613         self.assertEqual(exited, 0)
 614
 615     @stubs
 616     def test_submit_container_output_tags(self, stubs):
 617         output_tags = "tag0,tag1,tag2"
 618
 619         exited = arvados_cwl.main(
 620             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 621                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 622             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 623
 624         expect_container = copy.deepcopy(stubs.expect_container_spec)
 625         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 626                                        '--no-log-timestamps', '--disable-validate',
 627                                        '--eval-timeout=20', '--thread-count=1',
 628                                        '--enable-reuse', "--collection-cache-size=256",
 629                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 630                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 631
 632         stubs.api.container_requests().create.assert_called_with(
 633             body=JsonDiffMatcher(expect_container))
 634         self.assertEqual(stubs.capture_stdout.getvalue(),
 635                          stubs.expect_container_request_uuid + '\n')
 636         self.assertEqual(exited, 0)
 637
 638     @stubs
 639     def test_submit_container_runner_ram(self, stubs):
 640         exited = arvados_cwl.main(
 641             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 642                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 643             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 644
 645         expect_container = copy.deepcopy(stubs.expect_container_spec)
 646         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 647
 648         stubs.api.container_requests().create.assert_called_with(
 649             body=JsonDiffMatcher(expect_container))
 650         self.assertEqual(stubs.capture_stdout.getvalue(),
 651                          stubs.expect_container_request_uuid + '\n')
 652         self.assertEqual(exited, 0)
 653
 654     @mock.patch("arvados.collection.CollectionReader")
 655     @mock.patch("time.sleep")
 656     @stubs
 657     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 658         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 659         exited = arvados_cwl.main(
 660             ["--submit", "--no-wait", "--api=containers", "--debug",
 661              "tests/wf/submit_keepref_wf.cwl"],
 662             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 663         self.assertEqual(exited, 0)
 664
 665     @mock.patch("arvados.collection.CollectionReader")
 666     @mock.patch("time.sleep")
 667     @stubs
 668     def test_submit_keepref(self, stubs, tm, reader):
 669         with open("tests/wf/expect_arvworkflow.cwl") as f:
 670             reader().open().__enter__().read.return_value = f.read()
 671
 672         exited = arvados_cwl.main(
 673             ["--submit", "--no-wait", "--api=containers", "--debug",
 674              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 676
 677         expect_container = {
 678             'priority': 500,
 679             'mounts': {
 680                 '/var/spool/cwl': {
 681                     'writable': True,
 682                     'kind': 'collection'
 683                 },
 684                 'stdout': {
 685                     'path': '/var/spool/cwl/cwl.output.json',
 686                     'kind': 'file'
 687                 },
 688                 '/var/lib/cwl/workflow': {
 689                     'portable_data_hash': '99999999999999999999999999999994+99',
 690                     'kind': 'collection'
 691                 },
 692                 '/var/lib/cwl/cwl.input.json': {
 693                     'content': {
 694                         'x': 'XxX'
 695                     },
 696                     'kind': 'json'
 697                 }
 698             }, 'state': 'Committed',
 699             'output_path': '/var/spool/cwl',
 700             'name': 'expect_arvworkflow.cwl#main',
 701             'container_image': '999999999999999999999999999999d3+99',
 702             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 703                         '--no-log-timestamps', '--disable-validate',
 704                         '--eval-timeout=20', '--thread-count=1',
 705                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 706                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 707             'cwd': '/var/spool/cwl',
 708             'runtime_constraints': {
 709                 'API': True,
 710                 'vcpus': 1,
 711                 'ram': 1342177280
 712             },
 713             'use_existing': False,
 714             'properties': {},
 715             'secret_mounts': {}
 716         }
 717
 718         stubs.api.container_requests().create.assert_called_with(
 719             body=JsonDiffMatcher(expect_container))
 720         self.assertEqual(stubs.capture_stdout.getvalue(),
 721                          stubs.expect_container_request_uuid + '\n')
 722         self.assertEqual(exited, 0)
 723
 724     @mock.patch("time.sleep")
 725     @stubs
 726     def test_submit_arvworkflow(self, stubs, tm):
 727         with open("tests/wf/expect_arvworkflow.cwl") as f:
 728             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 729
 730         exited = arvados_cwl.main(
 731             ["--submit", "--no-wait", "--api=containers", "--debug",
 732              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 733             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 734
 735         expect_container = {
 736             'priority': 500,
 737             'mounts': {
 738                 '/var/spool/cwl': {
 739                     'writable': True,
 740                     'kind': 'collection'
 741                 },
 742                 'stdout': {
 743                     'path': '/var/spool/cwl/cwl.output.json',
 744                     'kind': 'file'
 745                 },
 746                 '/var/lib/cwl/workflow.json': {
 747                     'kind': 'json',
 748                     'content': {
 749                         'cwlVersion': 'v1.0',
 750                         '$graph': [
 751                             {
 752                                 'id': '#main',
 753                                 'inputs': [
 754                                     {'type': 'string', 'id': '#main/x'}
 755                                 ],
 756                                 'steps': [
 757                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 758                                      'run': '#submit_tool.cwl',
 759                                      'id': '#main/step1',
 760                                      'out': []}
 761                                 ],
 762                                 'class': 'Workflow',
 763                                 'outputs': []
 764                             },
 765                             {
 766                                 'inputs': [
 767                                     {
 768                                         'inputBinding': {'position': 1},
 769                                         'type': 'string',
 770                                         'id': '#submit_tool.cwl/x'}
 771                                 ],
 772                                 'requirements': [
 773                                     {
 774                                         'dockerPull': 'debian:8',
 775                                         'class': 'DockerRequirement',
 776                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 777                                     }
 778                                 ],
 779                                 'id': '#submit_tool.cwl',
 780                                 'outputs': [],
 781                                 'baseCommand': 'cat',
 782                                 'class': 'CommandLineTool'
 783                             }
 784                         ]
 785                     }
 786                 },
 787                 '/var/lib/cwl/cwl.input.json': {
 788                     'content': {
 789                         'x': 'XxX'
 790                     },
 791                     'kind': 'json'
 792                 }
 793             }, 'state': 'Committed',
 794             'output_path': '/var/spool/cwl',
 795             'name': 'a test workflow',
 796             'container_image': "999999999999999999999999999999d3+99",
 797             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 798                         '--no-log-timestamps', '--disable-validate',
 799                         '--eval-timeout=20', '--thread-count=1',
 800                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 801                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 802             'cwd': '/var/spool/cwl',
 803             'runtime_constraints': {
 804                 'API': True,
 805                 'vcpus': 1,
 806                 'ram': 1342177280
 807             },
 808             'use_existing': False,
 809             'properties': {
 810                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 811             },
 812             'secret_mounts': {}
 813         }
 814
 815         stubs.api.container_requests().create.assert_called_with(
 816             body=JsonDiffMatcher(expect_container))
 817         self.assertEqual(stubs.capture_stdout.getvalue(),
 818                          stubs.expect_container_request_uuid + '\n')
 819         self.assertEqual(exited, 0)
 820
 821     @stubs
 822     def test_submit_container_name(self, stubs):
 823         exited = arvados_cwl.main(
 824             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 825                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 826             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 827
 828         expect_container = copy.deepcopy(stubs.expect_container_spec)
 829         expect_container["name"] = "hello container 123"
 830
 831         stubs.api.container_requests().create.assert_called_with(
 832             body=JsonDiffMatcher(expect_container))
 833         self.assertEqual(stubs.capture_stdout.getvalue(),
 834                          stubs.expect_container_request_uuid + '\n')
 835         self.assertEqual(exited, 0)
 836
 837     @stubs
 838     def test_submit_missing_input(self, stubs):
 839         exited = arvados_cwl.main(
 840             ["--submit", "--no-wait", "--api=containers", "--debug",
 841              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 842             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 843         self.assertEqual(exited, 0)
 844
 845         exited = arvados_cwl.main(
 846             ["--submit", "--no-wait", "--api=containers", "--debug",
 847              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 848             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 849         self.assertEqual(exited, 1)
 850
 851     @stubs
 852     def test_submit_container_project(self, stubs):
 853         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 854         exited = arvados_cwl.main(
 855             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 856                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 857             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 858
 859         expect_container = copy.deepcopy(stubs.expect_container_spec)
 860         expect_container["owner_uuid"] = project_uuid
 861         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 862                                        '--no-log-timestamps', '--disable-validate',
 863                                        "--eval-timeout=20", "--thread-count=1",
 864                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 865                                        '--on-error=continue',
 866                                        '--project-uuid='+project_uuid,
 867                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 868
 869         stubs.api.container_requests().create.assert_called_with(
 870             body=JsonDiffMatcher(expect_container))
 871         self.assertEqual(stubs.capture_stdout.getvalue(),
 872                          stubs.expect_container_request_uuid + '\n')
 873         self.assertEqual(exited, 0)
 874
 875     @stubs
 876     def test_submit_container_eval_timeout(self, stubs):
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 879                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 884                                        '--no-log-timestamps', '--disable-validate',
 885                                        '--eval-timeout=60.0', '--thread-count=1',
 886                                        '--enable-reuse', "--collection-cache-size=256",
 887                                        '--debug', '--on-error=continue',
 888                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 889
 890         stubs.api.container_requests().create.assert_called_with(
 891             body=JsonDiffMatcher(expect_container))
 892         self.assertEqual(stubs.capture_stdout.getvalue(),
 893                          stubs.expect_container_request_uuid + '\n')
 894         self.assertEqual(exited, 0)
 895
 896     @stubs
 897     def test_submit_container_collection_cache(self, stubs):
 898         exited = arvados_cwl.main(
 899             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 900                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 901             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 902
 903         expect_container = copy.deepcopy(stubs.expect_container_spec)
 904         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 905                                        '--no-log-timestamps', '--disable-validate',
 906                                        '--eval-timeout=20', '--thread-count=1',
 907                                        '--enable-reuse', "--collection-cache-size=500",
 908                                        '--debug', '--on-error=continue',
 909                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 910         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 911
 912         stubs.api.container_requests().create.assert_called_with(
 913             body=JsonDiffMatcher(expect_container))
 914         self.assertEqual(stubs.capture_stdout.getvalue(),
 915                          stubs.expect_container_request_uuid + '\n')
 916         self.assertEqual(exited, 0)
 917
 918     @stubs
 919     def test_submit_container_thread_count(self, stubs):
 920         exited = arvados_cwl.main(
 921             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 922                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 923             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 924
 925         expect_container = copy.deepcopy(stubs.expect_container_spec)
 926         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 927                                        '--no-log-timestamps', '--disable-validate',
 928                                        '--eval-timeout=20', '--thread-count=20',
 929                                        '--enable-reuse', "--collection-cache-size=256",
 930                                        '--debug', '--on-error=continue',
 931                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 932
 933         stubs.api.container_requests().create.assert_called_with(
 934             body=JsonDiffMatcher(expect_container))
 935         self.assertEqual(stubs.capture_stdout.getvalue(),
 936                          stubs.expect_container_request_uuid + '\n')
 937         self.assertEqual(exited, 0)
 938
 939     @stubs
 940     def test_submit_container_runner_image(self, stubs):
 941         exited = arvados_cwl.main(
 942             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 943                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 944             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 945
 946         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 947
 948         expect_container = copy.deepcopy(stubs.expect_container_spec)
 949         stubs.api.container_requests().create.assert_called_with(
 950             body=JsonDiffMatcher(expect_container))
 951         self.assertEqual(stubs.capture_stdout.getvalue(),
 952                          stubs.expect_container_request_uuid + '\n')
 953         self.assertEqual(exited, 0)
 954
 955     @stubs
 956     def test_submit_priority(self, stubs):
 957         exited = arvados_cwl.main(
 958             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 959                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 960             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 961
 962         stubs.expect_container_spec["priority"] = 669
 963
 964         expect_container = copy.deepcopy(stubs.expect_container_spec)
 965         stubs.api.container_requests().create.assert_called_with(
 966             body=JsonDiffMatcher(expect_container))
 967         self.assertEqual(stubs.capture_stdout.getvalue(),
 968                          stubs.expect_container_request_uuid + '\n')
 969         self.assertEqual(exited, 0)
 970
 971     @stubs
 972     def test_submit_wf_runner_resources(self, stubs):
 973         exited = arvados_cwl.main(
 974             ["--submit", "--no-wait", "--api=containers", "--debug",
 975                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 976             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 977
 978         expect_container = copy.deepcopy(stubs.expect_container_spec)
 979         expect_container["runtime_constraints"] = {
 980             "API": True,
 981             "vcpus": 2,
 982             "ram": (2000+512) * 2**20
 983         }
 984         expect_container["name"] = "submit_wf_runner_resources.cwl"
 985         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 986             {
 987                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 988                 "coresMin": 2,
 989                 "ramMin": 2000,
 990                 "keep_cache": 512
 991             }
 992         ]
 993         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 994             "arv": "http://arvados.org/cwl#",
 995         }
 996         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 997                         '--no-log-timestamps', '--disable-validate',
 998                         '--eval-timeout=20', '--thread-count=1',
 999                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1000                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1001
1002         stubs.api.container_requests().create.assert_called_with(
1003             body=JsonDiffMatcher(expect_container))
1004         self.assertEqual(stubs.capture_stdout.getvalue(),
1005                          stubs.expect_container_request_uuid + '\n')
1006         self.assertEqual(exited, 0)
1007
1008     def tearDown(self):
1009         arvados_cwl.arvdocker.arv_docker_clear_cache()
1010
1011     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1012     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1013     @mock.patch("arvados.api")
1014     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1015         arvados_cwl.arvdocker.arv_docker_clear_cache()
1016
1017         arvrunner = mock.MagicMock()
1018         arvrunner.project_uuid = ""
1019         api.return_value = mock.MagicMock()
1020         arvrunner.api = api.return_value
1021         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1022                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1023                                                                         "link_class": "docker_image_repo+tag",
1024                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1025                                                                         "owner_uuid": "",
1026                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1027                                                             {"items": [{"created_at": "",
1028                                                                         "head_uuid": "",
1029                                                                         "link_class": "docker_image_hash",
1030                                                                         "name": "123456",
1031                                                                         "owner_uuid": "",
1032                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1033         )
1034         find_one_image_hash.return_value = "123456"
1035
1036         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1037                                                                               "owner_uuid": "",
1038                                                                               "manifest_text": "",
1039                                                                               "properties": ""
1040                                                                           }], "items_available": 1, "offset": 0},)
1041         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1042         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1043                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1044         self.assertEqual("9999999999999999999999999999999b+99",
1045                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1046
1047
1048     @stubs
1049     def test_submit_secrets(self, stubs):
1050         exited = arvados_cwl.main(
1051             ["--submit", "--no-wait", "--api=containers", "--debug",
1052                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1053             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1054
1055         expect_container = {
1056             "command": [
1057                 "arvados-cwl-runner",
1058                 "--local",
1059                 "--api=containers",
1060                 "--no-log-timestamps",
1061                 "--disable-validate",
1062                 "--eval-timeout=20",
1063                 '--thread-count=1',
1064                 "--enable-reuse",
1065                 "--collection-cache-size=256",
1066                 '--debug',
1067                 "--on-error=continue",
1068                 "/var/lib/cwl/workflow.json#main",
1069                 "/var/lib/cwl/cwl.input.json"
1070             ],
1071             "container_image": "999999999999999999999999999999d3+99",
1072             "cwd": "/var/spool/cwl",
1073             "mounts": {
1074                 "/var/lib/cwl/cwl.input.json": {
1075                     "content": {
1076                         "pw": {
1077                             "$include": "/secrets/s0"
1078                         }
1079                     },
1080                     "kind": "json"
1081                 },
1082                 "/var/lib/cwl/workflow.json": {
1083                     "content": {
1084                         "$graph": [
1085                             {
1086                                 "$namespaces": {
1087                                     "cwltool": "http://commonwl.org/cwltool#"
1088                                 },
1089                                 "arguments": [
1090                                     "md5sum",
1091                                     "example.conf"
1092                                 ],
1093                                 "class": "CommandLineTool",
1094                                 "hints": [
1095                                     {
1096                                         "class": "http://commonwl.org/cwltool#Secrets",
1097                                         "secrets": [
1098                                             "#secret_job.cwl/pw"
1099                                         ]
1100                                     }
1101                                 ],
1102                                 "id": "#secret_job.cwl",
1103                                 "inputs": [
1104                                     {
1105                                         "id": "#secret_job.cwl/pw",
1106                                         "type": "string"
1107                                     }
1108                                 ],
1109                                 "outputs": [
1110                                     {
1111                                         "id": "#secret_job.cwl/out",
1112                                         "type": "File",
1113                                         "outputBinding": {
1114                                               "glob": "hashed_example.txt"
1115                                         }
1116                                     }
1117                                 ],
1118                                 "stdout": "hashed_example.txt",
1119                                 "requirements": [
1120                                     {
1121                                         "class": "InitialWorkDirRequirement",
1122                                         "listing": [
1123                                             {
1124                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1125                                                 "entryname": "example.conf"
1126                                             }
1127                                         ]
1128                                     }
1129                                 ]
1130                             },
1131                             {
1132                                 "class": "Workflow",
1133                                 "hints": [
1134                                     {
1135                                         "class": "DockerRequirement",
1136                                         "dockerPull": "debian:8",
1137                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1138                                     },
1139                                     {
1140                                         "class": "http://commonwl.org/cwltool#Secrets",
1141                                         "secrets": [
1142                                             "#main/pw"
1143                                         ]
1144                                     }
1145                                 ],
1146                                 "id": "#main",
1147                                 "inputs": [
1148                                     {
1149                                         "id": "#main/pw",
1150                                         "type": "string"
1151                                     }
1152                                 ],
1153                                 "outputs": [
1154                                     {
1155                                         "id": "#main/out",
1156                                         "outputSource": "#main/step1/out",
1157                                         "type": "File"
1158                                     }
1159                                 ],
1160                                 "steps": [
1161                                     {
1162                                         "id": "#main/step1",
1163                                         "in": [
1164                                             {
1165                                                 "id": "#main/step1/pw",
1166                                                 "source": "#main/pw"
1167                                             }
1168                                         ],
1169                                         "out": [
1170                                             "#main/step1/out"
1171                                         ],
1172                                         "run": "#secret_job.cwl"
1173                                     }
1174                                 ]
1175                             }
1176                         ],
1177                         "cwlVersion": "v1.0"
1178                     },
1179                     "kind": "json"
1180                 },
1181                 "/var/spool/cwl": {
1182                     "kind": "collection",
1183                     "writable": True
1184                 },
1185                 "stdout": {
1186                     "kind": "file",
1187                     "path": "/var/spool/cwl/cwl.output.json"
1188                 }
1189             },
1190             "name": "secret_wf.cwl",
1191             "output_path": "/var/spool/cwl",
1192             "priority": 500,
1193             "properties": {},
1194             "runtime_constraints": {
1195                 "API": True,
1196                 "ram": 1342177280,
1197                 "vcpus": 1
1198             },
1199             "secret_mounts": {
1200                 "/secrets/s0": {
1201                     "content": "blorp",
1202                     "kind": "text"
1203                 }
1204             },
1205             "state": "Committed",
1206             "use_existing": False
1207         }
1208
1209         stubs.api.container_requests().create.assert_called_with(
1210             body=JsonDiffMatcher(expect_container))
1211         self.assertEqual(stubs.capture_stdout.getvalue(),
1212                          stubs.expect_container_request_uuid + '\n')
1213         self.assertEqual(exited, 0)
1214
1215     @stubs
1216     def test_submit_request_uuid(self, stubs):
1217         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1218         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1219
1220         stubs.api.container_requests().update().execute.return_value = {
1221             "uuid": stubs.expect_container_request_uuid,
1222             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1223             "state": "Queued"
1224         }
1225
1226         exited = arvados_cwl.main(
1227             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1228                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1229             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1230
1231         stubs.api.container_requests().update.assert_called_with(
1232             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1233         self.assertEqual(stubs.capture_stdout.getvalue(),
1234                          stubs.expect_container_request_uuid + '\n')
1235         self.assertEqual(exited, 0)
1236
1237     @stubs
1238     def test_submit_container_cluster_id(self, stubs):
1239         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1240
1241         exited = arvados_cwl.main(
1242             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1243                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1244             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1245
1246         expect_container = copy.deepcopy(stubs.expect_container_spec)
1247
1248         stubs.api.container_requests().create.assert_called_with(
1249             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1250         self.assertEqual(stubs.capture_stdout.getvalue(),
1251                          stubs.expect_container_request_uuid + '\n')
1252         self.assertEqual(exited, 0)
1253
1254     @stubs
1255     def test_submit_validate_cluster_id(self, stubs):
1256         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1257         exited = arvados_cwl.main(
1258             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1259              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1260             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1261         self.assertEqual(exited, 1)
1262
1263     @mock.patch("arvados.collection.CollectionReader")
1264     @stubs
1265     def test_submit_uuid_inputs(self, stubs, collectionReader):
1266         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1267         def list_side_effect(**kwargs):
1268             m = mock.MagicMock()
1269             if "count" in kwargs:
1270                 m.execute.return_value = {"items": [
1271                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1272                 ]}
1273             else:
1274                 m.execute.return_value = {"items": []}
1275             return m
1276         stubs.api.collections().list.side_effect = list_side_effect
1277
1278         exited = arvados_cwl.main(
1279             ["--submit", "--no-wait", "--api=containers", "--debug",
1280                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1281             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1282
1283         expect_container = copy.deepcopy(stubs.expect_container_spec)
1284         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1285         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1286         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1287
1288         stubs.api.collections().list.assert_has_calls([
1289             mock.call(count='none',
1290                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1291                       select=['uuid', 'portable_data_hash'])])
1292         stubs.api.container_requests().create.assert_called_with(
1293             body=JsonDiffMatcher(expect_container))
1294         self.assertEqual(stubs.capture_stdout.getvalue(),
1295                          stubs.expect_container_request_uuid + '\n')
1296         self.assertEqual(exited, 0)
1297
1298     @stubs
1299     def test_submit_mismatched_uuid_inputs(self, stubs):
1300         def list_side_effect(**kwargs):
1301             m = mock.MagicMock()
1302             if "count" in kwargs:
1303                 m.execute.return_value = {"items": [
1304                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1305                 ]}
1306             else:
1307                 m.execute.return_value = {"items": []}
1308             return m
1309         stubs.api.collections().list.side_effect = list_side_effect
1310
1311         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1312             capture_stderr = StringIO()
1313             cwltool_logger = logging.getLogger('cwltool')
1314             stderr_logger = logging.StreamHandler(capture_stderr)
1315             cwltool_logger.addHandler(stderr_logger)
1316
1317             try:
1318                 exited = arvados_cwl.main(
1319                     ["--submit", "--no-wait", "--api=containers", "--debug",
1320                         "tests/wf/submit_wf.cwl", infile],
1321                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1322
1323                 self.assertEqual(exited, 1)
1324                 self.assertRegex(
1325                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1326                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1327             finally:
1328                 cwltool_logger.removeHandler(stderr_logger)
1329
1330     @mock.patch("arvados.collection.CollectionReader")
1331     @stubs
1332     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1333         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1334         capture_stderr = StringIO()
1335
1336         cwltool_logger = logging.getLogger('cwltool')
1337         stderr_logger = logging.StreamHandler(capture_stderr)
1338         cwltool_logger.addHandler(stderr_logger)
1339
1340         exited = arvados_cwl.main(
1341             ["--submit", "--no-wait", "--api=containers", "--debug",
1342                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1343             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1344
1345         try:
1346             self.assertEqual(exited, 1)
1347             self.assertRegex(
1348                 capture_stderr.getvalue(),
1349                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1350         finally:
1351             cwltool_logger.removeHandler(stderr_logger)
1352
1353
1354 class TestCreateWorkflow(unittest.TestCase):
1355     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1356     expect_workflow = StripYAMLComments(
1357         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1358
1359     @stubs
1360     def test_create(self, stubs):
1361         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1362
1363         exited = arvados_cwl.main(
1364             ["--create-workflow", "--debug",
1365              "--api=containers",
1366              "--project-uuid", project_uuid,
1367              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1368             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1369
1370         stubs.api.pipeline_templates().create.refute_called()
1371         stubs.api.container_requests().create.refute_called()
1372
1373         body = {
1374             "workflow": {
1375                 "owner_uuid": project_uuid,
1376                 "name": "submit_wf.cwl",
1377                 "description": "",
1378                 "definition": self.expect_workflow,
1379             }
1380         }
1381         stubs.api.workflows().create.assert_called_with(
1382             body=JsonDiffMatcher(body))
1383
1384         self.assertEqual(stubs.capture_stdout.getvalue(),
1385                          stubs.expect_workflow_uuid + '\n')
1386         self.assertEqual(exited, 0)
1387
1388     @stubs
1389     def test_create_name(self, stubs):
1390         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1391
1392         exited = arvados_cwl.main(
1393             ["--create-workflow", "--debug",
1394              "--api=containers",
1395              "--project-uuid", project_uuid,
1396              "--name", "testing 123",
1397              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1398             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1399
1400         stubs.api.pipeline_templates().create.refute_called()
1401         stubs.api.container_requests().create.refute_called()
1402
1403         body = {
1404             "workflow": {
1405                 "owner_uuid": project_uuid,
1406                 "name": "testing 123",
1407                 "description": "",
1408                 "definition": self.expect_workflow,
1409             }
1410         }
1411         stubs.api.workflows().create.assert_called_with(
1412             body=JsonDiffMatcher(body))
1413
1414         self.assertEqual(stubs.capture_stdout.getvalue(),
1415                          stubs.expect_workflow_uuid + '\n')
1416         self.assertEqual(exited, 0)
1417
1418
1419     @stubs
1420     def test_update(self, stubs):
1421         exited = arvados_cwl.main(
1422             ["--update-workflow", self.existing_workflow_uuid,
1423              "--debug",
1424              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1425             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1426
1427         body = {
1428             "workflow": {
1429                 "name": "submit_wf.cwl",
1430                 "description": "",
1431                 "definition": self.expect_workflow,
1432             }
1433         }
1434         stubs.api.workflows().update.assert_called_with(
1435             uuid=self.existing_workflow_uuid,
1436             body=JsonDiffMatcher(body))
1437         self.assertEqual(stubs.capture_stdout.getvalue(),
1438                          self.existing_workflow_uuid + '\n')
1439         self.assertEqual(exited, 0)
1440
1441     @stubs
1442     def test_update_name(self, stubs):
1443         exited = arvados_cwl.main(
1444             ["--update-workflow", self.existing_workflow_uuid,
1445              "--debug", "--name", "testing 123",
1446              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1447             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1448
1449         body = {
1450             "workflow": {
1451                 "name": "testing 123",
1452                 "description": "",
1453                 "definition": self.expect_workflow,
1454             }
1455         }
1456         stubs.api.workflows().update.assert_called_with(
1457             uuid=self.existing_workflow_uuid,
1458             body=JsonDiffMatcher(body))
1459         self.assertEqual(stubs.capture_stdout.getvalue(),
1460                          self.existing_workflow_uuid + '\n')
1461         self.assertEqual(exited, 0)
1462
1463     @stubs
1464     def test_create_collection_per_tool(self, stubs):
1465         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1466
1467         exited = arvados_cwl.main(
1468             ["--create-workflow", "--debug",
1469              "--api=containers",
1470              "--project-uuid", project_uuid,
1471              "tests/collection_per_tool/collection_per_tool.cwl"],
1472             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1473
1474         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1475         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1476
1477         body = {
1478             "workflow": {
1479                 "owner_uuid": project_uuid,
1480                 "name": "collection_per_tool.cwl",
1481                 "description": "",
1482                 "definition": expect_workflow,
1483             }
1484         }
1485         stubs.api.workflows().create.assert_called_with(
1486             body=JsonDiffMatcher(body))
1487
1488         self.assertEqual(stubs.capture_stdout.getvalue(),
1489                          stubs.expect_workflow_uuid + '\n')
1490         self.assertEqual(exited, 0)
1491
1492     @stubs
1493     def test_create_with_imports(self, stubs):
1494         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1495
1496         exited = arvados_cwl.main(
1497             ["--create-workflow", "--debug",
1498              "--api=containers",
1499              "--project-uuid", project_uuid,
1500              "tests/wf/feddemo/feddemo.cwl"],
1501             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1502
1503         stubs.api.pipeline_templates().create.refute_called()
1504         stubs.api.container_requests().create.refute_called()
1505
1506         self.assertEqual(stubs.capture_stdout.getvalue(),
1507                          stubs.expect_workflow_uuid + '\n')
1508         self.assertEqual(exited, 0)
1509
1510     @stubs
1511     def test_create_with_no_input(self, stubs):
1512         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1513
1514         exited = arvados_cwl.main(
1515             ["--create-workflow", "--debug",
1516              "--api=containers",
1517              "--project-uuid", project_uuid,
1518              "tests/wf/revsort/revsort.cwl"],
1519             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1520
1521         stubs.api.pipeline_templates().create.refute_called()
1522         stubs.api.container_requests().create.refute_called()
1523
1524         self.assertEqual(stubs.capture_stdout.getvalue(),
1525                          stubs.expect_workflow_uuid + '\n')
1526         self.assertEqual(exited, 0)