sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate',
 306                         '--eval-timeout=20', '--thread-count=1',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343
 344     @stubs
 345     def test_error_when_multiple_storage_classes_specified(self, stubs):
 346         storage_classes = "foo,bar"
 347         exited = arvados_cwl.main(
 348                 ["--debug", "--storage-classes", storage_classes,
 349                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 350                 sys.stdin, sys.stderr, api_client=stubs.api)
 351         self.assertEqual(exited, 1)
 352
 353     @mock.patch("time.sleep")
 354     @stubs
 355     def test_submit_invalid_runner_ram(self, stubs, tm):
 356         exited = arvados_cwl.main(
 357             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 359             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 360         self.assertEqual(exited, 1)
 361
 362
 363     @stubs
 364     def test_submit_container(self, stubs):
 365         exited = arvados_cwl.main(
 366             ["--submit", "--no-wait", "--api=containers", "--debug",
 367                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 368             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 369
 370         stubs.api.collections().create.assert_has_calls([
 371             mock.call(body=JsonDiffMatcher({
 372                 'manifest_text':
 373                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 374                 'replication_desired': None,
 375                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 376             }), ensure_unique_name=False),
 377             mock.call(body=JsonDiffMatcher({
 378                 'manifest_text':
 379                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 380                 'replication_desired': None,
 381                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 382             }), ensure_unique_name=False),
 383             ])
 384
 385         expect_container = copy.deepcopy(stubs.expect_container_spec)
 386         stubs.api.container_requests().create.assert_called_with(
 387             body=JsonDiffMatcher(expect_container))
 388         self.assertEqual(stubs.capture_stdout.getvalue(),
 389                          stubs.expect_container_request_uuid + '\n')
 390         self.assertEqual(exited, 0)
 391
 392
 393     @stubs
 394     def test_submit_container_tool(self, stubs):
 395         # test for issue #16139
 396         exited = arvados_cwl.main(
 397             ["--submit", "--no-wait", "--api=containers", "--debug",
 398                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 399             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 400
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_submit_container_no_reuse(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = [
 414             'arvados-cwl-runner', '--local', '--api=containers',
 415             '--no-log-timestamps', '--disable-validate',
 416             '--eval-timeout=20', '--thread-count=1',
 417             '--disable-reuse', "--collection-cache-size=256",
 418             '--debug', '--on-error=continue',
 419             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420         expect_container["use_existing"] = False
 421
 422         stubs.api.container_requests().create.assert_called_with(
 423             body=JsonDiffMatcher(expect_container))
 424         self.assertEqual(stubs.capture_stdout.getvalue(),
 425                          stubs.expect_container_request_uuid + '\n')
 426         self.assertEqual(exited, 0)
 427
 428     @stubs
 429     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 430         exited = arvados_cwl.main(
 431             ["--submit", "--no-wait", "--api=containers", "--debug",
 432              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 433             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 434         self.assertEqual(exited, 0)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = [
 438             'arvados-cwl-runner', '--local', '--api=containers',
 439             '--no-log-timestamps', '--disable-validate',
 440             '--eval-timeout=20', '--thread-count=1',
 441             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 442             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["use_existing"] = False
 444         expect_container["name"] = "submit_wf_no_reuse.cwl"
 445         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 446             {
 447                 "class": "http://arvados.org/cwl#ReuseRequirement",
 448                 "enableReuse": False,
 449             },
 450         ]
 451         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 452             "arv": "http://arvados.org/cwl#",
 453             "cwltool": "http://commonwl.org/cwltool#"
 454         }
 455
 456         stubs.api.container_requests().create.assert_called_with(
 457             body=JsonDiffMatcher(expect_container))
 458         self.assertEqual(stubs.capture_stdout.getvalue(),
 459                          stubs.expect_container_request_uuid + '\n')
 460
 461
 462     @stubs
 463     def test_submit_container_on_error(self, stubs):
 464         exited = arvados_cwl.main(
 465             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 466                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468
 469         expect_container = copy.deepcopy(stubs.expect_container_spec)
 470         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 471                                        '--no-log-timestamps', '--disable-validate',
 472                                        '--eval-timeout=20', '--thread-count=1',
 473                                        '--enable-reuse', "--collection-cache-size=256",
 474                                        '--debug', '--on-error=stop',
 475                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 476
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @stubs
 484     def test_submit_container_output_name(self, stubs):
 485         output_name = "test_output_name"
 486
 487         exited = arvados_cwl.main(
 488             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 489                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 490             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 491
 492         expect_container = copy.deepcopy(stubs.expect_container_spec)
 493         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 494                                        '--no-log-timestamps', '--disable-validate',
 495                                        '--eval-timeout=20', '--thread-count=1',
 496                                        '--enable-reuse', "--collection-cache-size=256",
 497                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 498                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 499         expect_container["output_name"] = output_name
 500
 501         stubs.api.container_requests().create.assert_called_with(
 502             body=JsonDiffMatcher(expect_container))
 503         self.assertEqual(stubs.capture_stdout.getvalue(),
 504                          stubs.expect_container_request_uuid + '\n')
 505         self.assertEqual(exited, 0)
 506
 507     @stubs
 508     def test_submit_storage_classes(self, stubs):
 509         exited = arvados_cwl.main(
 510             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 511                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 512             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 513
 514         expect_container = copy.deepcopy(stubs.expect_container_spec)
 515         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 516                                        '--no-log-timestamps', '--disable-validate',
 517                                        '--eval-timeout=20', '--thread-count=1',
 518                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 519                                        "--storage-classes=foo", '--on-error=continue',
 520                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 521
 522         stubs.api.container_requests().create.assert_called_with(
 523             body=JsonDiffMatcher(expect_container))
 524         self.assertEqual(stubs.capture_stdout.getvalue(),
 525                          stubs.expect_container_request_uuid + '\n')
 526         self.assertEqual(exited, 0)
 527
 528     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 529     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 530     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 531     @stubs
 532     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 533         def set_final_output(job_order, output_callback, runtimeContext):
 534             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 535             return []
 536         job.side_effect = set_final_output
 537
 538         exited = arvados_cwl.main(
 539             ["--debug", "--local", "--storage-classes=foo",
 540                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 541             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 542
 543         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 544         self.assertEqual(exited, 0)
 545
 546     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 547     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 548     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 549     @stubs
 550     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 551         def set_final_output(job_order, output_callback, runtimeContext):
 552             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 553             return []
 554         job.side_effect = set_final_output
 555
 556         exited = arvados_cwl.main(
 557             ["--debug", "--local",
 558                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 559             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 560
 561         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 562         self.assertEqual(exited, 0)
 563
 564     @stubs
 565     def test_submit_container_output_ttl(self, stubs):
 566         exited = arvados_cwl.main(
 567             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 568                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 569             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 570
 571         expect_container = copy.deepcopy(stubs.expect_container_spec)
 572         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 573                                        '--no-log-timestamps', '--disable-validate',
 574                                        '--eval-timeout=20', '--thread-count=1',
 575                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 576                                        '--on-error=continue',
 577                                        "--intermediate-output-ttl=3600",
 578                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 579
 580         stubs.api.container_requests().create.assert_called_with(
 581             body=JsonDiffMatcher(expect_container))
 582         self.assertEqual(stubs.capture_stdout.getvalue(),
 583                          stubs.expect_container_request_uuid + '\n')
 584         self.assertEqual(exited, 0)
 585
 586     @stubs
 587     def test_submit_container_trash_intermediate(self, stubs):
 588         exited = arvados_cwl.main(
 589             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 590                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 591             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 592
 593
 594         expect_container = copy.deepcopy(stubs.expect_container_spec)
 595         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 596                                        '--no-log-timestamps', '--disable-validate',
 597                                        '--eval-timeout=20', '--thread-count=1',
 598                                        '--enable-reuse', "--collection-cache-size=256",
 599                                        '--debug', '--on-error=continue',
 600                                        "--trash-intermediate",
 601                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 602
 603         stubs.api.container_requests().create.assert_called_with(
 604             body=JsonDiffMatcher(expect_container))
 605         self.assertEqual(stubs.capture_stdout.getvalue(),
 606                          stubs.expect_container_request_uuid + '\n')
 607         self.assertEqual(exited, 0)
 608
 609     @stubs
 610     def test_submit_container_output_tags(self, stubs):
 611         output_tags = "tag0,tag1,tag2"
 612
 613         exited = arvados_cwl.main(
 614             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 615                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 616             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 617
 618         expect_container = copy.deepcopy(stubs.expect_container_spec)
 619         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 620                                        '--no-log-timestamps', '--disable-validate',
 621                                        '--eval-timeout=20', '--thread-count=1',
 622                                        '--enable-reuse', "--collection-cache-size=256",
 623                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 624                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 625
 626         stubs.api.container_requests().create.assert_called_with(
 627             body=JsonDiffMatcher(expect_container))
 628         self.assertEqual(stubs.capture_stdout.getvalue(),
 629                          stubs.expect_container_request_uuid + '\n')
 630         self.assertEqual(exited, 0)
 631
 632     @stubs
 633     def test_submit_container_runner_ram(self, stubs):
 634         exited = arvados_cwl.main(
 635             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 636                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 637             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 638
 639         expect_container = copy.deepcopy(stubs.expect_container_spec)
 640         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 641
 642         stubs.api.container_requests().create.assert_called_with(
 643             body=JsonDiffMatcher(expect_container))
 644         self.assertEqual(stubs.capture_stdout.getvalue(),
 645                          stubs.expect_container_request_uuid + '\n')
 646         self.assertEqual(exited, 0)
 647
 648     @mock.patch("arvados.collection.CollectionReader")
 649     @mock.patch("time.sleep")
 650     @stubs
 651     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 652         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 653         exited = arvados_cwl.main(
 654             ["--submit", "--no-wait", "--api=containers", "--debug",
 655              "tests/wf/submit_keepref_wf.cwl"],
 656             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 657         self.assertEqual(exited, 0)
 658
 659     @mock.patch("arvados.collection.CollectionReader")
 660     @mock.patch("time.sleep")
 661     @stubs
 662     def test_submit_keepref(self, stubs, tm, reader):
 663         with open("tests/wf/expect_arvworkflow.cwl") as f:
 664             reader().open().__enter__().read.return_value = f.read()
 665
 666         exited = arvados_cwl.main(
 667             ["--submit", "--no-wait", "--api=containers", "--debug",
 668              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 669             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 670
 671         expect_container = {
 672             'priority': 500,
 673             'mounts': {
 674                 '/var/spool/cwl': {
 675                     'writable': True,
 676                     'kind': 'collection'
 677                 },
 678                 'stdout': {
 679                     'path': '/var/spool/cwl/cwl.output.json',
 680                     'kind': 'file'
 681                 },
 682                 '/var/lib/cwl/workflow': {
 683                     'portable_data_hash': '99999999999999999999999999999994+99',
 684                     'kind': 'collection'
 685                 },
 686                 '/var/lib/cwl/cwl.input.json': {
 687                     'content': {
 688                         'x': 'XxX'
 689                     },
 690                     'kind': 'json'
 691                 }
 692             }, 'state': 'Committed',
 693             'output_path': '/var/spool/cwl',
 694             'name': 'expect_arvworkflow.cwl#main',
 695             'container_image': '999999999999999999999999999999d3+99',
 696             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 697                         '--no-log-timestamps', '--disable-validate',
 698                         '--eval-timeout=20', '--thread-count=1',
 699                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 700                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 701             'cwd': '/var/spool/cwl',
 702             'runtime_constraints': {
 703                 'API': True,
 704                 'vcpus': 1,
 705                 'ram': 1342177280
 706             },
 707             'use_existing': False,
 708             'properties': {},
 709             'secret_mounts': {}
 710         }
 711
 712         stubs.api.container_requests().create.assert_called_with(
 713             body=JsonDiffMatcher(expect_container))
 714         self.assertEqual(stubs.capture_stdout.getvalue(),
 715                          stubs.expect_container_request_uuid + '\n')
 716         self.assertEqual(exited, 0)
 717
 718     @mock.patch("time.sleep")
 719     @stubs
 720     def test_submit_arvworkflow(self, stubs, tm):
 721         with open("tests/wf/expect_arvworkflow.cwl") as f:
 722             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 723
 724         exited = arvados_cwl.main(
 725             ["--submit", "--no-wait", "--api=containers", "--debug",
 726              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 727             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 728
 729         expect_container = {
 730             'priority': 500,
 731             'mounts': {
 732                 '/var/spool/cwl': {
 733                     'writable': True,
 734                     'kind': 'collection'
 735                 },
 736                 'stdout': {
 737                     'path': '/var/spool/cwl/cwl.output.json',
 738                     'kind': 'file'
 739                 },
 740                 '/var/lib/cwl/workflow.json': {
 741                     'kind': 'json',
 742                     'content': {
 743                         'cwlVersion': 'v1.0',
 744                         '$graph': [
 745                             {
 746                                 'id': '#main',
 747                                 'inputs': [
 748                                     {'type': 'string', 'id': '#main/x'}
 749                                 ],
 750                                 'steps': [
 751                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 752                                      'run': '#submit_tool.cwl',
 753                                      'id': '#main/step1',
 754                                      'out': []}
 755                                 ],
 756                                 'class': 'Workflow',
 757                                 'outputs': []
 758                             },
 759                             {
 760                                 'inputs': [
 761                                     {
 762                                         'inputBinding': {'position': 1},
 763                                         'type': 'string',
 764                                         'id': '#submit_tool.cwl/x'}
 765                                 ],
 766                                 'requirements': [
 767                                     {
 768                                         'dockerPull': 'debian:8',
 769                                         'class': 'DockerRequirement',
 770                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 771                                     }
 772                                 ],
 773                                 'id': '#submit_tool.cwl',
 774                                 'outputs': [],
 775                                 'baseCommand': 'cat',
 776                                 'class': 'CommandLineTool'
 777                             }
 778                         ]
 779                     }
 780                 },
 781                 '/var/lib/cwl/cwl.input.json': {
 782                     'content': {
 783                         'x': 'XxX'
 784                     },
 785                     'kind': 'json'
 786                 }
 787             }, 'state': 'Committed',
 788             'output_path': '/var/spool/cwl',
 789             'name': 'a test workflow',
 790             'container_image': "999999999999999999999999999999d3+99",
 791             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 792                         '--no-log-timestamps', '--disable-validate',
 793                         '--eval-timeout=20', '--thread-count=1',
 794                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 795                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 796             'cwd': '/var/spool/cwl',
 797             'runtime_constraints': {
 798                 'API': True,
 799                 'vcpus': 1,
 800                 'ram': 1342177280
 801             },
 802             'use_existing': False,
 803             'properties': {
 804                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 805             },
 806             'secret_mounts': {}
 807         }
 808
 809         stubs.api.container_requests().create.assert_called_with(
 810             body=JsonDiffMatcher(expect_container))
 811         self.assertEqual(stubs.capture_stdout.getvalue(),
 812                          stubs.expect_container_request_uuid + '\n')
 813         self.assertEqual(exited, 0)
 814
 815     @stubs
 816     def test_submit_container_name(self, stubs):
 817         exited = arvados_cwl.main(
 818             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 819                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 820             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 821
 822         expect_container = copy.deepcopy(stubs.expect_container_spec)
 823         expect_container["name"] = "hello container 123"
 824
 825         stubs.api.container_requests().create.assert_called_with(
 826             body=JsonDiffMatcher(expect_container))
 827         self.assertEqual(stubs.capture_stdout.getvalue(),
 828                          stubs.expect_container_request_uuid + '\n')
 829         self.assertEqual(exited, 0)
 830
 831     @stubs
 832     def test_submit_missing_input(self, stubs):
 833         exited = arvados_cwl.main(
 834             ["--submit", "--no-wait", "--api=containers", "--debug",
 835              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 836             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 837         self.assertEqual(exited, 0)
 838
 839         exited = arvados_cwl.main(
 840             ["--submit", "--no-wait", "--api=containers", "--debug",
 841              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 842             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 843         self.assertEqual(exited, 1)
 844
 845     @stubs
 846     def test_submit_container_project(self, stubs):
 847         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 848         exited = arvados_cwl.main(
 849             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 850                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 851             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 852
 853         expect_container = copy.deepcopy(stubs.expect_container_spec)
 854         expect_container["owner_uuid"] = project_uuid
 855         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 856                                        '--no-log-timestamps', '--disable-validate',
 857                                        "--eval-timeout=20", "--thread-count=1",
 858                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 859                                        '--on-error=continue',
 860                                        '--project-uuid='+project_uuid,
 861                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 862
 863         stubs.api.container_requests().create.assert_called_with(
 864             body=JsonDiffMatcher(expect_container))
 865         self.assertEqual(stubs.capture_stdout.getvalue(),
 866                          stubs.expect_container_request_uuid + '\n')
 867         self.assertEqual(exited, 0)
 868
 869     @stubs
 870     def test_submit_container_eval_timeout(self, stubs):
 871         exited = arvados_cwl.main(
 872             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 873                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 874             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 875
 876         expect_container = copy.deepcopy(stubs.expect_container_spec)
 877         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 878                                        '--no-log-timestamps', '--disable-validate',
 879                                        '--eval-timeout=60.0', '--thread-count=1',
 880                                        '--enable-reuse', "--collection-cache-size=256",
 881                                        '--debug', '--on-error=continue',
 882                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 883
 884         stubs.api.container_requests().create.assert_called_with(
 885             body=JsonDiffMatcher(expect_container))
 886         self.assertEqual(stubs.capture_stdout.getvalue(),
 887                          stubs.expect_container_request_uuid + '\n')
 888         self.assertEqual(exited, 0)
 889
 890     @stubs
 891     def test_submit_container_collection_cache(self, stubs):
 892         exited = arvados_cwl.main(
 893             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 894                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 895             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 896
 897         expect_container = copy.deepcopy(stubs.expect_container_spec)
 898         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 899                                        '--no-log-timestamps', '--disable-validate',
 900                                        '--eval-timeout=20', '--thread-count=1',
 901                                        '--enable-reuse', "--collection-cache-size=500",
 902                                        '--debug', '--on-error=continue',
 903                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 904         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 905
 906         stubs.api.container_requests().create.assert_called_with(
 907             body=JsonDiffMatcher(expect_container))
 908         self.assertEqual(stubs.capture_stdout.getvalue(),
 909                          stubs.expect_container_request_uuid + '\n')
 910         self.assertEqual(exited, 0)
 911
 912     @stubs
 913     def test_submit_container_thread_count(self, stubs):
 914         exited = arvados_cwl.main(
 915             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 916                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 917             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 918
 919         expect_container = copy.deepcopy(stubs.expect_container_spec)
 920         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 921                                        '--no-log-timestamps', '--disable-validate',
 922                                        '--eval-timeout=20', '--thread-count=20',
 923                                        '--enable-reuse', "--collection-cache-size=256",
 924                                        '--debug', '--on-error=continue',
 925                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 926
 927         stubs.api.container_requests().create.assert_called_with(
 928             body=JsonDiffMatcher(expect_container))
 929         self.assertEqual(stubs.capture_stdout.getvalue(),
 930                          stubs.expect_container_request_uuid + '\n')
 931         self.assertEqual(exited, 0)
 932
 933     @stubs
 934     def test_submit_container_runner_image(self, stubs):
 935         exited = arvados_cwl.main(
 936             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 937                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 938             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 939
 940         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 941
 942         expect_container = copy.deepcopy(stubs.expect_container_spec)
 943         stubs.api.container_requests().create.assert_called_with(
 944             body=JsonDiffMatcher(expect_container))
 945         self.assertEqual(stubs.capture_stdout.getvalue(),
 946                          stubs.expect_container_request_uuid + '\n')
 947         self.assertEqual(exited, 0)
 948
 949     @stubs
 950     def test_submit_priority(self, stubs):
 951         exited = arvados_cwl.main(
 952             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 953                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 954             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 955
 956         stubs.expect_container_spec["priority"] = 669
 957
 958         expect_container = copy.deepcopy(stubs.expect_container_spec)
 959         stubs.api.container_requests().create.assert_called_with(
 960             body=JsonDiffMatcher(expect_container))
 961         self.assertEqual(stubs.capture_stdout.getvalue(),
 962                          stubs.expect_container_request_uuid + '\n')
 963         self.assertEqual(exited, 0)
 964
 965     @stubs
 966     def test_submit_wf_runner_resources(self, stubs):
 967         exited = arvados_cwl.main(
 968             ["--submit", "--no-wait", "--api=containers", "--debug",
 969                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 970             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 971
 972         expect_container = copy.deepcopy(stubs.expect_container_spec)
 973         expect_container["runtime_constraints"] = {
 974             "API": True,
 975             "vcpus": 2,
 976             "ram": (2000+512) * 2**20
 977         }
 978         expect_container["name"] = "submit_wf_runner_resources.cwl"
 979         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 980             {
 981                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 982                 "coresMin": 2,
 983                 "ramMin": 2000,
 984                 "keep_cache": 512
 985             }
 986         ]
 987         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 988             "arv": "http://arvados.org/cwl#",
 989         }
 990         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 991                         '--no-log-timestamps', '--disable-validate',
 992                         '--eval-timeout=20', '--thread-count=1',
 993                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
 994                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 995
 996         stubs.api.container_requests().create.assert_called_with(
 997             body=JsonDiffMatcher(expect_container))
 998         self.assertEqual(stubs.capture_stdout.getvalue(),
 999                          stubs.expect_container_request_uuid + '\n')
1000         self.assertEqual(exited, 0)
1001
1002     def tearDown(self):
1003         arvados_cwl.arvdocker.arv_docker_clear_cache()
1004
1005     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1006     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1007     @mock.patch("arvados.api")
1008     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1009         arvados_cwl.arvdocker.arv_docker_clear_cache()
1010
1011         arvrunner = mock.MagicMock()
1012         arvrunner.project_uuid = ""
1013         api.return_value = mock.MagicMock()
1014         arvrunner.api = api.return_value
1015         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1016                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1017                                                                         "link_class": "docker_image_repo+tag",
1018                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1019                                                                         "owner_uuid": "",
1020                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1021                                                             {"items": [{"created_at": "",
1022                                                                         "head_uuid": "",
1023                                                                         "link_class": "docker_image_hash",
1024                                                                         "name": "123456",
1025                                                                         "owner_uuid": "",
1026                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1027         )
1028         find_one_image_hash.return_value = "123456"
1029
1030         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1031                                                                               "owner_uuid": "",
1032                                                                               "manifest_text": "",
1033                                                                               "properties": ""
1034                                                                           }], "items_available": 1, "offset": 0},)
1035         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1036         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1037                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1038         self.assertEqual("9999999999999999999999999999999b+99",
1039                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1040
1041
1042     @stubs
1043     def test_submit_secrets(self, stubs):
1044         exited = arvados_cwl.main(
1045             ["--submit", "--no-wait", "--api=containers", "--debug",
1046                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1047             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1048
1049         expect_container = {
1050             "command": [
1051                 "arvados-cwl-runner",
1052                 "--local",
1053                 "--api=containers",
1054                 "--no-log-timestamps",
1055                 "--disable-validate",
1056                 "--eval-timeout=20",
1057                 '--thread-count=1',
1058                 "--enable-reuse",
1059                 "--collection-cache-size=256",
1060                 '--debug',
1061                 "--on-error=continue",
1062                 "/var/lib/cwl/workflow.json#main",
1063                 "/var/lib/cwl/cwl.input.json"
1064             ],
1065             "container_image": "999999999999999999999999999999d3+99",
1066             "cwd": "/var/spool/cwl",
1067             "mounts": {
1068                 "/var/lib/cwl/cwl.input.json": {
1069                     "content": {
1070                         "pw": {
1071                             "$include": "/secrets/s0"
1072                         }
1073                     },
1074                     "kind": "json"
1075                 },
1076                 "/var/lib/cwl/workflow.json": {
1077                     "content": {
1078                         "$graph": [
1079                             {
1080                                 "$namespaces": {
1081                                     "cwltool": "http://commonwl.org/cwltool#"
1082                                 },
1083                                 "arguments": [
1084                                     "md5sum",
1085                                     "example.conf"
1086                                 ],
1087                                 "class": "CommandLineTool",
1088                                 "hints": [
1089                                     {
1090                                         "class": "http://commonwl.org/cwltool#Secrets",
1091                                         "secrets": [
1092                                             "#secret_job.cwl/pw"
1093                                         ]
1094                                     }
1095                                 ],
1096                                 "id": "#secret_job.cwl",
1097                                 "inputs": [
1098                                     {
1099                                         "id": "#secret_job.cwl/pw",
1100                                         "type": "string"
1101                                     }
1102                                 ],
1103                                 "outputs": [
1104                                     {
1105                                         "id": "#secret_job.cwl/out",
1106                                         "type": "stdout"
1107                                     }
1108                                 ],
1109                                 "stdout": "hashed_example.txt",
1110                                 "requirements": [
1111                                     {
1112                                         "class": "InitialWorkDirRequirement",
1113                                         "listing": [
1114                                             {
1115                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1116                                                 "entryname": "example.conf"
1117                                             }
1118                                         ]
1119                                     }
1120                                 ]
1121                             },
1122                             {
1123                                 "class": "Workflow",
1124                                 "hints": [
1125                                     {
1126                                         "class": "DockerRequirement",
1127                                         "dockerPull": "debian:8",
1128                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1129                                     },
1130                                     {
1131                                         "class": "http://commonwl.org/cwltool#Secrets",
1132                                         "secrets": [
1133                                             "#main/pw"
1134                                         ]
1135                                     }
1136                                 ],
1137                                 "id": "#main",
1138                                 "inputs": [
1139                                     {
1140                                         "id": "#main/pw",
1141                                         "type": "string"
1142                                     }
1143                                 ],
1144                                 "outputs": [
1145                                     {
1146                                         "id": "#main/out",
1147                                         "outputSource": "#main/step1/out",
1148                                         "type": "File"
1149                                     }
1150                                 ],
1151                                 "steps": [
1152                                     {
1153                                         "id": "#main/step1",
1154                                         "in": [
1155                                             {
1156                                                 "id": "#main/step1/pw",
1157                                                 "source": "#main/pw"
1158                                             }
1159                                         ],
1160                                         "out": [
1161                                             "#main/step1/out"
1162                                         ],
1163                                         "run": "#secret_job.cwl"
1164                                     }
1165                                 ]
1166                             }
1167                         ],
1168                         "cwlVersion": "v1.0"
1169                     },
1170                     "kind": "json"
1171                 },
1172                 "/var/spool/cwl": {
1173                     "kind": "collection",
1174                     "writable": True
1175                 },
1176                 "stdout": {
1177                     "kind": "file",
1178                     "path": "/var/spool/cwl/cwl.output.json"
1179                 }
1180             },
1181             "name": "secret_wf.cwl",
1182             "output_path": "/var/spool/cwl",
1183             "priority": 500,
1184             "properties": {},
1185             "runtime_constraints": {
1186                 "API": True,
1187                 "ram": 1342177280,
1188                 "vcpus": 1
1189             },
1190             "secret_mounts": {
1191                 "/secrets/s0": {
1192                     "content": "blorp",
1193                     "kind": "text"
1194                 }
1195             },
1196             "state": "Committed",
1197             "use_existing": False
1198         }
1199
1200         stubs.api.container_requests().create.assert_called_with(
1201             body=JsonDiffMatcher(expect_container))
1202         self.assertEqual(stubs.capture_stdout.getvalue(),
1203                          stubs.expect_container_request_uuid + '\n')
1204         self.assertEqual(exited, 0)
1205
1206     @stubs
1207     def test_submit_request_uuid(self, stubs):
1208         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1209         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1210
1211         stubs.api.container_requests().update().execute.return_value = {
1212             "uuid": stubs.expect_container_request_uuid,
1213             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1214             "state": "Queued"
1215         }
1216
1217         exited = arvados_cwl.main(
1218             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1219                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1220             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1221
1222         stubs.api.container_requests().update.assert_called_with(
1223             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1224         self.assertEqual(stubs.capture_stdout.getvalue(),
1225                          stubs.expect_container_request_uuid + '\n')
1226         self.assertEqual(exited, 0)
1227
1228     @stubs
1229     def test_submit_container_cluster_id(self, stubs):
1230         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1231
1232         exited = arvados_cwl.main(
1233             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1234                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1235             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1236
1237         expect_container = copy.deepcopy(stubs.expect_container_spec)
1238
1239         stubs.api.container_requests().create.assert_called_with(
1240             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1241         self.assertEqual(stubs.capture_stdout.getvalue(),
1242                          stubs.expect_container_request_uuid + '\n')
1243         self.assertEqual(exited, 0)
1244
1245     @stubs
1246     def test_submit_validate_cluster_id(self, stubs):
1247         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1248         exited = arvados_cwl.main(
1249             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1250              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1251             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1252         self.assertEqual(exited, 1)
1253
1254     @mock.patch("arvados.collection.CollectionReader")
1255     @stubs
1256     def test_submit_uuid_inputs(self, stubs, collectionReader):
1257         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1258         def list_side_effect(**kwargs):
1259             m = mock.MagicMock()
1260             if "count" in kwargs:
1261                 m.execute.return_value = {"items": [
1262                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1263                 ]}
1264             else:
1265                 m.execute.return_value = {"items": []}
1266             return m
1267         stubs.api.collections().list.side_effect = list_side_effect
1268
1269         exited = arvados_cwl.main(
1270             ["--submit", "--no-wait", "--api=containers", "--debug",
1271                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1272             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1273
1274         expect_container = copy.deepcopy(stubs.expect_container_spec)
1275         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1276         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1277         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1278
1279         stubs.api.collections().list.assert_has_calls([
1280             mock.call(count='none',
1281                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1282                       select=['uuid', 'portable_data_hash'])])
1283         stubs.api.container_requests().create.assert_called_with(
1284             body=JsonDiffMatcher(expect_container))
1285         self.assertEqual(stubs.capture_stdout.getvalue(),
1286                          stubs.expect_container_request_uuid + '\n')
1287         self.assertEqual(exited, 0)
1288
1289     @stubs
1290     def test_submit_mismatched_uuid_inputs(self, stubs):
1291         def list_side_effect(**kwargs):
1292             m = mock.MagicMock()
1293             if "count" in kwargs:
1294                 m.execute.return_value = {"items": [
1295                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1296                 ]}
1297             else:
1298                 m.execute.return_value = {"items": []}
1299             return m
1300         stubs.api.collections().list.side_effect = list_side_effect
1301
1302         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1303             capture_stderr = StringIO()
1304             cwltool_logger = logging.getLogger('cwltool')
1305             stderr_logger = logging.StreamHandler(capture_stderr)
1306             cwltool_logger.addHandler(stderr_logger)
1307
1308             try:
1309                 exited = arvados_cwl.main(
1310                     ["--submit", "--no-wait", "--api=containers", "--debug",
1311                         "tests/wf/submit_wf.cwl", infile],
1312                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1313
1314                 self.assertEqual(exited, 1)
1315                 self.assertRegexpMatches(
1316                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1317                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1318             finally:
1319                 cwltool_logger.removeHandler(stderr_logger)
1320
1321     @mock.patch("arvados.collection.CollectionReader")
1322     @stubs
1323     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1324         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1325         capture_stderr = StringIO()
1326
1327         cwltool_logger = logging.getLogger('cwltool')
1328         stderr_logger = logging.StreamHandler(capture_stderr)
1329         cwltool_logger.addHandler(stderr_logger)
1330
1331         exited = arvados_cwl.main(
1332             ["--submit", "--no-wait", "--api=containers", "--debug",
1333                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1334             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1335
1336         try:
1337             self.assertEqual(exited, 1)
1338             self.assertRegexpMatches(
1339                 capture_stderr.getvalue(),
1340                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1341         finally:
1342             cwltool_logger.removeHandler(stderr_logger)
1343
1344
1345 class TestCreateWorkflow(unittest.TestCase):
1346     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1347     expect_workflow = StripYAMLComments(
1348         open("tests/wf/expect_packed.cwl").read())
1349
1350     @stubs
1351     def test_create(self, stubs):
1352         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1353
1354         exited = arvados_cwl.main(
1355             ["--create-workflow", "--debug",
1356              "--api=containers",
1357              "--project-uuid", project_uuid,
1358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1359             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1360
1361         stubs.api.pipeline_templates().create.refute_called()
1362         stubs.api.container_requests().create.refute_called()
1363
1364         body = {
1365             "workflow": {
1366                 "owner_uuid": project_uuid,
1367                 "name": "submit_wf.cwl",
1368                 "description": "",
1369                 "definition": self.expect_workflow,
1370             }
1371         }
1372         stubs.api.workflows().create.assert_called_with(
1373             body=JsonDiffMatcher(body))
1374
1375         self.assertEqual(stubs.capture_stdout.getvalue(),
1376                          stubs.expect_workflow_uuid + '\n')
1377         self.assertEqual(exited, 0)
1378
1379     @stubs
1380     def test_create_name(self, stubs):
1381         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1382
1383         exited = arvados_cwl.main(
1384             ["--create-workflow", "--debug",
1385              "--api=containers",
1386              "--project-uuid", project_uuid,
1387              "--name", "testing 123",
1388              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1389             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1390
1391         stubs.api.pipeline_templates().create.refute_called()
1392         stubs.api.container_requests().create.refute_called()
1393
1394         body = {
1395             "workflow": {
1396                 "owner_uuid": project_uuid,
1397                 "name": "testing 123",
1398                 "description": "",
1399                 "definition": self.expect_workflow,
1400             }
1401         }
1402         stubs.api.workflows().create.assert_called_with(
1403             body=JsonDiffMatcher(body))
1404
1405         self.assertEqual(stubs.capture_stdout.getvalue(),
1406                          stubs.expect_workflow_uuid + '\n')
1407         self.assertEqual(exited, 0)
1408
1409
1410     @stubs
1411     def test_update(self, stubs):
1412         exited = arvados_cwl.main(
1413             ["--update-workflow", self.existing_workflow_uuid,
1414              "--debug",
1415              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1416             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1417
1418         body = {
1419             "workflow": {
1420                 "name": "submit_wf.cwl",
1421                 "description": "",
1422                 "definition": self.expect_workflow,
1423             }
1424         }
1425         stubs.api.workflows().update.assert_called_with(
1426             uuid=self.existing_workflow_uuid,
1427             body=JsonDiffMatcher(body))
1428         self.assertEqual(stubs.capture_stdout.getvalue(),
1429                          self.existing_workflow_uuid + '\n')
1430         self.assertEqual(exited, 0)
1431
1432     @stubs
1433     def test_update_name(self, stubs):
1434         exited = arvados_cwl.main(
1435             ["--update-workflow", self.existing_workflow_uuid,
1436              "--debug", "--name", "testing 123",
1437              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1438             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1439
1440         body = {
1441             "workflow": {
1442                 "name": "testing 123",
1443                 "description": "",
1444                 "definition": self.expect_workflow,
1445             }
1446         }
1447         stubs.api.workflows().update.assert_called_with(
1448             uuid=self.existing_workflow_uuid,
1449             body=JsonDiffMatcher(body))
1450         self.assertEqual(stubs.capture_stdout.getvalue(),
1451                          self.existing_workflow_uuid + '\n')
1452         self.assertEqual(exited, 0)
1453
1454     @stubs
1455     def test_create_collection_per_tool(self, stubs):
1456         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1457
1458         exited = arvados_cwl.main(
1459             ["--create-workflow", "--debug",
1460              "--api=containers",
1461              "--project-uuid", project_uuid,
1462              "tests/collection_per_tool/collection_per_tool.cwl"],
1463             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1464
1465         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1466         expect_workflow = StripYAMLComments(open(toolfile).read())
1467
1468         body = {
1469             "workflow": {
1470                 "owner_uuid": project_uuid,
1471                 "name": "collection_per_tool.cwl",
1472                 "description": "",
1473                 "definition": expect_workflow,
1474             }
1475         }
1476         stubs.api.workflows().create.assert_called_with(
1477             body=JsonDiffMatcher(body))
1478
1479         self.assertEqual(stubs.capture_stdout.getvalue(),
1480                          stubs.expect_workflow_uuid + '\n')
1481         self.assertEqual(exited, 0)
1482
1483     @stubs
1484     def test_create_with_imports(self, stubs):
1485         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1486
1487         exited = arvados_cwl.main(
1488             ["--create-workflow", "--debug",
1489              "--api=containers",
1490              "--project-uuid", project_uuid,
1491              "tests/wf/feddemo/feddemo.cwl"],
1492             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1493
1494         stubs.api.pipeline_templates().create.refute_called()
1495         stubs.api.container_requests().create.refute_called()
1496
1497         self.assertEqual(stubs.capture_stdout.getvalue(),
1498                          stubs.expect_workflow_uuid + '\n')
1499         self.assertEqual(exited, 0)
1500
1501     @stubs
1502     def test_create_with_no_input(self, stubs):
1503         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1504
1505         exited = arvados_cwl.main(
1506             ["--create-workflow", "--debug",
1507              "--api=containers",
1508              "--project-uuid", project_uuid,
1509              "tests/wf/revsort/revsort.cwl"],
1510             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1511
1512         stubs.api.pipeline_templates().create.refute_called()
1513         stubs.api.container_requests().create.refute_called()
1514
1515         self.assertEqual(stubs.capture_stdout.getvalue(),
1516                          stubs.expect_workflow_uuid + '\n')
1517         self.assertEqual(exited, 0)