sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21
  22 from io import BytesIO
  23
  24 # StringIO.StringIO and io.StringIO have different behavior write() is
  25 # called with both python2 (byte) strings and unicode strings
  26 # (specifically there's some logging in cwltool that causes trouble).
  27 # This isn't a problem on python3 because all string are unicode.
  28 if sys.version_info[0] < 3:
  29     from StringIO import StringIO
  30 else:
  31     from io import StringIO
  32
  33 import arvados
  34 import arvados.collection
  35 import arvados_cwl
  36 import arvados_cwl.executor
  37 import arvados_cwl.runner
  38 import arvados.keep
  39
  40 from .matcher import JsonDiffMatcher, StripYAMLComments
  41 from .mock_discovery import get_rootDesc
  42
  43 import ruamel.yaml as yaml
  44
  45 _rootDesc = None
  46
  47 def stubs(func):
  48     @functools.wraps(func)
  49     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  50     @mock.patch("arvados.collection.KeepClient")
  51     @mock.patch("arvados.keep.KeepClient")
  52     @mock.patch("arvados.events.subscribe")
  53     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  54         class Stubs(object):
  55             pass
  56         stubs = Stubs()
  57         stubs.events = events
  58         stubs.keepdocker = keepdocker
  59
  60         def putstub(p, **kwargs):
  61             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  62         keep_client1().put.side_effect = putstub
  63         keep_client1.put.side_effect = putstub
  64         keep_client2().put.side_effect = putstub
  65         keep_client2.put.side_effect = putstub
  66
  67         stubs.keep_client = keep_client2
  68         stubs.docker_images = {
  69             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  70             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  71             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  72             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  73         }
  74         def kd(a, b, image_name=None, image_tag=None):
  75             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  76         stubs.keepdocker.side_effect = kd
  77
  78         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  79         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  80
  81         if sys.version_info[0] < 3:
  82             stubs.capture_stdout = BytesIO()
  83         else:
  84             stubs.capture_stdout = StringIO()
  85
  86         stubs.api = mock.MagicMock()
  87         stubs.api._rootDesc = get_rootDesc()
  88         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  89
  90         stubs.api.users().current().execute.return_value = {
  91             "uuid": stubs.fake_user_uuid,
  92         }
  93         stubs.api.collections().list().execute.return_value = {"items": []}
  94         stubs.api.containers().current().execute.return_value = {
  95             "uuid": stubs.fake_container_uuid,
  96         }
  97
  98         class CollectionExecute(object):
  99             def __init__(self, exe):
 100                 self.exe = exe
 101             def execute(self, num_retries=None):
 102                 return self.exe
 103
 104         def collection_createstub(created_collections, body, ensure_unique_name=None):
 105             mt = body["manifest_text"].encode('utf-8')
 106             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 107             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 108             created_collections[uuid] = {
 109                 "uuid": uuid,
 110                 "portable_data_hash": pdh,
 111                 "manifest_text": mt.decode('utf-8')
 112             }
 113             return CollectionExecute(created_collections[uuid])
 114
 115         def collection_getstub(created_collections, uuid):
 116             for v in viewvalues(created_collections):
 117                 if uuid in (v["uuid"], v["portable_data_hash"]):
 118                     return CollectionExecute(v)
 119
 120         created_collections = {
 121             "99999999999999999999999999999998+99": {
 122                 "uuid": "",
 123                 "portable_data_hash": "99999999999999999999999999999998+99",
 124                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 125             },
 126             "99999999999999999999999999999997+99": {
 127                 "uuid": "",
 128                 "portable_data_hash": "99999999999999999999999999999997+99",
 129                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 130             },
 131             "99999999999999999999999999999994+99": {
 132                 "uuid": "",
 133                 "portable_data_hash": "99999999999999999999999999999994+99",
 134                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 135             },
 136             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 137                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 138                 "portable_data_hash": "999999999999999999999999999999d3+99",
 139                 "manifest_text": ""
 140             },
 141             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 142                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 143                 "portable_data_hash": "999999999999999999999999999999d4+99",
 144                 "manifest_text": ""
 145             },
 146             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 147                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 148                 "portable_data_hash": "999999999999999999999999999999d5+99",
 149                 "manifest_text": ""
 150             },
 151             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 152                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 153                 "portable_data_hash": "999999999999999999999999999999d6+99",
 154                 "manifest_text": ""
 155             }
 156         }
 157         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 158         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 159
 160         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 161         stubs.api.jobs().create().execute.return_value = {
 162             "uuid": stubs.expect_job_uuid,
 163             "state": "Queued",
 164         }
 165
 166         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 167         stubs.api.container_requests().create().execute.return_value = {
 168             "uuid": stubs.expect_container_request_uuid,
 169             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 170             "state": "Queued"
 171         }
 172
 173         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 174         stubs.api.pipeline_templates().create().execute.return_value = {
 175             "uuid": stubs.expect_pipeline_template_uuid,
 176         }
 177         stubs.expect_job_spec = {
 178             'runtime_constraints': {
 179                 'docker_image': '999999999999999999999999999999d3+99',
 180                 'min_ram_mb_per_node': 1024
 181             },
 182             'script_parameters': {
 183                 'x': {
 184                     'basename': 'blorp.txt',
 185                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 186                     'class': 'File'
 187                 },
 188                 'y': {
 189                     'basename': '99999999999999999999999999999998+99',
 190                     'location': 'keep:99999999999999999999999999999998+99',
 191                     'class': 'Directory'
 192                 },
 193                 'z': {
 194                     'basename': 'anonymous',
 195                     "listing": [{
 196                         "basename": "renamed.txt",
 197                         "class": "File",
 198                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 199                         "size": 0
 200                     }],
 201                     'class': 'Directory'
 202                 },
 203                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 204             },
 205             'repository': 'arvados',
 206             'script_version': 'master',
 207             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 208             'script': 'cwl-runner'
 209         }
 210         stubs.pipeline_component = stubs.expect_job_spec.copy()
 211         stubs.expect_pipeline_instance = {
 212             'name': 'submit_wf.cwl',
 213             'state': 'RunningOnServer',
 214             'owner_uuid': None,
 215             "components": {
 216                 "cwl-runner": {
 217                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 218                     'script_parameters': {
 219                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 220                         'x': {"value": {
 221                             'basename': 'blorp.txt',
 222                             'class': 'File',
 223                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 224                             "size": 16
 225                         }},
 226                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 227                               'listing': [
 228                                   {
 229                                       'basename': 'renamed.txt',
 230                                       'class': 'File', 'location':
 231                                       'keep:99999999999999999999999999999998+99/file1.txt',
 232                                       'size': 0
 233                                   }
 234                               ]}},
 235                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 236                         'arv:debug': True,
 237                         'arv:enable_reuse': True,
 238                         'arv:on_error': 'continue'
 239                     },
 240                     'repository': 'arvados',
 241                     'script_version': 'master',
 242                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 243                     'script': 'cwl-runner',
 244                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 245                 }
 246             }
 247         }
 248         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 249         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 250         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 251         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 252         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 253             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 254             "state": "Queued"
 255         }
 256         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 257         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 258
 259         with open("tests/wf/submit_wf_packed.cwl") as f:
 260             expect_packed_workflow = yaml.round_trip_load(f)
 261
 262         stubs.expect_container_spec = {
 263             'priority': 500,
 264             'mounts': {
 265                 '/var/spool/cwl': {
 266                     'writable': True,
 267                     'kind': 'collection'
 268                 },
 269                 '/var/lib/cwl/workflow.json': {
 270                     'content': expect_packed_workflow,
 271                     'kind': 'json'
 272                 },
 273                 'stdout': {
 274                     'path': '/var/spool/cwl/cwl.output.json',
 275                     'kind': 'file'
 276                 },
 277                 '/var/lib/cwl/cwl.input.json': {
 278                     'kind': 'json',
 279                     'content': {
 280                         'y': {
 281                             'basename': '99999999999999999999999999999998+99',
 282                             'location': 'keep:99999999999999999999999999999998+99',
 283                             'class': 'Directory'},
 284                         'x': {
 285                             'basename': u'blorp.txt',
 286                             'class': 'File',
 287                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 288                             "size": 16
 289                         },
 290                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 291                             {'basename': 'renamed.txt',
 292                              'class': 'File',
 293                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 294                              'size': 0
 295                             }
 296                         ]}
 297                     },
 298                     'kind': 'json'
 299                 }
 300             },
 301             'secret_mounts': {},
 302             'state': 'Committed',
 303             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 304                         '--no-log-timestamps', '--disable-validate',
 305                         '--eval-timeout=20', '--thread-count=1',
 306                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 307                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 308             'name': 'submit_wf.cwl',
 309             'container_image': '999999999999999999999999999999d3+99',
 310             'output_path': '/var/spool/cwl',
 311             'cwd': '/var/spool/cwl',
 312             'runtime_constraints': {
 313                 'API': True,
 314                 'vcpus': 1,
 315                 'ram': (1024+256)*1024*1024
 316             },
 317             'use_existing': False,
 318             'properties': {},
 319             'secret_mounts': {}
 320         }
 321
 322         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 323         stubs.api.workflows().create().execute.return_value = {
 324             "uuid": stubs.expect_workflow_uuid,
 325         }
 326         def update_mock(**kwargs):
 327             stubs.updated_uuid = kwargs.get('uuid')
 328             return mock.DEFAULT
 329         stubs.api.workflows().update.side_effect = update_mock
 330         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 331             "uuid": stubs.updated_uuid,
 332         }
 333
 334         return func(self, stubs, *args, **kwargs)
 335     return wrapped
 336
 337
 338 class TestSubmit(unittest.TestCase):
 339
 340     def setUp(self):
 341         cwltool.process._names = set()
 342
 343     @stubs
 344     def test_error_when_multiple_storage_classes_specified(self, stubs):
 345         storage_classes = "foo,bar"
 346         exited = arvados_cwl.main(
 347                 ["--debug", "--storage-classes", storage_classes,
 348                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 349                 sys.stdin, sys.stderr, api_client=stubs.api)
 350         self.assertEqual(exited, 1)
 351
 352     @mock.patch("time.sleep")
 353     @stubs
 354     def test_submit_invalid_runner_ram(self, stubs, tm):
 355         exited = arvados_cwl.main(
 356             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 358             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 359         self.assertEqual(exited, 1)
 360
 361
 362     @stubs
 363     def test_submit_container(self, stubs):
 364         exited = arvados_cwl.main(
 365             ["--submit", "--no-wait", "--api=containers", "--debug",
 366                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 367             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 368
 369         stubs.api.collections().create.assert_has_calls([
 370             mock.call(body=JsonDiffMatcher({
 371                 'manifest_text':
 372                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 373                 'replication_desired': None,
 374                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 375             }), ensure_unique_name=False),
 376             mock.call(body=JsonDiffMatcher({
 377                 'manifest_text':
 378                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 379                 'replication_desired': None,
 380                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 381             }), ensure_unique_name=False),
 382             ])
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         stubs.api.container_requests().create.assert_called_with(
 386             body=JsonDiffMatcher(expect_container))
 387         self.assertEqual(stubs.capture_stdout.getvalue(),
 388                          stubs.expect_container_request_uuid + '\n')
 389         self.assertEqual(exited, 0)
 390
 391
 392     @stubs
 393     def test_submit_container_tool(self, stubs):
 394         # test for issue #16139
 395         exited = arvados_cwl.main(
 396             ["--submit", "--no-wait", "--api=containers", "--debug",
 397                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 398             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 399
 400         self.assertEqual(stubs.capture_stdout.getvalue(),
 401                          stubs.expect_container_request_uuid + '\n')
 402         self.assertEqual(exited, 0)
 403
 404     @stubs
 405     def test_submit_container_no_reuse(self, stubs):
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 408                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 410
 411         expect_container = copy.deepcopy(stubs.expect_container_spec)
 412         expect_container["command"] = [
 413             'arvados-cwl-runner', '--local', '--api=containers',
 414             '--no-log-timestamps', '--disable-validate',
 415             '--eval-timeout=20', '--thread-count=1',
 416             '--disable-reuse', "--collection-cache-size=256",
 417             '--debug', '--on-error=continue',
 418             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 419         expect_container["use_existing"] = False
 420
 421         stubs.api.container_requests().create.assert_called_with(
 422             body=JsonDiffMatcher(expect_container))
 423         self.assertEqual(stubs.capture_stdout.getvalue(),
 424                          stubs.expect_container_request_uuid + '\n')
 425         self.assertEqual(exited, 0)
 426
 427     @stubs
 428     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 429         exited = arvados_cwl.main(
 430             ["--submit", "--no-wait", "--api=containers", "--debug",
 431              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 432             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 433         self.assertEqual(exited, 0)
 434
 435         expect_container = copy.deepcopy(stubs.expect_container_spec)
 436         expect_container["command"] = [
 437             'arvados-cwl-runner', '--local', '--api=containers',
 438             '--no-log-timestamps', '--disable-validate',
 439             '--eval-timeout=20', '--thread-count=1',
 440             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 441             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 442         expect_container["use_existing"] = False
 443         expect_container["name"] = "submit_wf_no_reuse.cwl"
 444         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 445             {
 446                 "class": "http://arvados.org/cwl#ReuseRequirement",
 447                 "enableReuse": False,
 448             },
 449         ]
 450         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 451             "arv": "http://arvados.org/cwl#",
 452             "cwltool": "http://commonwl.org/cwltool#"
 453         }
 454
 455         stubs.api.container_requests().create.assert_called_with(
 456             body=JsonDiffMatcher(expect_container))
 457         self.assertEqual(stubs.capture_stdout.getvalue(),
 458                          stubs.expect_container_request_uuid + '\n')
 459
 460
 461     @stubs
 462     def test_submit_container_on_error(self, stubs):
 463         exited = arvados_cwl.main(
 464             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 465                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 466             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 467
 468         expect_container = copy.deepcopy(stubs.expect_container_spec)
 469         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 470                                        '--no-log-timestamps', '--disable-validate',
 471                                        '--eval-timeout=20', '--thread-count=1',
 472                                        '--enable-reuse', "--collection-cache-size=256",
 473                                        '--debug', '--on-error=stop',
 474                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 475
 476         stubs.api.container_requests().create.assert_called_with(
 477             body=JsonDiffMatcher(expect_container))
 478         self.assertEqual(stubs.capture_stdout.getvalue(),
 479                          stubs.expect_container_request_uuid + '\n')
 480         self.assertEqual(exited, 0)
 481
 482     @stubs
 483     def test_submit_container_output_name(self, stubs):
 484         output_name = "test_output_name"
 485
 486         exited = arvados_cwl.main(
 487             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 488                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 489             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 490
 491         expect_container = copy.deepcopy(stubs.expect_container_spec)
 492         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 493                                        '--no-log-timestamps', '--disable-validate',
 494                                        '--eval-timeout=20', '--thread-count=1',
 495                                        '--enable-reuse', "--collection-cache-size=256",
 496                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 497                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 498         expect_container["output_name"] = output_name
 499
 500         stubs.api.container_requests().create.assert_called_with(
 501             body=JsonDiffMatcher(expect_container))
 502         self.assertEqual(stubs.capture_stdout.getvalue(),
 503                          stubs.expect_container_request_uuid + '\n')
 504         self.assertEqual(exited, 0)
 505
 506     @stubs
 507     def test_submit_storage_classes(self, stubs):
 508         exited = arvados_cwl.main(
 509             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 510                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 511             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 512
 513         expect_container = copy.deepcopy(stubs.expect_container_spec)
 514         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 515                                        '--no-log-timestamps', '--disable-validate',
 516                                        '--eval-timeout=20', '--thread-count=1',
 517                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 518                                        "--storage-classes=foo", '--on-error=continue',
 519                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 520
 521         stubs.api.container_requests().create.assert_called_with(
 522             body=JsonDiffMatcher(expect_container))
 523         self.assertEqual(stubs.capture_stdout.getvalue(),
 524                          stubs.expect_container_request_uuid + '\n')
 525         self.assertEqual(exited, 0)
 526
 527     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 528     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 529     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 530     @stubs
 531     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 532         def set_final_output(job_order, output_callback, runtimeContext):
 533             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 534             return []
 535         job.side_effect = set_final_output
 536
 537         exited = arvados_cwl.main(
 538             ["--debug", "--local", "--storage-classes=foo",
 539                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 540             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 541
 542         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 543         self.assertEqual(exited, 0)
 544
 545     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 546     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 547     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 548     @stubs
 549     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 550         def set_final_output(job_order, output_callback, runtimeContext):
 551             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 552             return []
 553         job.side_effect = set_final_output
 554
 555         exited = arvados_cwl.main(
 556             ["--debug", "--local",
 557                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 558             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 559
 560         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 561         self.assertEqual(exited, 0)
 562
 563     @stubs
 564     def test_submit_container_output_ttl(self, stubs):
 565         exited = arvados_cwl.main(
 566             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 567                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 568             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 569
 570         expect_container = copy.deepcopy(stubs.expect_container_spec)
 571         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 572                                        '--no-log-timestamps', '--disable-validate',
 573                                        '--eval-timeout=20', '--thread-count=1',
 574                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 575                                        '--on-error=continue',
 576                                        "--intermediate-output-ttl=3600",
 577                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 578
 579         stubs.api.container_requests().create.assert_called_with(
 580             body=JsonDiffMatcher(expect_container))
 581         self.assertEqual(stubs.capture_stdout.getvalue(),
 582                          stubs.expect_container_request_uuid + '\n')
 583         self.assertEqual(exited, 0)
 584
 585     @stubs
 586     def test_submit_container_trash_intermediate(self, stubs):
 587         exited = arvados_cwl.main(
 588             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 589                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 590             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 591
 592
 593         expect_container = copy.deepcopy(stubs.expect_container_spec)
 594         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 595                                        '--no-log-timestamps', '--disable-validate',
 596                                        '--eval-timeout=20', '--thread-count=1',
 597                                        '--enable-reuse', "--collection-cache-size=256",
 598                                        '--debug', '--on-error=continue',
 599                                        "--trash-intermediate",
 600                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 601
 602         stubs.api.container_requests().create.assert_called_with(
 603             body=JsonDiffMatcher(expect_container))
 604         self.assertEqual(stubs.capture_stdout.getvalue(),
 605                          stubs.expect_container_request_uuid + '\n')
 606         self.assertEqual(exited, 0)
 607
 608     @stubs
 609     def test_submit_container_output_tags(self, stubs):
 610         output_tags = "tag0,tag1,tag2"
 611
 612         exited = arvados_cwl.main(
 613             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 614                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 615             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 616
 617         expect_container = copy.deepcopy(stubs.expect_container_spec)
 618         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 619                                        '--no-log-timestamps', '--disable-validate',
 620                                        '--eval-timeout=20', '--thread-count=1',
 621                                        '--enable-reuse', "--collection-cache-size=256",
 622                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 623                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 624
 625         stubs.api.container_requests().create.assert_called_with(
 626             body=JsonDiffMatcher(expect_container))
 627         self.assertEqual(stubs.capture_stdout.getvalue(),
 628                          stubs.expect_container_request_uuid + '\n')
 629         self.assertEqual(exited, 0)
 630
 631     @stubs
 632     def test_submit_container_runner_ram(self, stubs):
 633         exited = arvados_cwl.main(
 634             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 635                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 636             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 637
 638         expect_container = copy.deepcopy(stubs.expect_container_spec)
 639         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 640
 641         stubs.api.container_requests().create.assert_called_with(
 642             body=JsonDiffMatcher(expect_container))
 643         self.assertEqual(stubs.capture_stdout.getvalue(),
 644                          stubs.expect_container_request_uuid + '\n')
 645         self.assertEqual(exited, 0)
 646
 647     @mock.patch("arvados.collection.CollectionReader")
 648     @mock.patch("time.sleep")
 649     @stubs
 650     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 651         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 652         exited = arvados_cwl.main(
 653             ["--submit", "--no-wait", "--api=containers", "--debug",
 654              "tests/wf/submit_keepref_wf.cwl"],
 655             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 656         self.assertEqual(exited, 0)
 657
 658     @mock.patch("arvados.collection.CollectionReader")
 659     @mock.patch("time.sleep")
 660     @stubs
 661     def test_submit_keepref(self, stubs, tm, reader):
 662         with open("tests/wf/expect_arvworkflow.cwl") as f:
 663             reader().open().__enter__().read.return_value = f.read()
 664
 665         exited = arvados_cwl.main(
 666             ["--submit", "--no-wait", "--api=containers", "--debug",
 667              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 668             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 669
 670         expect_container = {
 671             'priority': 500,
 672             'mounts': {
 673                 '/var/spool/cwl': {
 674                     'writable': True,
 675                     'kind': 'collection'
 676                 },
 677                 'stdout': {
 678                     'path': '/var/spool/cwl/cwl.output.json',
 679                     'kind': 'file'
 680                 },
 681                 '/var/lib/cwl/workflow': {
 682                     'portable_data_hash': '99999999999999999999999999999994+99',
 683                     'kind': 'collection'
 684                 },
 685                 '/var/lib/cwl/cwl.input.json': {
 686                     'content': {
 687                         'x': 'XxX'
 688                     },
 689                     'kind': 'json'
 690                 }
 691             }, 'state': 'Committed',
 692             'output_path': '/var/spool/cwl',
 693             'name': 'expect_arvworkflow.cwl#main',
 694             'container_image': '999999999999999999999999999999d3+99',
 695             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 696                         '--no-log-timestamps', '--disable-validate',
 697                         '--eval-timeout=20', '--thread-count=1',
 698                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 699                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 700             'cwd': '/var/spool/cwl',
 701             'runtime_constraints': {
 702                 'API': True,
 703                 'vcpus': 1,
 704                 'ram': 1342177280
 705             },
 706             'use_existing': False,
 707             'properties': {},
 708             'secret_mounts': {}
 709         }
 710
 711         stubs.api.container_requests().create.assert_called_with(
 712             body=JsonDiffMatcher(expect_container))
 713         self.assertEqual(stubs.capture_stdout.getvalue(),
 714                          stubs.expect_container_request_uuid + '\n')
 715         self.assertEqual(exited, 0)
 716
 717     @mock.patch("time.sleep")
 718     @stubs
 719     def test_submit_arvworkflow(self, stubs, tm):
 720         with open("tests/wf/expect_arvworkflow.cwl") as f:
 721             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 722
 723         exited = arvados_cwl.main(
 724             ["--submit", "--no-wait", "--api=containers", "--debug",
 725              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 726             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 727
 728         expect_container = {
 729             'priority': 500,
 730             'mounts': {
 731                 '/var/spool/cwl': {
 732                     'writable': True,
 733                     'kind': 'collection'
 734                 },
 735                 'stdout': {
 736                     'path': '/var/spool/cwl/cwl.output.json',
 737                     'kind': 'file'
 738                 },
 739                 '/var/lib/cwl/workflow.json': {
 740                     'kind': 'json',
 741                     'content': {
 742                         'cwlVersion': 'v1.0',
 743                         '$graph': [
 744                             {
 745                                 'id': '#main',
 746                                 'inputs': [
 747                                     {'type': 'string', 'id': '#main/x'}
 748                                 ],
 749                                 'steps': [
 750                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 751                                      'run': '#submit_tool.cwl',
 752                                      'id': '#main/step1',
 753                                      'out': []}
 754                                 ],
 755                                 'class': 'Workflow',
 756                                 'outputs': []
 757                             },
 758                             {
 759                                 'inputs': [
 760                                     {
 761                                         'inputBinding': {'position': 1},
 762                                         'type': 'string',
 763                                         'id': '#submit_tool.cwl/x'}
 764                                 ],
 765                                 'requirements': [
 766                                     {
 767                                         'dockerPull': 'debian:8',
 768                                         'class': 'DockerRequirement',
 769                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 770                                     }
 771                                 ],
 772                                 'id': '#submit_tool.cwl',
 773                                 'outputs': [],
 774                                 'baseCommand': 'cat',
 775                                 'class': 'CommandLineTool'
 776                             }
 777                         ]
 778                     }
 779                 },
 780                 '/var/lib/cwl/cwl.input.json': {
 781                     'content': {
 782                         'x': 'XxX'
 783                     },
 784                     'kind': 'json'
 785                 }
 786             }, 'state': 'Committed',
 787             'output_path': '/var/spool/cwl',
 788             'name': 'a test workflow',
 789             'container_image': "999999999999999999999999999999d3+99",
 790             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 791                         '--no-log-timestamps', '--disable-validate',
 792                         '--eval-timeout=20', '--thread-count=1',
 793                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 794                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 795             'cwd': '/var/spool/cwl',
 796             'runtime_constraints': {
 797                 'API': True,
 798                 'vcpus': 1,
 799                 'ram': 1342177280
 800             },
 801             'use_existing': False,
 802             'properties': {
 803                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 804             },
 805             'secret_mounts': {}
 806         }
 807
 808         stubs.api.container_requests().create.assert_called_with(
 809             body=JsonDiffMatcher(expect_container))
 810         self.assertEqual(stubs.capture_stdout.getvalue(),
 811                          stubs.expect_container_request_uuid + '\n')
 812         self.assertEqual(exited, 0)
 813
 814     @stubs
 815     def test_submit_container_name(self, stubs):
 816         exited = arvados_cwl.main(
 817             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 818                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 819             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 820
 821         expect_container = copy.deepcopy(stubs.expect_container_spec)
 822         expect_container["name"] = "hello container 123"
 823
 824         stubs.api.container_requests().create.assert_called_with(
 825             body=JsonDiffMatcher(expect_container))
 826         self.assertEqual(stubs.capture_stdout.getvalue(),
 827                          stubs.expect_container_request_uuid + '\n')
 828         self.assertEqual(exited, 0)
 829
 830     @stubs
 831     def test_submit_missing_input(self, stubs):
 832         exited = arvados_cwl.main(
 833             ["--submit", "--no-wait", "--api=containers", "--debug",
 834              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 835             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 836         self.assertEqual(exited, 0)
 837
 838         exited = arvados_cwl.main(
 839             ["--submit", "--no-wait", "--api=containers", "--debug",
 840              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 841             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 842         self.assertEqual(exited, 1)
 843
 844     @stubs
 845     def test_submit_container_project(self, stubs):
 846         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 849                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851
 852         expect_container = copy.deepcopy(stubs.expect_container_spec)
 853         expect_container["owner_uuid"] = project_uuid
 854         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 855                                        '--no-log-timestamps', '--disable-validate',
 856                                        "--eval-timeout=20", "--thread-count=1",
 857                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 858                                        '--on-error=continue',
 859                                        '--project-uuid='+project_uuid,
 860                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 861
 862         stubs.api.container_requests().create.assert_called_with(
 863             body=JsonDiffMatcher(expect_container))
 864         self.assertEqual(stubs.capture_stdout.getvalue(),
 865                          stubs.expect_container_request_uuid + '\n')
 866         self.assertEqual(exited, 0)
 867
 868     @stubs
 869     def test_submit_container_eval_timeout(self, stubs):
 870         exited = arvados_cwl.main(
 871             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 872                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 873             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 874
 875         expect_container = copy.deepcopy(stubs.expect_container_spec)
 876         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 877                                        '--no-log-timestamps', '--disable-validate',
 878                                        '--eval-timeout=60.0', '--thread-count=1',
 879                                        '--enable-reuse', "--collection-cache-size=256",
 880                                        '--debug', '--on-error=continue',
 881                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 882
 883         stubs.api.container_requests().create.assert_called_with(
 884             body=JsonDiffMatcher(expect_container))
 885         self.assertEqual(stubs.capture_stdout.getvalue(),
 886                          stubs.expect_container_request_uuid + '\n')
 887         self.assertEqual(exited, 0)
 888
 889     @stubs
 890     def test_submit_container_collection_cache(self, stubs):
 891         exited = arvados_cwl.main(
 892             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 893                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 894             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 895
 896         expect_container = copy.deepcopy(stubs.expect_container_spec)
 897         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 898                                        '--no-log-timestamps', '--disable-validate',
 899                                        '--eval-timeout=20', '--thread-count=1',
 900                                        '--enable-reuse', "--collection-cache-size=500",
 901                                        '--debug', '--on-error=continue',
 902                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 903         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 904
 905         stubs.api.container_requests().create.assert_called_with(
 906             body=JsonDiffMatcher(expect_container))
 907         self.assertEqual(stubs.capture_stdout.getvalue(),
 908                          stubs.expect_container_request_uuid + '\n')
 909         self.assertEqual(exited, 0)
 910
 911     @stubs
 912     def test_submit_container_thread_count(self, stubs):
 913         exited = arvados_cwl.main(
 914             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 915                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 916             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 917
 918         expect_container = copy.deepcopy(stubs.expect_container_spec)
 919         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 920                                        '--no-log-timestamps', '--disable-validate',
 921                                        '--eval-timeout=20', '--thread-count=20',
 922                                        '--enable-reuse', "--collection-cache-size=256",
 923                                        '--debug', '--on-error=continue',
 924                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 925
 926         stubs.api.container_requests().create.assert_called_with(
 927             body=JsonDiffMatcher(expect_container))
 928         self.assertEqual(stubs.capture_stdout.getvalue(),
 929                          stubs.expect_container_request_uuid + '\n')
 930         self.assertEqual(exited, 0)
 931
 932     @stubs
 933     def test_submit_container_runner_image(self, stubs):
 934         exited = arvados_cwl.main(
 935             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 936                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 937             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 938
 939         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 940
 941         expect_container = copy.deepcopy(stubs.expect_container_spec)
 942         stubs.api.container_requests().create.assert_called_with(
 943             body=JsonDiffMatcher(expect_container))
 944         self.assertEqual(stubs.capture_stdout.getvalue(),
 945                          stubs.expect_container_request_uuid + '\n')
 946         self.assertEqual(exited, 0)
 947
 948     @stubs
 949     def test_submit_priority(self, stubs):
 950         exited = arvados_cwl.main(
 951             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 952                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 953             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 954
 955         stubs.expect_container_spec["priority"] = 669
 956
 957         expect_container = copy.deepcopy(stubs.expect_container_spec)
 958         stubs.api.container_requests().create.assert_called_with(
 959             body=JsonDiffMatcher(expect_container))
 960         self.assertEqual(stubs.capture_stdout.getvalue(),
 961                          stubs.expect_container_request_uuid + '\n')
 962         self.assertEqual(exited, 0)
 963
 964     @stubs
 965     def test_submit_wf_runner_resources(self, stubs):
 966         exited = arvados_cwl.main(
 967             ["--submit", "--no-wait", "--api=containers", "--debug",
 968                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 969             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 970
 971         expect_container = copy.deepcopy(stubs.expect_container_spec)
 972         expect_container["runtime_constraints"] = {
 973             "API": True,
 974             "vcpus": 2,
 975             "ram": (2000+512) * 2**20
 976         }
 977         expect_container["name"] = "submit_wf_runner_resources.cwl"
 978         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 979             {
 980                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 981                 "coresMin": 2,
 982                 "ramMin": 2000,
 983                 "keep_cache": 512
 984             }
 985         ]
 986         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 987             "arv": "http://arvados.org/cwl#",
 988         }
 989         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 990                         '--no-log-timestamps', '--disable-validate',
 991                         '--eval-timeout=20', '--thread-count=1',
 992                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
 993                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 994
 995         stubs.api.container_requests().create.assert_called_with(
 996             body=JsonDiffMatcher(expect_container))
 997         self.assertEqual(stubs.capture_stdout.getvalue(),
 998                          stubs.expect_container_request_uuid + '\n')
 999         self.assertEqual(exited, 0)
1000
1001     def tearDown(self):
1002         arvados_cwl.arvdocker.arv_docker_clear_cache()
1003
1004     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1005     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1006     @mock.patch("arvados.api")
1007     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1008         arvados_cwl.arvdocker.arv_docker_clear_cache()
1009
1010         arvrunner = mock.MagicMock()
1011         arvrunner.project_uuid = ""
1012         api.return_value = mock.MagicMock()
1013         arvrunner.api = api.return_value
1014         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1015                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1016                                                                         "link_class": "docker_image_repo+tag",
1017                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1018                                                                         "owner_uuid": "",
1019                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1020                                                             {"items": [{"created_at": "",
1021                                                                         "head_uuid": "",
1022                                                                         "link_class": "docker_image_hash",
1023                                                                         "name": "123456",
1024                                                                         "owner_uuid": "",
1025                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1026         )
1027         find_one_image_hash.return_value = "123456"
1028
1029         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1030                                                                               "owner_uuid": "",
1031                                                                               "manifest_text": "",
1032                                                                               "properties": ""
1033                                                                           }], "items_available": 1, "offset": 0},)
1034         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1035         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1036                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1037         self.assertEqual("9999999999999999999999999999999b+99",
1038                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1039
1040
1041     @stubs
1042     def test_submit_secrets(self, stubs):
1043         exited = arvados_cwl.main(
1044             ["--submit", "--no-wait", "--api=containers", "--debug",
1045                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1046             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1047
1048         expect_container = {
1049             "command": [
1050                 "arvados-cwl-runner",
1051                 "--local",
1052                 "--api=containers",
1053                 "--no-log-timestamps",
1054                 "--disable-validate",
1055                 "--eval-timeout=20",
1056                 '--thread-count=1',
1057                 "--enable-reuse",
1058                 "--collection-cache-size=256",
1059                 '--debug',
1060                 "--on-error=continue",
1061                 "/var/lib/cwl/workflow.json#main",
1062                 "/var/lib/cwl/cwl.input.json"
1063             ],
1064             "container_image": "999999999999999999999999999999d3+99",
1065             "cwd": "/var/spool/cwl",
1066             "mounts": {
1067                 "/var/lib/cwl/cwl.input.json": {
1068                     "content": {
1069                         "pw": {
1070                             "$include": "/secrets/s0"
1071                         }
1072                     },
1073                     "kind": "json"
1074                 },
1075                 "/var/lib/cwl/workflow.json": {
1076                     "content": {
1077                         "$graph": [
1078                             {
1079                                 "$namespaces": {
1080                                     "cwltool": "http://commonwl.org/cwltool#"
1081                                 },
1082                                 "arguments": [
1083                                     "md5sum",
1084                                     "example.conf"
1085                                 ],
1086                                 "class": "CommandLineTool",
1087                                 "hints": [
1088                                     {
1089                                         "class": "http://commonwl.org/cwltool#Secrets",
1090                                         "secrets": [
1091                                             "#secret_job.cwl/pw"
1092                                         ]
1093                                     }
1094                                 ],
1095                                 "id": "#secret_job.cwl",
1096                                 "inputs": [
1097                                     {
1098                                         "id": "#secret_job.cwl/pw",
1099                                         "type": "string"
1100                                     }
1101                                 ],
1102                                 "outputs": [
1103                                     {
1104                                         "id": "#secret_job.cwl/out",
1105                                         "type": "stdout"
1106                                     }
1107                                 ],
1108                                 "stdout": "hashed_example.txt",
1109                                 "requirements": [
1110                                     {
1111                                         "class": "InitialWorkDirRequirement",
1112                                         "listing": [
1113                                             {
1114                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1115                                                 "entryname": "example.conf"
1116                                             }
1117                                         ]
1118                                     }
1119                                 ]
1120                             },
1121                             {
1122                                 "class": "Workflow",
1123                                 "hints": [
1124                                     {
1125                                         "class": "DockerRequirement",
1126                                         "dockerPull": "debian:8",
1127                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1128                                     },
1129                                     {
1130                                         "class": "http://commonwl.org/cwltool#Secrets",
1131                                         "secrets": [
1132                                             "#main/pw"
1133                                         ]
1134                                     }
1135                                 ],
1136                                 "id": "#main",
1137                                 "inputs": [
1138                                     {
1139                                         "id": "#main/pw",
1140                                         "type": "string"
1141                                     }
1142                                 ],
1143                                 "outputs": [
1144                                     {
1145                                         "id": "#main/out",
1146                                         "outputSource": "#main/step1/out",
1147                                         "type": "File"
1148                                     }
1149                                 ],
1150                                 "steps": [
1151                                     {
1152                                         "id": "#main/step1",
1153                                         "in": [
1154                                             {
1155                                                 "id": "#main/step1/pw",
1156                                                 "source": "#main/pw"
1157                                             }
1158                                         ],
1159                                         "out": [
1160                                             "#main/step1/out"
1161                                         ],
1162                                         "run": "#secret_job.cwl"
1163                                     }
1164                                 ]
1165                             }
1166                         ],
1167                         "cwlVersion": "v1.0"
1168                     },
1169                     "kind": "json"
1170                 },
1171                 "/var/spool/cwl": {
1172                     "kind": "collection",
1173                     "writable": True
1174                 },
1175                 "stdout": {
1176                     "kind": "file",
1177                     "path": "/var/spool/cwl/cwl.output.json"
1178                 }
1179             },
1180             "name": "secret_wf.cwl",
1181             "output_path": "/var/spool/cwl",
1182             "priority": 500,
1183             "properties": {},
1184             "runtime_constraints": {
1185                 "API": True,
1186                 "ram": 1342177280,
1187                 "vcpus": 1
1188             },
1189             "secret_mounts": {
1190                 "/secrets/s0": {
1191                     "content": "blorp",
1192                     "kind": "text"
1193                 }
1194             },
1195             "state": "Committed",
1196             "use_existing": False
1197         }
1198
1199         stubs.api.container_requests().create.assert_called_with(
1200             body=JsonDiffMatcher(expect_container))
1201         self.assertEqual(stubs.capture_stdout.getvalue(),
1202                          stubs.expect_container_request_uuid + '\n')
1203         self.assertEqual(exited, 0)
1204
1205     @stubs
1206     def test_submit_request_uuid(self, stubs):
1207         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1208         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1209
1210         stubs.api.container_requests().update().execute.return_value = {
1211             "uuid": stubs.expect_container_request_uuid,
1212             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1213             "state": "Queued"
1214         }
1215
1216         exited = arvados_cwl.main(
1217             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1218                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1219             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1220
1221         stubs.api.container_requests().update.assert_called_with(
1222             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1223         self.assertEqual(stubs.capture_stdout.getvalue(),
1224                          stubs.expect_container_request_uuid + '\n')
1225         self.assertEqual(exited, 0)
1226
1227     @stubs
1228     def test_submit_container_cluster_id(self, stubs):
1229         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1230
1231         exited = arvados_cwl.main(
1232             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1233                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1234             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1235
1236         expect_container = copy.deepcopy(stubs.expect_container_spec)
1237
1238         stubs.api.container_requests().create.assert_called_with(
1239             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1240         self.assertEqual(stubs.capture_stdout.getvalue(),
1241                          stubs.expect_container_request_uuid + '\n')
1242         self.assertEqual(exited, 0)
1243
1244     @stubs
1245     def test_submit_validate_cluster_id(self, stubs):
1246         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1247         exited = arvados_cwl.main(
1248             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1249              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1250             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1251         self.assertEqual(exited, 1)
1252
1253     @mock.patch("arvados.collection.CollectionReader")
1254     @stubs
1255     def test_submit_uuid_inputs(self, stubs, collectionReader):
1256         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1257         def list_side_effect(**kwargs):
1258             m = mock.MagicMock()
1259             if "count" in kwargs:
1260                 m.execute.return_value = {"items": [
1261                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1262                 ]}
1263             else:
1264                 m.execute.return_value = {"items": []}
1265             return m
1266         stubs.api.collections().list.side_effect = list_side_effect
1267
1268         exited = arvados_cwl.main(
1269             ["--submit", "--no-wait", "--api=containers", "--debug",
1270                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1271             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1272
1273         expect_container = copy.deepcopy(stubs.expect_container_spec)
1274         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1275         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1276         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1277
1278         stubs.api.collections().list.assert_has_calls([
1279             mock.call(count='none',
1280                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1281                       select=['uuid', 'portable_data_hash'])])
1282         stubs.api.container_requests().create.assert_called_with(
1283             body=JsonDiffMatcher(expect_container))
1284         self.assertEqual(stubs.capture_stdout.getvalue(),
1285                          stubs.expect_container_request_uuid + '\n')
1286         self.assertEqual(exited, 0)
1287
1288     @stubs
1289     def test_submit_mismatched_uuid_inputs(self, stubs):
1290         def list_side_effect(**kwargs):
1291             m = mock.MagicMock()
1292             if "count" in kwargs:
1293                 m.execute.return_value = {"items": [
1294                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1295                 ]}
1296             else:
1297                 m.execute.return_value = {"items": []}
1298             return m
1299         stubs.api.collections().list.side_effect = list_side_effect
1300
1301         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1302             capture_stderr = StringIO()
1303             cwltool_logger = logging.getLogger('cwltool')
1304             stderr_logger = logging.StreamHandler(capture_stderr)
1305             cwltool_logger.addHandler(stderr_logger)
1306
1307             try:
1308                 exited = arvados_cwl.main(
1309                     ["--submit", "--no-wait", "--api=containers", "--debug",
1310                         "tests/wf/submit_wf.cwl", infile],
1311                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1312
1313                 self.assertEqual(exited, 1)
1314                 self.assertRegexpMatches(
1315                     capture_stderr.getvalue(),
1316                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1317             finally:
1318                 cwltool_logger.removeHandler(stderr_logger)
1319
1320     @mock.patch("arvados.collection.CollectionReader")
1321     @stubs
1322     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1323         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1324         capture_stderr = StringIO()
1325
1326         cwltool_logger = logging.getLogger('cwltool')
1327         stderr_logger = logging.StreamHandler(capture_stderr)
1328         cwltool_logger.addHandler(stderr_logger)
1329
1330         exited = arvados_cwl.main(
1331             ["--submit", "--no-wait", "--api=containers", "--debug",
1332                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1333             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1334
1335         try:
1336             self.assertEqual(exited, 1)
1337             self.assertRegexpMatches(
1338                 capture_stderr.getvalue(),
1339                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1340         finally:
1341             cwltool_logger.removeHandler(stderr_logger)
1342
1343
1344 class TestCreateWorkflow(unittest.TestCase):
1345     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1346     expect_workflow = StripYAMLComments(
1347         open("tests/wf/expect_packed.cwl").read())
1348
1349     @stubs
1350     def test_create(self, stubs):
1351         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1352
1353         exited = arvados_cwl.main(
1354             ["--create-workflow", "--debug",
1355              "--api=containers",
1356              "--project-uuid", project_uuid,
1357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1358             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1359
1360         stubs.api.pipeline_templates().create.refute_called()
1361         stubs.api.container_requests().create.refute_called()
1362
1363         body = {
1364             "workflow": {
1365                 "owner_uuid": project_uuid,
1366                 "name": "submit_wf.cwl",
1367                 "description": "",
1368                 "definition": self.expect_workflow,
1369             }
1370         }
1371         stubs.api.workflows().create.assert_called_with(
1372             body=JsonDiffMatcher(body))
1373
1374         self.assertEqual(stubs.capture_stdout.getvalue(),
1375                          stubs.expect_workflow_uuid + '\n')
1376         self.assertEqual(exited, 0)
1377
1378     @stubs
1379     def test_create_name(self, stubs):
1380         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1381
1382         exited = arvados_cwl.main(
1383             ["--create-workflow", "--debug",
1384              "--api=containers",
1385              "--project-uuid", project_uuid,
1386              "--name", "testing 123",
1387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1388             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1389
1390         stubs.api.pipeline_templates().create.refute_called()
1391         stubs.api.container_requests().create.refute_called()
1392
1393         body = {
1394             "workflow": {
1395                 "owner_uuid": project_uuid,
1396                 "name": "testing 123",
1397                 "description": "",
1398                 "definition": self.expect_workflow,
1399             }
1400         }
1401         stubs.api.workflows().create.assert_called_with(
1402             body=JsonDiffMatcher(body))
1403
1404         self.assertEqual(stubs.capture_stdout.getvalue(),
1405                          stubs.expect_workflow_uuid + '\n')
1406         self.assertEqual(exited, 0)
1407
1408
1409     @stubs
1410     def test_update(self, stubs):
1411         exited = arvados_cwl.main(
1412             ["--update-workflow", self.existing_workflow_uuid,
1413              "--debug",
1414              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1415             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1416
1417         body = {
1418             "workflow": {
1419                 "name": "submit_wf.cwl",
1420                 "description": "",
1421                 "definition": self.expect_workflow,
1422             }
1423         }
1424         stubs.api.workflows().update.assert_called_with(
1425             uuid=self.existing_workflow_uuid,
1426             body=JsonDiffMatcher(body))
1427         self.assertEqual(stubs.capture_stdout.getvalue(),
1428                          self.existing_workflow_uuid + '\n')
1429         self.assertEqual(exited, 0)
1430
1431     @stubs
1432     def test_update_name(self, stubs):
1433         exited = arvados_cwl.main(
1434             ["--update-workflow", self.existing_workflow_uuid,
1435              "--debug", "--name", "testing 123",
1436              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1437             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1438
1439         body = {
1440             "workflow": {
1441                 "name": "testing 123",
1442                 "description": "",
1443                 "definition": self.expect_workflow,
1444             }
1445         }
1446         stubs.api.workflows().update.assert_called_with(
1447             uuid=self.existing_workflow_uuid,
1448             body=JsonDiffMatcher(body))
1449         self.assertEqual(stubs.capture_stdout.getvalue(),
1450                          self.existing_workflow_uuid + '\n')
1451         self.assertEqual(exited, 0)
1452
1453     @stubs
1454     def test_create_collection_per_tool(self, stubs):
1455         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1456
1457         exited = arvados_cwl.main(
1458             ["--create-workflow", "--debug",
1459              "--api=containers",
1460              "--project-uuid", project_uuid,
1461              "tests/collection_per_tool/collection_per_tool.cwl"],
1462             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1463
1464         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1465         expect_workflow = StripYAMLComments(open(toolfile).read())
1466
1467         body = {
1468             "workflow": {
1469                 "owner_uuid": project_uuid,
1470                 "name": "collection_per_tool.cwl",
1471                 "description": "",
1472                 "definition": expect_workflow,
1473             }
1474         }
1475         stubs.api.workflows().create.assert_called_with(
1476             body=JsonDiffMatcher(body))
1477
1478         self.assertEqual(stubs.capture_stdout.getvalue(),
1479                          stubs.expect_workflow_uuid + '\n')
1480         self.assertEqual(exited, 0)
1481
1482     @stubs
1483     def test_create_with_imports(self, stubs):
1484         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1485
1486         exited = arvados_cwl.main(
1487             ["--create-workflow", "--debug",
1488              "--api=containers",
1489              "--project-uuid", project_uuid,
1490              "tests/wf/feddemo/feddemo.cwl"],
1491             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1492
1493         stubs.api.pipeline_templates().create.refute_called()
1494         stubs.api.container_requests().create.refute_called()
1495
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_workflow_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500     @stubs
1501     def test_create_with_no_input(self, stubs):
1502         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1503
1504         exited = arvados_cwl.main(
1505             ["--create-workflow", "--debug",
1506              "--api=containers",
1507              "--project-uuid", project_uuid,
1508              "tests/wf/revsort/revsort.cwl"],
1509             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1510
1511         stubs.api.pipeline_templates().create.refute_called()
1512         stubs.api.container_requests().create.refute_called()
1513
1514         self.assertEqual(stubs.capture_stdout.getvalue(),
1515                          stubs.expect_workflow_uuid + '\n')
1516         self.assertEqual(exited, 0)