sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21
  22 from io import BytesIO
  23
  24 # StringIO.StringIO and io.StringIO have different behavior write() is
  25 # called with both python2 (byte) strings and unicode strings
  26 # (specifically there's some logging in cwltool that causes trouble).
  27 # This isn't a problem on python3 because all string are unicode.
  28 if sys.version_info[0] < 3:
  29     from StringIO import StringIO
  30 else:
  31     from io import StringIO
  32
  33 import arvados
  34 import arvados.collection
  35 import arvados_cwl
  36 import arvados_cwl.executor
  37 import arvados_cwl.runner
  38 import arvados.keep
  39
  40 from .matcher import JsonDiffMatcher, StripYAMLComments
  41 from .mock_discovery import get_rootDesc
  42
  43 import ruamel.yaml as yaml
  44
  45 _rootDesc = None
  46
  47 def stubs(func):
  48     @functools.wraps(func)
  49     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  50     @mock.patch("arvados.collection.KeepClient")
  51     @mock.patch("arvados.keep.KeepClient")
  52     @mock.patch("arvados.events.subscribe")
  53     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  54         class Stubs(object):
  55             pass
  56         stubs = Stubs()
  57         stubs.events = events
  58         stubs.keepdocker = keepdocker
  59
  60         def putstub(p, **kwargs):
  61             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  62         keep_client1().put.side_effect = putstub
  63         keep_client1.put.side_effect = putstub
  64         keep_client2().put.side_effect = putstub
  65         keep_client2.put.side_effect = putstub
  66
  67         stubs.keep_client = keep_client2
  68         stubs.docker_images = {
  69             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  70             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  71             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  72             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  73         }
  74         def kd(a, b, image_name=None, image_tag=None):
  75             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  76         stubs.keepdocker.side_effect = kd
  77
  78         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  79         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  80
  81         if sys.version_info[0] < 3:
  82             stubs.capture_stdout = BytesIO()
  83         else:
  84             stubs.capture_stdout = StringIO()
  85
  86         stubs.api = mock.MagicMock()
  87         stubs.api._rootDesc = get_rootDesc()
  88         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  89
  90         stubs.api.users().current().execute.return_value = {
  91             "uuid": stubs.fake_user_uuid,
  92         }
  93         stubs.api.collections().list().execute.return_value = {"items": []}
  94         stubs.api.containers().current().execute.return_value = {
  95             "uuid": stubs.fake_container_uuid,
  96         }
  97
  98         class CollectionExecute(object):
  99             def __init__(self, exe):
 100                 self.exe = exe
 101             def execute(self, num_retries=None):
 102                 return self.exe
 103
 104         def collection_createstub(created_collections, body, ensure_unique_name=None):
 105             mt = body["manifest_text"].encode('utf-8')
 106             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 107             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 108             created_collections[uuid] = {
 109                 "uuid": uuid,
 110                 "portable_data_hash": pdh,
 111                 "manifest_text": mt.decode('utf-8')
 112             }
 113             return CollectionExecute(created_collections[uuid])
 114
 115         def collection_getstub(created_collections, uuid):
 116             for v in viewvalues(created_collections):
 117                 if uuid in (v["uuid"], v["portable_data_hash"]):
 118                     return CollectionExecute(v)
 119
 120         created_collections = {
 121             "99999999999999999999999999999998+99": {
 122                 "uuid": "",
 123                 "portable_data_hash": "99999999999999999999999999999998+99",
 124                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 125             },
 126             "99999999999999999999999999999997+99": {
 127                 "uuid": "",
 128                 "portable_data_hash": "99999999999999999999999999999997+99",
 129                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 130             },
 131             "99999999999999999999999999999994+99": {
 132                 "uuid": "",
 133                 "portable_data_hash": "99999999999999999999999999999994+99",
 134                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 135             },
 136             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 137                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 138                 "portable_data_hash": "999999999999999999999999999999d3+99",
 139                 "manifest_text": ""
 140             },
 141             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 142                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 143                 "portable_data_hash": "999999999999999999999999999999d4+99",
 144                 "manifest_text": ""
 145             },
 146             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 147                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 148                 "portable_data_hash": "999999999999999999999999999999d5+99",
 149                 "manifest_text": ""
 150             },
 151             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 152                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 153                 "portable_data_hash": "999999999999999999999999999999d6+99",
 154                 "manifest_text": ""
 155             }
 156         }
 157         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 158         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 159
 160         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 161         stubs.api.jobs().create().execute.return_value = {
 162             "uuid": stubs.expect_job_uuid,
 163             "state": "Queued",
 164         }
 165
 166         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 167         stubs.api.container_requests().create().execute.return_value = {
 168             "uuid": stubs.expect_container_request_uuid,
 169             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 170             "state": "Queued"
 171         }
 172
 173         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 174         stubs.api.pipeline_templates().create().execute.return_value = {
 175             "uuid": stubs.expect_pipeline_template_uuid,
 176         }
 177         stubs.expect_job_spec = {
 178             'runtime_constraints': {
 179                 'docker_image': '999999999999999999999999999999d3+99',
 180                 'min_ram_mb_per_node': 1024
 181             },
 182             'script_parameters': {
 183                 'x': {
 184                     'basename': 'blorp.txt',
 185                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 186                     'class': 'File'
 187                 },
 188                 'y': {
 189                     'basename': '99999999999999999999999999999998+99',
 190                     'location': 'keep:99999999999999999999999999999998+99',
 191                     'class': 'Directory'
 192                 },
 193                 'z': {
 194                     'basename': 'anonymous',
 195                     "listing": [{
 196                         "basename": "renamed.txt",
 197                         "class": "File",
 198                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 199                         "size": 0
 200                     }],
 201                     'class': 'Directory'
 202                 },
 203                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 204             },
 205             'repository': 'arvados',
 206             'script_version': 'master',
 207             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 208             'script': 'cwl-runner'
 209         }
 210         stubs.pipeline_component = stubs.expect_job_spec.copy()
 211         stubs.expect_pipeline_instance = {
 212             'name': 'submit_wf.cwl',
 213             'state': 'RunningOnServer',
 214             'owner_uuid': None,
 215             "components": {
 216                 "cwl-runner": {
 217                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 218                     'script_parameters': {
 219                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 220                         'x': {"value": {
 221                             'basename': 'blorp.txt',
 222                             'class': 'File',
 223                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 224                             "size": 16
 225                         }},
 226                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 227                               'listing': [
 228                                   {
 229                                       'basename': 'renamed.txt',
 230                                       'class': 'File', 'location':
 231                                       'keep:99999999999999999999999999999998+99/file1.txt',
 232                                       'size': 0
 233                                   }
 234                               ]}},
 235                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 236                         'arv:debug': True,
 237                         'arv:enable_reuse': True,
 238                         'arv:on_error': 'continue'
 239                     },
 240                     'repository': 'arvados',
 241                     'script_version': 'master',
 242                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 243                     'script': 'cwl-runner',
 244                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 245                 }
 246             }
 247         }
 248         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 249         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 250         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 251         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 252         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 253             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 254             "state": "Queued"
 255         }
 256         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 257         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 258
 259         with open("tests/wf/submit_wf_packed.cwl") as f:
 260             expect_packed_workflow = yaml.round_trip_load(f)
 261
 262         stubs.expect_container_spec = {
 263             'priority': 500,
 264             'mounts': {
 265                 '/var/spool/cwl': {
 266                     'writable': True,
 267                     'kind': 'collection'
 268                 },
 269                 '/var/lib/cwl/workflow.json': {
 270                     'content': expect_packed_workflow,
 271                     'kind': 'json'
 272                 },
 273                 'stdout': {
 274                     'path': '/var/spool/cwl/cwl.output.json',
 275                     'kind': 'file'
 276                 },
 277                 '/var/lib/cwl/cwl.input.json': {
 278                     'kind': 'json',
 279                     'content': {
 280                         'y': {
 281                             'basename': '99999999999999999999999999999998+99',
 282                             'location': 'keep:99999999999999999999999999999998+99',
 283                             'class': 'Directory'},
 284                         'x': {
 285                             'basename': u'blorp.txt',
 286                             'class': 'File',
 287                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 288                             "size": 16
 289                         },
 290                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 291                             {'basename': 'renamed.txt',
 292                              'class': 'File',
 293                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 294                              'size': 0
 295                             }
 296                         ]}
 297                     },
 298                     'kind': 'json'
 299                 }
 300             },
 301             'secret_mounts': {},
 302             'state': 'Committed',
 303             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 304                         '--no-log-timestamps', '--disable-validate',
 305                         '--eval-timeout=20', '--thread-count=1',
 306                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 307                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 308             'name': 'submit_wf.cwl',
 309             'container_image': '999999999999999999999999999999d3+99',
 310             'output_path': '/var/spool/cwl',
 311             'cwd': '/var/spool/cwl',
 312             'runtime_constraints': {
 313                 'API': True,
 314                 'vcpus': 1,
 315                 'ram': (1024+256)*1024*1024
 316             },
 317             'use_existing': False,
 318             'properties': {},
 319             'secret_mounts': {}
 320         }
 321
 322         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 323         stubs.api.workflows().create().execute.return_value = {
 324             "uuid": stubs.expect_workflow_uuid,
 325         }
 326         def update_mock(**kwargs):
 327             stubs.updated_uuid = kwargs.get('uuid')
 328             return mock.DEFAULT
 329         stubs.api.workflows().update.side_effect = update_mock
 330         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 331             "uuid": stubs.updated_uuid,
 332         }
 333
 334         return func(self, stubs, *args, **kwargs)
 335     return wrapped
 336
 337
 338 class TestSubmit(unittest.TestCase):
 339
 340     def setUp(self):
 341         cwltool.process._names = set()
 342
 343     @stubs
 344     def test_error_when_multiple_storage_classes_specified(self, stubs):
 345         storage_classes = "foo,bar"
 346         exited = arvados_cwl.main(
 347                 ["--debug", "--storage-classes", storage_classes,
 348                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 349                 sys.stdin, sys.stderr, api_client=stubs.api)
 350         self.assertEqual(exited, 1)
 351
 352     @mock.patch("time.sleep")
 353     @stubs
 354     def test_submit_invalid_runner_ram(self, stubs, tm):
 355         exited = arvados_cwl.main(
 356             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 358             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 359         self.assertEqual(exited, 1)
 360
 361
 362     @stubs
 363     def test_submit_container(self, stubs):
 364         exited = arvados_cwl.main(
 365             ["--submit", "--no-wait", "--api=containers", "--debug",
 366                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 367             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 368
 369         stubs.api.collections().create.assert_has_calls([
 370             mock.call(body=JsonDiffMatcher({
 371                 'manifest_text':
 372                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 373                 'replication_desired': None,
 374                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 375             }), ensure_unique_name=False),
 376             mock.call(body=JsonDiffMatcher({
 377                 'manifest_text':
 378                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 379                 'replication_desired': None,
 380                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 381             }), ensure_unique_name=False),
 382             ])
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         stubs.api.container_requests().create.assert_called_with(
 386             body=JsonDiffMatcher(expect_container))
 387         self.assertEqual(stubs.capture_stdout.getvalue(),
 388                          stubs.expect_container_request_uuid + '\n')
 389         self.assertEqual(exited, 0)
 390
 391     @stubs
 392     def test_submit_container_no_reuse(self, stubs):
 393         exited = arvados_cwl.main(
 394             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 395                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 396             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 397
 398         expect_container = copy.deepcopy(stubs.expect_container_spec)
 399         expect_container["command"] = [
 400             'arvados-cwl-runner', '--local', '--api=containers',
 401             '--no-log-timestamps', '--disable-validate',
 402             '--eval-timeout=20', '--thread-count=1',
 403             '--disable-reuse', "--collection-cache-size=256",
 404             '--debug', '--on-error=continue',
 405             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 406         expect_container["use_existing"] = False
 407
 408         stubs.api.container_requests().create.assert_called_with(
 409             body=JsonDiffMatcher(expect_container))
 410         self.assertEqual(stubs.capture_stdout.getvalue(),
 411                          stubs.expect_container_request_uuid + '\n')
 412         self.assertEqual(exited, 0)
 413
 414     @stubs
 415     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 416         exited = arvados_cwl.main(
 417             ["--submit", "--no-wait", "--api=containers", "--debug",
 418              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 419             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 420         self.assertEqual(exited, 0)
 421
 422         expect_container = copy.deepcopy(stubs.expect_container_spec)
 423         expect_container["command"] = [
 424             'arvados-cwl-runner', '--local', '--api=containers',
 425             '--no-log-timestamps', '--disable-validate',
 426             '--eval-timeout=20', '--thread-count=1',
 427             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 428             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 429         expect_container["use_existing"] = False
 430         expect_container["name"] = "submit_wf_no_reuse.cwl"
 431         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 432             {
 433                 "class": "http://arvados.org/cwl#ReuseRequirement",
 434                 "enableReuse": False,
 435             },
 436         ]
 437         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 438             "arv": "http://arvados.org/cwl#",
 439             "cwltool": "http://commonwl.org/cwltool#"
 440         }
 441
 442         stubs.api.container_requests().create.assert_called_with(
 443             body=JsonDiffMatcher(expect_container))
 444         self.assertEqual(stubs.capture_stdout.getvalue(),
 445                          stubs.expect_container_request_uuid + '\n')
 446
 447
 448     @stubs
 449     def test_submit_container_on_error(self, stubs):
 450         exited = arvados_cwl.main(
 451             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 452                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 453             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 454
 455         expect_container = copy.deepcopy(stubs.expect_container_spec)
 456         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 457                                        '--no-log-timestamps', '--disable-validate',
 458                                        '--eval-timeout=20', '--thread-count=1',
 459                                        '--enable-reuse', "--collection-cache-size=256",
 460                                        '--debug', '--on-error=stop',
 461                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 462
 463         stubs.api.container_requests().create.assert_called_with(
 464             body=JsonDiffMatcher(expect_container))
 465         self.assertEqual(stubs.capture_stdout.getvalue(),
 466                          stubs.expect_container_request_uuid + '\n')
 467         self.assertEqual(exited, 0)
 468
 469     @stubs
 470     def test_submit_container_output_name(self, stubs):
 471         output_name = "test_output_name"
 472
 473         exited = arvados_cwl.main(
 474             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 475                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 477
 478         expect_container = copy.deepcopy(stubs.expect_container_spec)
 479         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 480                                        '--no-log-timestamps', '--disable-validate',
 481                                        '--eval-timeout=20', '--thread-count=1',
 482                                        '--enable-reuse', "--collection-cache-size=256",
 483                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 484                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 485         expect_container["output_name"] = output_name
 486
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_container))
 489         self.assertEqual(stubs.capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491         self.assertEqual(exited, 0)
 492
 493     @stubs
 494     def test_submit_storage_classes(self, stubs):
 495         exited = arvados_cwl.main(
 496             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 497                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 498             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 499
 500         expect_container = copy.deepcopy(stubs.expect_container_spec)
 501         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 502                                        '--no-log-timestamps', '--disable-validate',
 503                                        '--eval-timeout=20', '--thread-count=1',
 504                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 505                                        "--storage-classes=foo", '--on-error=continue',
 506                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 507
 508         stubs.api.container_requests().create.assert_called_with(
 509             body=JsonDiffMatcher(expect_container))
 510         self.assertEqual(stubs.capture_stdout.getvalue(),
 511                          stubs.expect_container_request_uuid + '\n')
 512         self.assertEqual(exited, 0)
 513
 514     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 515     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 516     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 517     @stubs
 518     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 519         def set_final_output(job_order, output_callback, runtimeContext):
 520             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 521             return []
 522         job.side_effect = set_final_output
 523
 524         exited = arvados_cwl.main(
 525             ["--debug", "--local", "--storage-classes=foo",
 526                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 527             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 528
 529         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 530         self.assertEqual(exited, 0)
 531
 532     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 533     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 534     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 535     @stubs
 536     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 537         def set_final_output(job_order, output_callback, runtimeContext):
 538             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 539             return []
 540         job.side_effect = set_final_output
 541
 542         exited = arvados_cwl.main(
 543             ["--debug", "--local",
 544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 545             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 546
 547         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 548         self.assertEqual(exited, 0)
 549
 550     @stubs
 551     def test_submit_container_output_ttl(self, stubs):
 552         exited = arvados_cwl.main(
 553             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 554                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 555             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 556
 557         expect_container = copy.deepcopy(stubs.expect_container_spec)
 558         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 559                                        '--no-log-timestamps', '--disable-validate',
 560                                        '--eval-timeout=20', '--thread-count=1',
 561                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 562                                        '--on-error=continue',
 563                                        "--intermediate-output-ttl=3600",
 564                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 565
 566         stubs.api.container_requests().create.assert_called_with(
 567             body=JsonDiffMatcher(expect_container))
 568         self.assertEqual(stubs.capture_stdout.getvalue(),
 569                          stubs.expect_container_request_uuid + '\n')
 570         self.assertEqual(exited, 0)
 571
 572     @stubs
 573     def test_submit_container_trash_intermediate(self, stubs):
 574         exited = arvados_cwl.main(
 575             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 576                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 577             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 578
 579
 580         expect_container = copy.deepcopy(stubs.expect_container_spec)
 581         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 582                                        '--no-log-timestamps', '--disable-validate',
 583                                        '--eval-timeout=20', '--thread-count=1',
 584                                        '--enable-reuse', "--collection-cache-size=256",
 585                                        '--debug', '--on-error=continue',
 586                                        "--trash-intermediate",
 587                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 588
 589         stubs.api.container_requests().create.assert_called_with(
 590             body=JsonDiffMatcher(expect_container))
 591         self.assertEqual(stubs.capture_stdout.getvalue(),
 592                          stubs.expect_container_request_uuid + '\n')
 593         self.assertEqual(exited, 0)
 594
 595     @stubs
 596     def test_submit_container_output_tags(self, stubs):
 597         output_tags = "tag0,tag1,tag2"
 598
 599         exited = arvados_cwl.main(
 600             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 601                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 602             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 603
 604         expect_container = copy.deepcopy(stubs.expect_container_spec)
 605         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 606                                        '--no-log-timestamps', '--disable-validate',
 607                                        '--eval-timeout=20', '--thread-count=1',
 608                                        '--enable-reuse', "--collection-cache-size=256",
 609                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 610                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 611
 612         stubs.api.container_requests().create.assert_called_with(
 613             body=JsonDiffMatcher(expect_container))
 614         self.assertEqual(stubs.capture_stdout.getvalue(),
 615                          stubs.expect_container_request_uuid + '\n')
 616         self.assertEqual(exited, 0)
 617
 618     @stubs
 619     def test_submit_container_runner_ram(self, stubs):
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(stubs.capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632         self.assertEqual(exited, 0)
 633
 634     @mock.patch("arvados.collection.CollectionReader")
 635     @mock.patch("time.sleep")
 636     @stubs
 637     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 638         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 639         exited = arvados_cwl.main(
 640             ["--submit", "--no-wait", "--api=containers", "--debug",
 641              "tests/wf/submit_keepref_wf.cwl"],
 642             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 643         self.assertEqual(exited, 0)
 644
 645     @mock.patch("arvados.collection.CollectionReader")
 646     @mock.patch("time.sleep")
 647     @stubs
 648     def test_submit_keepref(self, stubs, tm, reader):
 649         with open("tests/wf/expect_arvworkflow.cwl") as f:
 650             reader().open().__enter__().read.return_value = f.read()
 651
 652         exited = arvados_cwl.main(
 653             ["--submit", "--no-wait", "--api=containers", "--debug",
 654              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 655             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 656
 657         expect_container = {
 658             'priority': 500,
 659             'mounts': {
 660                 '/var/spool/cwl': {
 661                     'writable': True,
 662                     'kind': 'collection'
 663                 },
 664                 'stdout': {
 665                     'path': '/var/spool/cwl/cwl.output.json',
 666                     'kind': 'file'
 667                 },
 668                 '/var/lib/cwl/workflow': {
 669                     'portable_data_hash': '99999999999999999999999999999994+99',
 670                     'kind': 'collection'
 671                 },
 672                 '/var/lib/cwl/cwl.input.json': {
 673                     'content': {
 674                         'x': 'XxX'
 675                     },
 676                     'kind': 'json'
 677                 }
 678             }, 'state': 'Committed',
 679             'output_path': '/var/spool/cwl',
 680             'name': 'expect_arvworkflow.cwl#main',
 681             'container_image': '999999999999999999999999999999d3+99',
 682             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 683                         '--no-log-timestamps', '--disable-validate',
 684                         '--eval-timeout=20', '--thread-count=1',
 685                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 686                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 687             'cwd': '/var/spool/cwl',
 688             'runtime_constraints': {
 689                 'API': True,
 690                 'vcpus': 1,
 691                 'ram': 1342177280
 692             },
 693             'use_existing': False,
 694             'properties': {},
 695             'secret_mounts': {}
 696         }
 697
 698         stubs.api.container_requests().create.assert_called_with(
 699             body=JsonDiffMatcher(expect_container))
 700         self.assertEqual(stubs.capture_stdout.getvalue(),
 701                          stubs.expect_container_request_uuid + '\n')
 702         self.assertEqual(exited, 0)
 703
 704     @mock.patch("time.sleep")
 705     @stubs
 706     def test_submit_arvworkflow(self, stubs, tm):
 707         with open("tests/wf/expect_arvworkflow.cwl") as f:
 708             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 709
 710         exited = arvados_cwl.main(
 711             ["--submit", "--no-wait", "--api=containers", "--debug",
 712              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 713             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 714
 715         expect_container = {
 716             'priority': 500,
 717             'mounts': {
 718                 '/var/spool/cwl': {
 719                     'writable': True,
 720                     'kind': 'collection'
 721                 },
 722                 'stdout': {
 723                     'path': '/var/spool/cwl/cwl.output.json',
 724                     'kind': 'file'
 725                 },
 726                 '/var/lib/cwl/workflow.json': {
 727                     'kind': 'json',
 728                     'content': {
 729                         'cwlVersion': 'v1.0',
 730                         '$graph': [
 731                             {
 732                                 'id': '#main',
 733                                 'inputs': [
 734                                     {'type': 'string', 'id': '#main/x'}
 735                                 ],
 736                                 'steps': [
 737                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 738                                      'run': '#submit_tool.cwl',
 739                                      'id': '#main/step1',
 740                                      'out': []}
 741                                 ],
 742                                 'class': 'Workflow',
 743                                 'outputs': []
 744                             },
 745                             {
 746                                 'inputs': [
 747                                     {
 748                                         'inputBinding': {'position': 1},
 749                                         'type': 'string',
 750                                         'id': '#submit_tool.cwl/x'}
 751                                 ],
 752                                 'requirements': [
 753                                     {
 754                                         'dockerPull': 'debian:8',
 755                                         'class': 'DockerRequirement',
 756                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 757                                     }
 758                                 ],
 759                                 'id': '#submit_tool.cwl',
 760                                 'outputs': [],
 761                                 'baseCommand': 'cat',
 762                                 'class': 'CommandLineTool'
 763                             }
 764                         ]
 765                     }
 766                 },
 767                 '/var/lib/cwl/cwl.input.json': {
 768                     'content': {
 769                         'x': 'XxX'
 770                     },
 771                     'kind': 'json'
 772                 }
 773             }, 'state': 'Committed',
 774             'output_path': '/var/spool/cwl',
 775             'name': 'a test workflow',
 776             'container_image': "999999999999999999999999999999d3+99",
 777             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 778                         '--no-log-timestamps', '--disable-validate',
 779                         '--eval-timeout=20', '--thread-count=1',
 780                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 781                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 782             'cwd': '/var/spool/cwl',
 783             'runtime_constraints': {
 784                 'API': True,
 785                 'vcpus': 1,
 786                 'ram': 1342177280
 787             },
 788             'use_existing': False,
 789             'properties': {
 790                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 791             },
 792             'secret_mounts': {}
 793         }
 794
 795         stubs.api.container_requests().create.assert_called_with(
 796             body=JsonDiffMatcher(expect_container))
 797         self.assertEqual(stubs.capture_stdout.getvalue(),
 798                          stubs.expect_container_request_uuid + '\n')
 799         self.assertEqual(exited, 0)
 800
 801     @stubs
 802     def test_submit_container_name(self, stubs):
 803         exited = arvados_cwl.main(
 804             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 805                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 806             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 807
 808         expect_container = copy.deepcopy(stubs.expect_container_spec)
 809         expect_container["name"] = "hello container 123"
 810
 811         stubs.api.container_requests().create.assert_called_with(
 812             body=JsonDiffMatcher(expect_container))
 813         self.assertEqual(stubs.capture_stdout.getvalue(),
 814                          stubs.expect_container_request_uuid + '\n')
 815         self.assertEqual(exited, 0)
 816
 817     @stubs
 818     def test_submit_missing_input(self, stubs):
 819         exited = arvados_cwl.main(
 820             ["--submit", "--no-wait", "--api=containers", "--debug",
 821              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 822             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 823         self.assertEqual(exited, 0)
 824
 825         exited = arvados_cwl.main(
 826             ["--submit", "--no-wait", "--api=containers", "--debug",
 827              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 828             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 829         self.assertEqual(exited, 1)
 830
 831     @stubs
 832     def test_submit_container_project(self, stubs):
 833         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 834         exited = arvados_cwl.main(
 835             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 836                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 837             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 838
 839         expect_container = copy.deepcopy(stubs.expect_container_spec)
 840         expect_container["owner_uuid"] = project_uuid
 841         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 842                                        '--no-log-timestamps', '--disable-validate',
 843                                        "--eval-timeout=20", "--thread-count=1",
 844                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 845                                        '--on-error=continue',
 846                                        '--project-uuid='+project_uuid,
 847                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 848
 849         stubs.api.container_requests().create.assert_called_with(
 850             body=JsonDiffMatcher(expect_container))
 851         self.assertEqual(stubs.capture_stdout.getvalue(),
 852                          stubs.expect_container_request_uuid + '\n')
 853         self.assertEqual(exited, 0)
 854
 855     @stubs
 856     def test_submit_container_eval_timeout(self, stubs):
 857         exited = arvados_cwl.main(
 858             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 859                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 860             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 861
 862         expect_container = copy.deepcopy(stubs.expect_container_spec)
 863         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 864                                        '--no-log-timestamps', '--disable-validate',
 865                                        '--eval-timeout=60.0', '--thread-count=1',
 866                                        '--enable-reuse', "--collection-cache-size=256",
 867                                        '--debug', '--on-error=continue',
 868                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 869
 870         stubs.api.container_requests().create.assert_called_with(
 871             body=JsonDiffMatcher(expect_container))
 872         self.assertEqual(stubs.capture_stdout.getvalue(),
 873                          stubs.expect_container_request_uuid + '\n')
 874         self.assertEqual(exited, 0)
 875
 876     @stubs
 877     def test_submit_container_collection_cache(self, stubs):
 878         exited = arvados_cwl.main(
 879             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 880                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 881             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 882
 883         expect_container = copy.deepcopy(stubs.expect_container_spec)
 884         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 885                                        '--no-log-timestamps', '--disable-validate',
 886                                        '--eval-timeout=20', '--thread-count=1',
 887                                        '--enable-reuse', "--collection-cache-size=500",
 888                                        '--debug', '--on-error=continue',
 889                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 890         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 891
 892         stubs.api.container_requests().create.assert_called_with(
 893             body=JsonDiffMatcher(expect_container))
 894         self.assertEqual(stubs.capture_stdout.getvalue(),
 895                          stubs.expect_container_request_uuid + '\n')
 896         self.assertEqual(exited, 0)
 897
 898     @stubs
 899     def test_submit_container_thread_count(self, stubs):
 900         exited = arvados_cwl.main(
 901             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 902                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904
 905         expect_container = copy.deepcopy(stubs.expect_container_spec)
 906         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 907                                        '--no-log-timestamps', '--disable-validate',
 908                                        '--eval-timeout=20', '--thread-count=20',
 909                                        '--enable-reuse', "--collection-cache-size=256",
 910                                        '--debug', '--on-error=continue',
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(stubs.capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917         self.assertEqual(exited, 0)
 918
 919     @stubs
 920     def test_submit_container_runner_image(self, stubs):
 921         exited = arvados_cwl.main(
 922             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 923                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 924             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 925
 926         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 927
 928         expect_container = copy.deepcopy(stubs.expect_container_spec)
 929         stubs.api.container_requests().create.assert_called_with(
 930             body=JsonDiffMatcher(expect_container))
 931         self.assertEqual(stubs.capture_stdout.getvalue(),
 932                          stubs.expect_container_request_uuid + '\n')
 933         self.assertEqual(exited, 0)
 934
 935     @stubs
 936     def test_submit_priority(self, stubs):
 937         exited = arvados_cwl.main(
 938             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 939                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 940             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 941
 942         stubs.expect_container_spec["priority"] = 669
 943
 944         expect_container = copy.deepcopy(stubs.expect_container_spec)
 945         stubs.api.container_requests().create.assert_called_with(
 946             body=JsonDiffMatcher(expect_container))
 947         self.assertEqual(stubs.capture_stdout.getvalue(),
 948                          stubs.expect_container_request_uuid + '\n')
 949         self.assertEqual(exited, 0)
 950
 951     @stubs
 952     def test_submit_wf_runner_resources(self, stubs):
 953         exited = arvados_cwl.main(
 954             ["--submit", "--no-wait", "--api=containers", "--debug",
 955                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 956             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 957
 958         expect_container = copy.deepcopy(stubs.expect_container_spec)
 959         expect_container["runtime_constraints"] = {
 960             "API": True,
 961             "vcpus": 2,
 962             "ram": (2000+512) * 2**20
 963         }
 964         expect_container["name"] = "submit_wf_runner_resources.cwl"
 965         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 966             {
 967                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 968                 "coresMin": 2,
 969                 "ramMin": 2000,
 970                 "keep_cache": 512
 971             }
 972         ]
 973         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 974             "arv": "http://arvados.org/cwl#",
 975         }
 976         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 977                         '--no-log-timestamps', '--disable-validate',
 978                         '--eval-timeout=20', '--thread-count=1',
 979                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
 980                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 981
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     def tearDown(self):
 989         arvados_cwl.arvdocker.arv_docker_clear_cache()
 990
 991     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 992     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
 993     @mock.patch("arvados.api")
 994     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 995         arvados_cwl.arvdocker.arv_docker_clear_cache()
 996
 997         arvrunner = mock.MagicMock()
 998         arvrunner.project_uuid = ""
 999         api.return_value = mock.MagicMock()
1000         arvrunner.api = api.return_value
1001         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1002                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1003                                                                         "link_class": "docker_image_repo+tag",
1004                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1005                                                                         "owner_uuid": "",
1006                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1007                                                             {"items": [{"created_at": "",
1008                                                                         "head_uuid": "",
1009                                                                         "link_class": "docker_image_hash",
1010                                                                         "name": "123456",
1011                                                                         "owner_uuid": "",
1012                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1013         )
1014         find_one_image_hash.return_value = "123456"
1015
1016         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1017                                                                               "owner_uuid": "",
1018                                                                               "manifest_text": "",
1019                                                                               "properties": ""
1020                                                                           }], "items_available": 1, "offset": 0},)
1021         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1022         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1023                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1024         self.assertEqual("9999999999999999999999999999999b+99",
1025                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1026
1027
1028     @stubs
1029     def test_submit_secrets(self, stubs):
1030         exited = arvados_cwl.main(
1031             ["--submit", "--no-wait", "--api=containers", "--debug",
1032                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1033             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1034
1035         expect_container = {
1036             "command": [
1037                 "arvados-cwl-runner",
1038                 "--local",
1039                 "--api=containers",
1040                 "--no-log-timestamps",
1041                 "--disable-validate",
1042                 "--eval-timeout=20",
1043                 '--thread-count=1',
1044                 "--enable-reuse",
1045                 "--collection-cache-size=256",
1046                 '--debug',
1047                 "--on-error=continue",
1048                 "/var/lib/cwl/workflow.json#main",
1049                 "/var/lib/cwl/cwl.input.json"
1050             ],
1051             "container_image": "999999999999999999999999999999d3+99",
1052             "cwd": "/var/spool/cwl",
1053             "mounts": {
1054                 "/var/lib/cwl/cwl.input.json": {
1055                     "content": {
1056                         "pw": {
1057                             "$include": "/secrets/s0"
1058                         }
1059                     },
1060                     "kind": "json"
1061                 },
1062                 "/var/lib/cwl/workflow.json": {
1063                     "content": {
1064                         "$graph": [
1065                             {
1066                                 "$namespaces": {
1067                                     "cwltool": "http://commonwl.org/cwltool#"
1068                                 },
1069                                 "arguments": [
1070                                     "md5sum",
1071                                     "example.conf"
1072                                 ],
1073                                 "class": "CommandLineTool",
1074                                 "hints": [
1075                                     {
1076                                         "class": "http://commonwl.org/cwltool#Secrets",
1077                                         "secrets": [
1078                                             "#secret_job.cwl/pw"
1079                                         ]
1080                                     }
1081                                 ],
1082                                 "id": "#secret_job.cwl",
1083                                 "inputs": [
1084                                     {
1085                                         "id": "#secret_job.cwl/pw",
1086                                         "type": "string"
1087                                     }
1088                                 ],
1089                                 "outputs": [
1090                                     {
1091                                         "id": "#secret_job.cwl/out",
1092                                         "type": "stdout"
1093                                     }
1094                                 ],
1095                                 "stdout": "hashed_example.txt",
1096                                 "requirements": [
1097                                     {
1098                                         "class": "InitialWorkDirRequirement",
1099                                         "listing": [
1100                                             {
1101                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1102                                                 "entryname": "example.conf"
1103                                             }
1104                                         ]
1105                                     }
1106                                 ]
1107                             },
1108                             {
1109                                 "class": "Workflow",
1110                                 "hints": [
1111                                     {
1112                                         "class": "DockerRequirement",
1113                                         "dockerPull": "debian:8",
1114                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1115                                     },
1116                                     {
1117                                         "class": "http://commonwl.org/cwltool#Secrets",
1118                                         "secrets": [
1119                                             "#main/pw"
1120                                         ]
1121                                     }
1122                                 ],
1123                                 "id": "#main",
1124                                 "inputs": [
1125                                     {
1126                                         "id": "#main/pw",
1127                                         "type": "string"
1128                                     }
1129                                 ],
1130                                 "outputs": [
1131                                     {
1132                                         "id": "#main/out",
1133                                         "outputSource": "#main/step1/out",
1134                                         "type": "File"
1135                                     }
1136                                 ],
1137                                 "steps": [
1138                                     {
1139                                         "id": "#main/step1",
1140                                         "in": [
1141                                             {
1142                                                 "id": "#main/step1/pw",
1143                                                 "source": "#main/pw"
1144                                             }
1145                                         ],
1146                                         "out": [
1147                                             "#main/step1/out"
1148                                         ],
1149                                         "run": "#secret_job.cwl"
1150                                     }
1151                                 ]
1152                             }
1153                         ],
1154                         "cwlVersion": "v1.0"
1155                     },
1156                     "kind": "json"
1157                 },
1158                 "/var/spool/cwl": {
1159                     "kind": "collection",
1160                     "writable": True
1161                 },
1162                 "stdout": {
1163                     "kind": "file",
1164                     "path": "/var/spool/cwl/cwl.output.json"
1165                 }
1166             },
1167             "name": "secret_wf.cwl",
1168             "output_path": "/var/spool/cwl",
1169             "priority": 500,
1170             "properties": {},
1171             "runtime_constraints": {
1172                 "API": True,
1173                 "ram": 1342177280,
1174                 "vcpus": 1
1175             },
1176             "secret_mounts": {
1177                 "/secrets/s0": {
1178                     "content": "blorp",
1179                     "kind": "text"
1180                 }
1181             },
1182             "state": "Committed",
1183             "use_existing": False
1184         }
1185
1186         stubs.api.container_requests().create.assert_called_with(
1187             body=JsonDiffMatcher(expect_container))
1188         self.assertEqual(stubs.capture_stdout.getvalue(),
1189                          stubs.expect_container_request_uuid + '\n')
1190         self.assertEqual(exited, 0)
1191
1192     @stubs
1193     def test_submit_request_uuid(self, stubs):
1194         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1195         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1196
1197         stubs.api.container_requests().update().execute.return_value = {
1198             "uuid": stubs.expect_container_request_uuid,
1199             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1200             "state": "Queued"
1201         }
1202
1203         exited = arvados_cwl.main(
1204             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1205                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1206             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1207
1208         stubs.api.container_requests().update.assert_called_with(
1209             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1210         self.assertEqual(stubs.capture_stdout.getvalue(),
1211                          stubs.expect_container_request_uuid + '\n')
1212         self.assertEqual(exited, 0)
1213
1214     @stubs
1215     def test_submit_container_cluster_id(self, stubs):
1216         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1217
1218         exited = arvados_cwl.main(
1219             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1220                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1221             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1222
1223         expect_container = copy.deepcopy(stubs.expect_container_spec)
1224
1225         stubs.api.container_requests().create.assert_called_with(
1226             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1227         self.assertEqual(stubs.capture_stdout.getvalue(),
1228                          stubs.expect_container_request_uuid + '\n')
1229         self.assertEqual(exited, 0)
1230
1231     @stubs
1232     def test_submit_validate_cluster_id(self, stubs):
1233         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1234         exited = arvados_cwl.main(
1235             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1236              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1237             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1238         self.assertEqual(exited, 1)
1239
1240     @mock.patch("arvados.collection.CollectionReader")
1241     @stubs
1242     def test_submit_uuid_inputs(self, stubs, collectionReader):
1243         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1244         def list_side_effect(**kwargs):
1245             m = mock.MagicMock()
1246             if "count" in kwargs:
1247                 m.execute.return_value = {"items": [
1248                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1249                 ]}
1250             else:
1251                 m.execute.return_value = {"items": []}
1252             return m
1253         stubs.api.collections().list.side_effect = list_side_effect
1254
1255         exited = arvados_cwl.main(
1256             ["--submit", "--no-wait", "--api=containers", "--debug",
1257                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1258             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1259
1260         expect_container = copy.deepcopy(stubs.expect_container_spec)
1261         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1262         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1263         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1264
1265         stubs.api.collections().list.assert_has_calls([
1266             mock.call(count='none',
1267                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1268                       select=['uuid', 'portable_data_hash'])])
1269         stubs.api.container_requests().create.assert_called_with(
1270             body=JsonDiffMatcher(expect_container))
1271         self.assertEqual(stubs.capture_stdout.getvalue(),
1272                          stubs.expect_container_request_uuid + '\n')
1273         self.assertEqual(exited, 0)
1274
1275     @stubs
1276     def test_submit_mismatched_uuid_inputs(self, stubs):
1277         def list_side_effect(**kwargs):
1278             m = mock.MagicMock()
1279             if "count" in kwargs:
1280                 m.execute.return_value = {"items": [
1281                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1282                 ]}
1283             else:
1284                 m.execute.return_value = {"items": []}
1285             return m
1286         stubs.api.collections().list.side_effect = list_side_effect
1287
1288         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1289             capture_stderr = StringIO()
1290             cwltool_logger = logging.getLogger('cwltool')
1291             stderr_logger = logging.StreamHandler(capture_stderr)
1292             cwltool_logger.addHandler(stderr_logger)
1293
1294             try:
1295                 exited = arvados_cwl.main(
1296                     ["--submit", "--no-wait", "--api=containers", "--debug",
1297                         "tests/wf/submit_wf.cwl", infile],
1298                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1299
1300                 self.assertEqual(exited, 1)
1301                 self.assertRegexpMatches(
1302                     capture_stderr.getvalue(),
1303                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1304             finally:
1305                 cwltool_logger.removeHandler(stderr_logger)
1306
1307     @mock.patch("arvados.collection.CollectionReader")
1308     @stubs
1309     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1310         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1311         capture_stderr = StringIO()
1312
1313         cwltool_logger = logging.getLogger('cwltool')
1314         stderr_logger = logging.StreamHandler(capture_stderr)
1315         cwltool_logger.addHandler(stderr_logger)
1316
1317         exited = arvados_cwl.main(
1318             ["--submit", "--no-wait", "--api=containers", "--debug",
1319                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1320             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1321
1322         try:
1323             self.assertEqual(exited, 1)
1324             self.assertRegexpMatches(
1325                 capture_stderr.getvalue(),
1326                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1327         finally:
1328             cwltool_logger.removeHandler(stderr_logger)
1329
1330
1331 class TestCreateWorkflow(unittest.TestCase):
1332     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1333     expect_workflow = StripYAMLComments(
1334         open("tests/wf/expect_packed.cwl").read())
1335
1336     @stubs
1337     def test_create(self, stubs):
1338         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1339
1340         exited = arvados_cwl.main(
1341             ["--create-workflow", "--debug",
1342              "--api=containers",
1343              "--project-uuid", project_uuid,
1344              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1345             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1346
1347         stubs.api.pipeline_templates().create.refute_called()
1348         stubs.api.container_requests().create.refute_called()
1349
1350         body = {
1351             "workflow": {
1352                 "owner_uuid": project_uuid,
1353                 "name": "submit_wf.cwl",
1354                 "description": "",
1355                 "definition": self.expect_workflow,
1356             }
1357         }
1358         stubs.api.workflows().create.assert_called_with(
1359             body=JsonDiffMatcher(body))
1360
1361         self.assertEqual(stubs.capture_stdout.getvalue(),
1362                          stubs.expect_workflow_uuid + '\n')
1363         self.assertEqual(exited, 0)
1364
1365     @stubs
1366     def test_create_name(self, stubs):
1367         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1368
1369         exited = arvados_cwl.main(
1370             ["--create-workflow", "--debug",
1371              "--api=containers",
1372              "--project-uuid", project_uuid,
1373              "--name", "testing 123",
1374              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1375             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1376
1377         stubs.api.pipeline_templates().create.refute_called()
1378         stubs.api.container_requests().create.refute_called()
1379
1380         body = {
1381             "workflow": {
1382                 "owner_uuid": project_uuid,
1383                 "name": "testing 123",
1384                 "description": "",
1385                 "definition": self.expect_workflow,
1386             }
1387         }
1388         stubs.api.workflows().create.assert_called_with(
1389             body=JsonDiffMatcher(body))
1390
1391         self.assertEqual(stubs.capture_stdout.getvalue(),
1392                          stubs.expect_workflow_uuid + '\n')
1393         self.assertEqual(exited, 0)
1394
1395
1396     @stubs
1397     def test_update(self, stubs):
1398         exited = arvados_cwl.main(
1399             ["--update-workflow", self.existing_workflow_uuid,
1400              "--debug",
1401              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1402             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1403
1404         body = {
1405             "workflow": {
1406                 "name": "submit_wf.cwl",
1407                 "description": "",
1408                 "definition": self.expect_workflow,
1409             }
1410         }
1411         stubs.api.workflows().update.assert_called_with(
1412             uuid=self.existing_workflow_uuid,
1413             body=JsonDiffMatcher(body))
1414         self.assertEqual(stubs.capture_stdout.getvalue(),
1415                          self.existing_workflow_uuid + '\n')
1416         self.assertEqual(exited, 0)
1417
1418     @stubs
1419     def test_update_name(self, stubs):
1420         exited = arvados_cwl.main(
1421             ["--update-workflow", self.existing_workflow_uuid,
1422              "--debug", "--name", "testing 123",
1423              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1424             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1425
1426         body = {
1427             "workflow": {
1428                 "name": "testing 123",
1429                 "description": "",
1430                 "definition": self.expect_workflow,
1431             }
1432         }
1433         stubs.api.workflows().update.assert_called_with(
1434             uuid=self.existing_workflow_uuid,
1435             body=JsonDiffMatcher(body))
1436         self.assertEqual(stubs.capture_stdout.getvalue(),
1437                          self.existing_workflow_uuid + '\n')
1438         self.assertEqual(exited, 0)
1439
1440     @stubs
1441     def test_create_collection_per_tool(self, stubs):
1442         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1443
1444         exited = arvados_cwl.main(
1445             ["--create-workflow", "--debug",
1446              "--api=containers",
1447              "--project-uuid", project_uuid,
1448              "tests/collection_per_tool/collection_per_tool.cwl"],
1449             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1450
1451         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1452         expect_workflow = StripYAMLComments(open(toolfile).read())
1453
1454         body = {
1455             "workflow": {
1456                 "owner_uuid": project_uuid,
1457                 "name": "collection_per_tool.cwl",
1458                 "description": "",
1459                 "definition": expect_workflow,
1460             }
1461         }
1462         stubs.api.workflows().create.assert_called_with(
1463             body=JsonDiffMatcher(body))
1464
1465         self.assertEqual(stubs.capture_stdout.getvalue(),
1466                          stubs.expect_workflow_uuid + '\n')
1467         self.assertEqual(exited, 0)
1468
1469     @stubs
1470     def test_create_with_imports(self, stubs):
1471         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1472
1473         exited = arvados_cwl.main(
1474             ["--create-workflow", "--debug",
1475              "--api=containers",
1476              "--project-uuid", project_uuid,
1477              "tests/wf/feddemo/feddemo.cwl"],
1478             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1479
1480         stubs.api.pipeline_templates().create.refute_called()
1481         stubs.api.container_requests().create.refute_called()
1482
1483         self.assertEqual(stubs.capture_stdout.getvalue(),
1484                          stubs.expect_workflow_uuid + '\n')
1485         self.assertEqual(exited, 0)
1486
1487     @stubs
1488     def test_create_with_no_input(self, stubs):
1489         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1490
1491         exited = arvados_cwl.main(
1492             ["--create-workflow", "--debug",
1493              "--api=containers",
1494              "--project-uuid", project_uuid,
1495              "tests/wf/revsort/revsort.cwl"],
1496             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1497
1498         stubs.api.pipeline_templates().create.refute_called()
1499         stubs.api.container_requests().create.refute_called()
1500
1501         self.assertEqual(stubs.capture_stdout.getvalue(),
1502                          stubs.expect_workflow_uuid + '\n')
1503         self.assertEqual(exited, 0)