sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 import ruamel.yaml as yaml
  21
  22 _rootDesc = None
  23
  24 def stubs(func):
  25     @functools.wraps(func)
  26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  27     @mock.patch("arvados.collection.KeepClient")
  28     @mock.patch("arvados.keep.KeepClient")
  29     @mock.patch("arvados.events.subscribe")
  30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  31         class Stubs:
  32             pass
  33         stubs = Stubs()
  34         stubs.events = events
  35         stubs.keepdocker = keepdocker
  36
  37         def putstub(p, **kwargs):
  38             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  39         keep_client1().put.side_effect = putstub
  40         keep_client1.put.side_effect = putstub
  41         keep_client2().put.side_effect = putstub
  42         keep_client2.put.side_effect = putstub
  43
  44         stubs.keep_client = keep_client2
  45         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  46         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  47
  48         stubs.api = mock.MagicMock()
  49         stubs.api._rootDesc = get_rootDesc()
  50
  51         stubs.api.users().current().execute.return_value = {
  52             "uuid": stubs.fake_user_uuid,
  53         }
  54         stubs.api.collections().list().execute.return_value = {"items": []}
  55
  56         class CollectionExecute(object):
  57             def __init__(self, exe):
  58                 self.exe = exe
  59             def execute(self, num_retries=None):
  60                 return self.exe
  61
  62         def collection_createstub(created_collections, body, ensure_unique_name=None):
  63             mt = body["manifest_text"]
  64             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  65             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  66             created_collections[uuid] = {
  67                 "uuid": uuid,
  68                 "portable_data_hash": pdh,
  69                 "manifest_text": mt
  70             }
  71             return CollectionExecute(created_collections[uuid])
  72
  73         def collection_getstub(created_collections, uuid):
  74             for v in created_collections.itervalues():
  75                 if uuid in (v["uuid"], v["portable_data_hash"]):
  76                     return CollectionExecute(v)
  77
  78         created_collections = {}
  79         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  80         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 102                 'min_ram_mb_per_node': 1024
 103             },
 104             'script_parameters': {
 105                 'x': {
 106                     'basename': 'blorp.txt',
 107                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 108                     'class': 'File'
 109                 },
 110                 'y': {
 111                     'basename': '99999999999999999999999999999998+99',
 112                     'location': 'keep:99999999999999999999999999999998+99',
 113                     'class': 'Directory'
 114                 },
 115                 'z': {
 116                     'basename': 'anonymous',
 117                     "listing": [{
 118                         "basename": "renamed.txt",
 119                         "class": "File",
 120                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 121                     }],
 122                     'class': 'Directory'
 123                 },
 124                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 125             },
 126             'repository': 'arvados',
 127             'script_version': 'master',
 128             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 129             'script': 'cwl-runner'
 130         }
 131         stubs.pipeline_component = stubs.expect_job_spec.copy()
 132         stubs.expect_pipeline_instance = {
 133             'name': 'submit_wf.cwl',
 134             'state': 'RunningOnServer',
 135             'owner_uuid': None,
 136             "components": {
 137                 "cwl-runner": {
 138                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 139                     'script_parameters': {
 140                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 141                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
 142                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 143                               'listing': [
 144                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 145                               ]}},
 146                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 147                         'arv:enable_reuse': True,
 148                         'arv:on_error': 'continue'
 149                     },
 150                     'repository': 'arvados',
 151                     'script_version': 'master',
 152                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 153                     'script': 'cwl-runner',
 154                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 155                 }
 156             }
 157         }
 158         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 159         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 160         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 161         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 162         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 163             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 164             "state": "Queued"
 165         }
 166         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 167         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 168
 169         with open("tests/wf/submit_wf_packed.cwl") as f:
 170             expect_packed_workflow = yaml.round_trip_load(f)
 171
 172         stubs.expect_container_spec = {
 173             'priority': 1,
 174             'mounts': {
 175                 '/var/spool/cwl': {
 176                     'writable': True,
 177                     'kind': 'collection'
 178                 },
 179                 '/var/lib/cwl/workflow.json': {
 180                     'content': expect_packed_workflow,
 181                     'kind': 'json'
 182                 },
 183                 'stdout': {
 184                     'path': '/var/spool/cwl/cwl.output.json',
 185                     'kind': 'file'
 186                 },
 187                 '/var/lib/cwl/cwl.input.json': {
 188                     'kind': 'json',
 189                     'content': {
 190                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 191                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
 192                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 193                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 194                         ]}
 195                     },
 196                     'kind': 'json'
 197                 }
 198             },
 199             'state': 'Committed',
 200             'owner_uuid': None,
 201             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 202                         '--enable-reuse', '--on-error=continue',
 203                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 204             'name': 'submit_wf.cwl',
 205             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 206             'output_path': '/var/spool/cwl',
 207             'cwd': '/var/spool/cwl',
 208             'runtime_constraints': {
 209                 'API': True,
 210                 'vcpus': 1,
 211                 'ram': 1024*1024*1024
 212             },
 213             "properties": {}
 214         }
 215
 216         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 217         stubs.api.workflows().create().execute.return_value = {
 218             "uuid": stubs.expect_workflow_uuid,
 219         }
 220         def update_mock(**kwargs):
 221             stubs.updated_uuid = kwargs.get('uuid')
 222             return mock.DEFAULT
 223         stubs.api.workflows().update.side_effect = update_mock
 224         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 225             "uuid": stubs.updated_uuid,
 226         }
 227
 228         return func(self, stubs, *args, **kwargs)
 229     return wrapped
 230
 231
 232 class TestSubmit(unittest.TestCase):
 233     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 234     @mock.patch("time.sleep")
 235     @stubs
 236     def test_submit(self, stubs, tm, arvdock):
 237         capture_stdout = cStringIO.StringIO()
 238         exited = arvados_cwl.main(
 239             ["--submit", "--no-wait", "--api=jobs", "--debug",
 240              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 241             capture_stdout, sys.stderr, api_client=stubs.api)
 242         self.assertEqual(exited, 0)
 243
 244         stubs.api.collections().create.assert_has_calls([
 245             mock.call(body=JsonDiffMatcher({
 246                 'manifest_text':
 247                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 248                 'replication_desired': None,
 249                 'name': 'submit_tool.cwl dependencies',
 250             }), ensure_unique_name=True),
 251             mock.call(body=JsonDiffMatcher({
 252                 'manifest_text':
 253                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 254                 'replication_desired': None,
 255                 'name': 'submit_wf.cwl input',
 256             }), ensure_unique_name=True),
 257             mock.call(body=JsonDiffMatcher({
 258                 'manifest_text':
 259                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 260                 'replication_desired': None,
 261                 'name': 'submit_wf.cwl',
 262             }), ensure_unique_name=True)        ])
 263
 264         arvdock.assert_has_calls([
 265             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 266             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 267         ])
 268
 269         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 270         stubs.api.pipeline_instances().create.assert_called_with(
 271             body=JsonDiffMatcher(expect_pipeline))
 272         self.assertEqual(capture_stdout.getvalue(),
 273                          stubs.expect_pipeline_uuid + '\n')
 274
 275
 276     @mock.patch("time.sleep")
 277     @stubs
 278     def test_submit_no_reuse(self, stubs, tm):
 279         capture_stdout = cStringIO.StringIO()
 280         exited = arvados_cwl.main(
 281             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 282              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 283             capture_stdout, sys.stderr, api_client=stubs.api)
 284         self.assertEqual(exited, 0)
 285
 286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 287         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 288
 289         stubs.api.pipeline_instances().create.assert_called_with(
 290             body=JsonDiffMatcher(expect_pipeline))
 291         self.assertEqual(capture_stdout.getvalue(),
 292                          stubs.expect_pipeline_uuid + '\n')
 293
 294     @mock.patch("time.sleep")
 295     @stubs
 296     def test_submit_on_error(self, stubs, tm):
 297         capture_stdout = cStringIO.StringIO()
 298         exited = arvados_cwl.main(
 299             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 301             capture_stdout, sys.stderr, api_client=stubs.api)
 302         self.assertEqual(exited, 0)
 303
 304         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 305         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 306
 307         stubs.api.pipeline_instances().create.assert_called_with(
 308             body=JsonDiffMatcher(expect_pipeline))
 309         self.assertEqual(capture_stdout.getvalue(),
 310                          stubs.expect_pipeline_uuid + '\n')
 311
 312
 313     @mock.patch("time.sleep")
 314     @stubs
 315     def test_submit_runner_ram(self, stubs, tm):
 316         capture_stdout = cStringIO.StringIO()
 317         exited = arvados_cwl.main(
 318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 0)
 322
 323         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 324         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331
 332     @mock.patch("time.sleep")
 333     @stubs
 334     def test_submit_invalid_runner_ram(self, stubs, tm):
 335         capture_stdout = cStringIO.StringIO()
 336         exited = arvados_cwl.main(
 337             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 338              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 339             capture_stdout, sys.stderr, api_client=stubs.api)
 340         self.assertEqual(exited, 1)
 341
 342     @mock.patch("time.sleep")
 343     @stubs
 344     def test_submit_output_name(self, stubs, tm):
 345         output_name = "test_output_name"
 346
 347         capture_stdout = cStringIO.StringIO()
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             capture_stdout, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 0)
 353
 354         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 355         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 356
 357         stubs.api.pipeline_instances().create.assert_called_with(
 358             body=JsonDiffMatcher(expect_pipeline))
 359         self.assertEqual(capture_stdout.getvalue(),
 360                          stubs.expect_pipeline_uuid + '\n')
 361
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_pipeline_name(self, stubs, tm):
 366         capture_stdout = cStringIO.StringIO()
 367         exited = arvados_cwl.main(
 368             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 369              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 370             capture_stdout, sys.stderr, api_client=stubs.api)
 371         self.assertEqual(exited, 0)
 372
 373         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 374         expect_pipeline["name"] = "hello job 123"
 375
 376         stubs.api.pipeline_instances().create.assert_called_with(
 377             body=JsonDiffMatcher(expect_pipeline))
 378         self.assertEqual(capture_stdout.getvalue(),
 379                          stubs.expect_pipeline_uuid + '\n')
 380
 381     @mock.patch("time.sleep")
 382     @stubs
 383     def test_submit_output_tags(self, stubs, tm):
 384         output_tags = "tag0,tag1,tag2"
 385
 386         capture_stdout = cStringIO.StringIO()
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 390             capture_stdout, sys.stderr, api_client=stubs.api)
 391         self.assertEqual(exited, 0)
 392
 393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 395
 396         stubs.api.pipeline_instances().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_pipeline))
 398         self.assertEqual(capture_stdout.getvalue(),
 399                          stubs.expect_pipeline_uuid + '\n')
 400
 401     @mock.patch("time.sleep")
 402     @stubs
 403     def test_submit_with_project_uuid(self, stubs, tm):
 404         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 405
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait",
 408              "--project-uuid", project_uuid,
 409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             sys.stdout, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 414         expect_pipeline["owner_uuid"] = project_uuid
 415         stubs.api.pipeline_instances().create.assert_called_with(
 416             body=JsonDiffMatcher(expect_pipeline))
 417
 418     @stubs
 419     def test_submit_container(self, stubs):
 420         capture_stdout = cStringIO.StringIO()
 421         try:
 422             exited = arvados_cwl.main(
 423                 ["--submit", "--no-wait", "--api=containers", "--debug",
 424                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 426             self.assertEqual(exited, 0)
 427         except:
 428             logging.exception("")
 429
 430         stubs.api.collections().create.assert_has_calls([
 431             mock.call(body=JsonDiffMatcher({
 432                 'manifest_text':
 433                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 434                 'replication_desired': None,
 435                 'name': 'submit_tool.cwl dependencies',
 436             }), ensure_unique_name=True),
 437             mock.call(body=JsonDiffMatcher({
 438                 'manifest_text':
 439                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 440                 'replication_desired': None,
 441                 'name': 'submit_wf.cwl input',
 442             }), ensure_unique_name=True)])
 443
 444         expect_container = copy.deepcopy(stubs.expect_container_spec)
 445         stubs.api.container_requests().create.assert_called_with(
 446             body=JsonDiffMatcher(expect_container))
 447         self.assertEqual(capture_stdout.getvalue(),
 448                          stubs.expect_container_request_uuid + '\n')
 449
 450     @stubs
 451     def test_submit_container_no_reuse(self, stubs):
 452         capture_stdout = cStringIO.StringIO()
 453         try:
 454             exited = arvados_cwl.main(
 455                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 456                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 457                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 458             self.assertEqual(exited, 0)
 459         except:
 460             logging.exception("")
 461
 462         expect_container = copy.deepcopy(stubs.expect_container_spec)
 463         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 464                                                   '--disable-reuse', '--on-error=continue',
 465                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 466
 467         stubs.api.container_requests().create.assert_called_with(
 468             body=JsonDiffMatcher(expect_container))
 469         self.assertEqual(capture_stdout.getvalue(),
 470                          stubs.expect_container_request_uuid + '\n')
 471
 472
 473     @stubs
 474     def test_submit_container_on_error(self, stubs):
 475         capture_stdout = cStringIO.StringIO()
 476         try:
 477             exited = arvados_cwl.main(
 478                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 479                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 480                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 481             self.assertEqual(exited, 0)
 482         except:
 483             logging.exception("")
 484
 485         expect_container = copy.deepcopy(stubs.expect_container_spec)
 486         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 487                                                   '--enable-reuse', '--on-error=stop',
 488                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 489
 490         stubs.api.container_requests().create.assert_called_with(
 491             body=JsonDiffMatcher(expect_container))
 492         self.assertEqual(capture_stdout.getvalue(),
 493                          stubs.expect_container_request_uuid + '\n')
 494
 495     @stubs
 496     def test_submit_container_output_name(self, stubs):
 497         output_name = "test_output_name"
 498
 499         capture_stdout = cStringIO.StringIO()
 500         try:
 501             exited = arvados_cwl.main(
 502                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 503                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505             self.assertEqual(exited, 0)
 506         except:
 507             logging.exception("")
 508
 509         expect_container = copy.deepcopy(stubs.expect_container_spec)
 510         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 511                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 512                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 513         expect_container["output_name"] = output_name
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519
 520
 521     @stubs
 522     def test_submit_container_output_ttl(self, stubs):
 523         capture_stdout = cStringIO.StringIO()
 524         try:
 525             exited = arvados_cwl.main(
 526                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 527                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 528                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 529             self.assertEqual(exited, 0)
 530         except:
 531             logging.exception("")
 532
 533         expect_container = copy.deepcopy(stubs.expect_container_spec)
 534         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 535                                        '--enable-reuse', '--on-error=continue',
 536                                        "--intermediate-output-ttl=3600",
 537                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 538
 539         stubs.api.container_requests().create.assert_called_with(
 540             body=JsonDiffMatcher(expect_container))
 541         self.assertEqual(capture_stdout.getvalue(),
 542                          stubs.expect_container_request_uuid + '\n')
 543
 544     @stubs
 545     def test_submit_container_trash_intermediate(self, stubs):
 546         capture_stdout = cStringIO.StringIO()
 547         try:
 548             exited = arvados_cwl.main(
 549                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 550                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 551                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 552             self.assertEqual(exited, 0)
 553         except:
 554             logging.exception("")
 555
 556         expect_container = copy.deepcopy(stubs.expect_container_spec)
 557         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 558                                        '--enable-reuse', '--on-error=continue',
 559                                        "--trash-intermediate",
 560                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 561
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566
 567     @stubs
 568     def test_submit_container_output_tags(self, stubs):
 569         output_tags = "tag0,tag1,tag2"
 570
 571         capture_stdout = cStringIO.StringIO()
 572         try:
 573             exited = arvados_cwl.main(
 574                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 575                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 576                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 577             self.assertEqual(exited, 0)
 578         except:
 579             logging.exception("")
 580
 581         expect_container = copy.deepcopy(stubs.expect_container_spec)
 582         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 583                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 584                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 585
 586         stubs.api.container_requests().create.assert_called_with(
 587             body=JsonDiffMatcher(expect_container))
 588         self.assertEqual(capture_stdout.getvalue(),
 589                          stubs.expect_container_request_uuid + '\n')
 590
 591     @stubs
 592     def test_submit_container_runner_ram(self, stubs):
 593         capture_stdout = cStringIO.StringIO()
 594         try:
 595             exited = arvados_cwl.main(
 596                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 597                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599             self.assertEqual(exited, 0)
 600         except:
 601             logging.exception("")
 602
 603         expect_container = copy.deepcopy(stubs.expect_container_spec)
 604         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 605
 606         stubs.api.container_requests().create.assert_called_with(
 607             body=JsonDiffMatcher(expect_container))
 608         self.assertEqual(capture_stdout.getvalue(),
 609                          stubs.expect_container_request_uuid + '\n')
 610
 611     @mock.patch("arvados.collection.CollectionReader")
 612     @mock.patch("time.sleep")
 613     @stubs
 614     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 615         capture_stdout = cStringIO.StringIO()
 616         exited = arvados_cwl.main(
 617             ["--submit", "--no-wait", "--api=containers", "--debug",
 618              "tests/wf/submit_keepref_wf.cwl"],
 619             capture_stdout, sys.stderr, api_client=stubs.api)
 620         self.assertEqual(exited, 0)
 621
 622
 623     @mock.patch("arvados.collection.CollectionReader")
 624     @mock.patch("time.sleep")
 625     @stubs
 626     def test_submit_keepref(self, stubs, tm, reader):
 627         capture_stdout = cStringIO.StringIO()
 628
 629         with open("tests/wf/expect_arvworkflow.cwl") as f:
 630             reader().open().__enter__().read.return_value = f.read()
 631
 632         exited = arvados_cwl.main(
 633             ["--submit", "--no-wait", "--api=containers", "--debug",
 634              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 635             capture_stdout, sys.stderr, api_client=stubs.api)
 636         self.assertEqual(exited, 0)
 637
 638         expect_container = {
 639             'priority': 1,
 640             'mounts': {
 641                 '/var/spool/cwl': {
 642                     'writable': True,
 643                     'kind': 'collection'
 644                 },
 645                 'stdout': {
 646                     'path': '/var/spool/cwl/cwl.output.json',
 647                     'kind': 'file'
 648                 },
 649                 '/var/lib/cwl/workflow': {
 650                     'portable_data_hash': '99999999999999999999999999999994+99',
 651                     'kind': 'collection'
 652                 },
 653                 '/var/lib/cwl/cwl.input.json': {
 654                     'content': {
 655                         'x': 'XxX'
 656                     },
 657                     'kind': 'json'
 658                 }
 659             }, 'state': 'Committed',
 660             'owner_uuid': None,
 661             'output_path': '/var/spool/cwl',
 662             'name': 'expect_arvworkflow.cwl#main',
 663             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 664             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 665                         '--enable-reuse', '--on-error=continue',
 666                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 667             'cwd': '/var/spool/cwl',
 668             'runtime_constraints': {
 669                 'API': True,
 670                 'vcpus': 1,
 671                 'ram': 1073741824
 672             },
 673             "properties": {}
 674         }
 675
 676         stubs.api.container_requests().create.assert_called_with(
 677             body=JsonDiffMatcher(expect_container))
 678         self.assertEqual(capture_stdout.getvalue(),
 679                          stubs.expect_container_request_uuid + '\n')
 680
 681
 682     @mock.patch("arvados.collection.CollectionReader")
 683     @mock.patch("time.sleep")
 684     @stubs
 685     def test_submit_jobs_keepref(self, stubs, tm, reader):
 686         capture_stdout = cStringIO.StringIO()
 687
 688         with open("tests/wf/expect_arvworkflow.cwl") as f:
 689             reader().open().__enter__().read.return_value = f.read()
 690
 691         exited = arvados_cwl.main(
 692             ["--submit", "--no-wait", "--api=jobs", "--debug",
 693              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 694             capture_stdout, sys.stderr, api_client=stubs.api)
 695         self.assertEqual(exited, 0)
 696
 697         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 698         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 699         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 700         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 701         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 702         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 703         stubs.api.pipeline_instances().create.assert_called_with(
 704             body=JsonDiffMatcher(expect_pipeline))
 705
 706     @mock.patch("time.sleep")
 707     @stubs
 708     def test_submit_arvworkflow(self, stubs, tm):
 709         capture_stdout = cStringIO.StringIO()
 710
 711         with open("tests/wf/expect_arvworkflow.cwl") as f:
 712             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 713
 714         exited = arvados_cwl.main(
 715             ["--submit", "--no-wait", "--api=containers", "--debug",
 716              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 717             capture_stdout, sys.stderr, api_client=stubs.api)
 718         self.assertEqual(exited, 0)
 719
 720         expect_container = {
 721             'priority': 1,
 722             'mounts': {
 723                 '/var/spool/cwl': {
 724                     'writable': True,
 725                     'kind': 'collection'
 726                 },
 727                 'stdout': {
 728                     'path': '/var/spool/cwl/cwl.output.json',
 729                     'kind': 'file'
 730                 },
 731                 '/var/lib/cwl/workflow.json': {
 732                     'kind': 'json',
 733                     'content': {
 734                         'cwlVersion': 'v1.0',
 735                         '$graph': [
 736                             {
 737                                 'id': '#main',
 738                                 'inputs': [
 739                                     {'type': 'string', 'id': '#main/x'}
 740                                 ],
 741                                 'steps': [
 742                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 743                                      'run': '#submit_tool.cwl',
 744                                      'id': '#main/step1',
 745                                      'out': []}
 746                                 ],
 747                                 'class': 'Workflow',
 748                                 'outputs': []
 749                             },
 750                             {
 751                                 'inputs': [
 752                                     {
 753                                         'inputBinding': {'position': 1},
 754                                         'type': 'string',
 755                                         'id': '#submit_tool.cwl/x'}
 756                                 ],
 757                                 'requirements': [
 758                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 759                                 ],
 760                                 'id': '#submit_tool.cwl',
 761                                 'outputs': [],
 762                                 'baseCommand': 'cat',
 763                                 'class': 'CommandLineTool'
 764                             }
 765                         ]
 766                     }
 767                 },
 768                 '/var/lib/cwl/cwl.input.json': {
 769                     'content': {
 770                         'x': 'XxX'
 771                     },
 772                     'kind': 'json'
 773                 }
 774             }, 'state': 'Committed',
 775             'owner_uuid': None,
 776             'output_path': '/var/spool/cwl',
 777             'name': 'a test workflow',
 778             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 779             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 780                         '--enable-reuse', '--on-error=continue',
 781                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 782             'cwd': '/var/spool/cwl',
 783             'runtime_constraints': {
 784                 'API': True,
 785                 'vcpus': 1,
 786                 'ram': 1073741824
 787             },
 788             "properties": {
 789                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 790             }
 791         }
 792
 793         stubs.api.container_requests().create.assert_called_with(
 794             body=JsonDiffMatcher(expect_container))
 795         self.assertEqual(capture_stdout.getvalue(),
 796                          stubs.expect_container_request_uuid + '\n')
 797
 798
 799     @stubs
 800     def test_submit_container_name(self, stubs):
 801         capture_stdout = cStringIO.StringIO()
 802         try:
 803             exited = arvados_cwl.main(
 804                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 805                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 806                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 807             self.assertEqual(exited, 0)
 808         except:
 809             logging.exception("")
 810
 811         expect_container = copy.deepcopy(stubs.expect_container_spec)
 812         expect_container["name"] = "hello container 123"
 813
 814         stubs.api.container_requests().create.assert_called_with(
 815             body=JsonDiffMatcher(expect_container))
 816         self.assertEqual(capture_stdout.getvalue(),
 817                          stubs.expect_container_request_uuid + '\n')
 818
 819
 820     @stubs
 821     def test_submit_job_runner_image(self, stubs):
 822         capture_stdout = cStringIO.StringIO()
 823         try:
 824             exited = arvados_cwl.main(
 825                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 826                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 827                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 828             self.assertEqual(exited, 0)
 829         except:
 830             logging.exception("")
 831
 832         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 833
 834         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 835         stubs.api.pipeline_instances().create.assert_called_with(
 836             body=JsonDiffMatcher(expect_pipeline))
 837         self.assertEqual(capture_stdout.getvalue(),
 838                          stubs.expect_pipeline_uuid + '\n')
 839
 840     @stubs
 841     def test_submit_container_runner_image(self, stubs):
 842         capture_stdout = cStringIO.StringIO()
 843         try:
 844             exited = arvados_cwl.main(
 845                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 846                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 847                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 848             self.assertEqual(exited, 0)
 849         except:
 850             logging.exception("")
 851
 852         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 853
 854         expect_container = copy.deepcopy(stubs.expect_container_spec)
 855         stubs.api.container_requests().create.assert_called_with(
 856             body=JsonDiffMatcher(expect_container))
 857         self.assertEqual(capture_stdout.getvalue(),
 858                          stubs.expect_container_request_uuid + '\n')
 859
 860
 861     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 862     @mock.patch("cwltool.docker.get_image")
 863     @mock.patch("arvados.api")
 864     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 865         arvrunner = mock.MagicMock()
 866         arvrunner.project_uuid = ""
 867         api.return_value = mock.MagicMock()
 868         arvrunner.api = api.return_value
 869         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 870                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 871                                                                         "link_class": "docker_image_repo+tag",
 872                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 873                                                                         "owner_uuid": "",
 874                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 875                                                             {"items": [{"created_at": "",
 876                                                                         "head_uuid": "",
 877                                                                         "link_class": "docker_image_hash",
 878                                                                         "name": "123456",
 879                                                                         "owner_uuid": "",
 880                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 881         )
 882         find_one_image_hash.return_value = "123456"
 883
 884         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 885                                                                               "owner_uuid": "",
 886                                                                               "manifest_text": "",
 887                                                                               "properties": ""
 888                                                                           }], "items_available": 1, "offset": 0},)
 889         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 890         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 891                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 892
 893 class TestCreateTemplate(unittest.TestCase):
 894     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 895
 896     def _adjust_script_params(self, expect_component):
 897         expect_component['script_parameters']['x'] = {
 898             'dataclass': 'File',
 899             'required': True,
 900             'type': 'File',
 901             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 902         }
 903         expect_component['script_parameters']['y'] = {
 904             'dataclass': 'Collection',
 905             'required': True,
 906             'type': 'Directory',
 907             'value': '99999999999999999999999999999998+99',
 908         }
 909         expect_component['script_parameters']['z'] = {
 910             'dataclass': 'Collection',
 911             'required': True,
 912             'type': 'Directory',
 913         }
 914
 915     @stubs
 916     def test_create(self, stubs):
 917         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 918
 919         capture_stdout = cStringIO.StringIO()
 920
 921         exited = arvados_cwl.main(
 922             ["--create-workflow", "--debug",
 923              "--api=jobs",
 924              "--project-uuid", project_uuid,
 925              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 926             capture_stdout, sys.stderr, api_client=stubs.api)
 927         self.assertEqual(exited, 0)
 928
 929         stubs.api.pipeline_instances().create.refute_called()
 930         stubs.api.jobs().create.refute_called()
 931
 932         expect_component = copy.deepcopy(stubs.expect_job_spec)
 933         self._adjust_script_params(expect_component)
 934         expect_template = {
 935             "components": {
 936                 "submit_wf.cwl": expect_component,
 937             },
 938             "name": "submit_wf.cwl",
 939             "owner_uuid": project_uuid,
 940         }
 941         stubs.api.pipeline_templates().create.assert_called_with(
 942             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 943
 944         self.assertEqual(capture_stdout.getvalue(),
 945                          stubs.expect_pipeline_template_uuid + '\n')
 946
 947
 948     @stubs
 949     def test_create_name(self, stubs):
 950         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 951
 952         capture_stdout = cStringIO.StringIO()
 953
 954         exited = arvados_cwl.main(
 955             ["--create-workflow", "--debug",
 956              "--project-uuid", project_uuid,
 957              "--api=jobs",
 958              "--name", "testing 123",
 959              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 960             capture_stdout, sys.stderr, api_client=stubs.api)
 961         self.assertEqual(exited, 0)
 962
 963         stubs.api.pipeline_instances().create.refute_called()
 964         stubs.api.jobs().create.refute_called()
 965
 966         expect_component = copy.deepcopy(stubs.expect_job_spec)
 967         self._adjust_script_params(expect_component)
 968         expect_template = {
 969             "components": {
 970                 "testing 123": expect_component,
 971             },
 972             "name": "testing 123",
 973             "owner_uuid": project_uuid,
 974         }
 975         stubs.api.pipeline_templates().create.assert_called_with(
 976             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 977
 978         self.assertEqual(capture_stdout.getvalue(),
 979                          stubs.expect_pipeline_template_uuid + '\n')
 980
 981
 982     @stubs
 983     def test_update_name(self, stubs):
 984         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 985
 986         capture_stdout = cStringIO.StringIO()
 987
 988         exited = arvados_cwl.main(
 989             ["--update-workflow", self.existing_template_uuid,
 990              "--debug",
 991              "--project-uuid", project_uuid,
 992              "--api=jobs",
 993              "--name", "testing 123",
 994              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 995             capture_stdout, sys.stderr, api_client=stubs.api)
 996         self.assertEqual(exited, 0)
 997
 998         stubs.api.pipeline_instances().create.refute_called()
 999         stubs.api.jobs().create.refute_called()
1000
1001         expect_component = copy.deepcopy(stubs.expect_job_spec)
1002         self._adjust_script_params(expect_component)
1003         expect_template = {
1004             "components": {
1005                 "testing 123": expect_component,
1006             },
1007             "name": "testing 123",
1008             "owner_uuid": project_uuid,
1009         }
1010         stubs.api.pipeline_templates().create.refute_called()
1011         stubs.api.pipeline_templates().update.assert_called_with(
1012             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1013
1014         self.assertEqual(capture_stdout.getvalue(),
1015                          self.existing_template_uuid + '\n')
1016
1017
1018 class TestCreateWorkflow(unittest.TestCase):
1019     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1020     expect_workflow = open("tests/wf/expect_packed.cwl").read()
1021
1022     @stubs
1023     def test_create(self, stubs):
1024         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1025
1026         capture_stdout = cStringIO.StringIO()
1027
1028         exited = arvados_cwl.main(
1029             ["--create-workflow", "--debug",
1030              "--api=containers",
1031              "--project-uuid", project_uuid,
1032              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1033             capture_stdout, sys.stderr, api_client=stubs.api)
1034         self.assertEqual(exited, 0)
1035
1036         stubs.api.pipeline_templates().create.refute_called()
1037         stubs.api.container_requests().create.refute_called()
1038
1039         body = {
1040             "workflow": {
1041                 "owner_uuid": project_uuid,
1042                 "name": "submit_wf.cwl",
1043                 "description": "",
1044                 "definition": self.expect_workflow,
1045             }
1046         }
1047         stubs.api.workflows().create.assert_called_with(
1048             body=JsonDiffMatcher(body))
1049
1050         self.assertEqual(capture_stdout.getvalue(),
1051                          stubs.expect_workflow_uuid + '\n')
1052
1053
1054     @stubs
1055     def test_create_name(self, stubs):
1056         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1057
1058         capture_stdout = cStringIO.StringIO()
1059
1060         exited = arvados_cwl.main(
1061             ["--create-workflow", "--debug",
1062              "--api=containers",
1063              "--project-uuid", project_uuid,
1064              "--name", "testing 123",
1065              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1066             capture_stdout, sys.stderr, api_client=stubs.api)
1067         self.assertEqual(exited, 0)
1068
1069         stubs.api.pipeline_templates().create.refute_called()
1070         stubs.api.container_requests().create.refute_called()
1071
1072         body = {
1073             "workflow": {
1074                 "owner_uuid": project_uuid,
1075                 "name": "testing 123",
1076                 "description": "",
1077                 "definition": self.expect_workflow,
1078             }
1079         }
1080         stubs.api.workflows().create.assert_called_with(
1081             body=JsonDiffMatcher(body))
1082
1083         self.assertEqual(capture_stdout.getvalue(),
1084                          stubs.expect_workflow_uuid + '\n')
1085
1086     @stubs
1087     def test_incompatible_api(self, stubs):
1088         capture_stderr = cStringIO.StringIO()
1089         logging.getLogger('arvados.cwl-runner').addHandler(
1090             logging.StreamHandler(capture_stderr))
1091
1092         exited = arvados_cwl.main(
1093             ["--update-workflow", self.existing_workflow_uuid,
1094              "--api=jobs",
1095              "--debug",
1096              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1097             sys.stderr, sys.stderr, api_client=stubs.api)
1098         self.assertEqual(exited, 1)
1099         self.assertRegexpMatches(
1100             capture_stderr.getvalue(),
1101             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1102
1103     @stubs
1104     def test_update(self, stubs):
1105         capture_stdout = cStringIO.StringIO()
1106
1107         exited = arvados_cwl.main(
1108             ["--update-workflow", self.existing_workflow_uuid,
1109              "--debug",
1110              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1111             capture_stdout, sys.stderr, api_client=stubs.api)
1112         self.assertEqual(exited, 0)
1113
1114         body = {
1115             "workflow": {
1116                 "name": "submit_wf.cwl",
1117                 "description": "",
1118                 "definition": self.expect_workflow,
1119             }
1120         }
1121         stubs.api.workflows().update.assert_called_with(
1122             uuid=self.existing_workflow_uuid,
1123             body=JsonDiffMatcher(body))
1124         self.assertEqual(capture_stdout.getvalue(),
1125                          self.existing_workflow_uuid + '\n')
1126
1127
1128     @stubs
1129     def test_update_name(self, stubs):
1130         capture_stdout = cStringIO.StringIO()
1131
1132         exited = arvados_cwl.main(
1133             ["--update-workflow", self.existing_workflow_uuid,
1134              "--debug", "--name", "testing 123",
1135              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1136             capture_stdout, sys.stderr, api_client=stubs.api)
1137         self.assertEqual(exited, 0)
1138
1139         body = {
1140             "workflow": {
1141                 "name": "testing 123",
1142                 "description": "",
1143                 "definition": self.expect_workflow,
1144             }
1145         }
1146         stubs.api.workflows().update.assert_called_with(
1147             uuid=self.existing_workflow_uuid,
1148             body=JsonDiffMatcher(body))
1149         self.assertEqual(capture_stdout.getvalue(),
1150                          self.existing_workflow_uuid + '\n')
1151
1152
1153     @stubs
1154     def test_create_collection_per_tool(self, stubs):
1155         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1156
1157         capture_stdout = cStringIO.StringIO()
1158
1159         exited = arvados_cwl.main(
1160             ["--create-workflow", "--debug",
1161              "--api=containers",
1162              "--project-uuid", project_uuid,
1163              "tests/collection_per_tool/collection_per_tool.cwl"],
1164             capture_stdout, sys.stderr, api_client=stubs.api)
1165         self.assertEqual(exited, 0)
1166
1167         expect_workflow = open("tests/collection_per_tool/collection_per_tool_packed.cwl").read()
1168
1169         body = {
1170             "workflow": {
1171                 "owner_uuid": project_uuid,
1172                 "name": "collection_per_tool.cwl",
1173                 "description": "",
1174                 "definition": expect_workflow,
1175             }
1176         }
1177         stubs.api.workflows().create.assert_called_with(
1178             body=JsonDiffMatcher(body))
1179
1180         self.assertEqual(capture_stdout.getvalue(),
1181                          stubs.expect_workflow_uuid + '\n')
1182
1183 class TestTemplateInputs(unittest.TestCase):
1184     expect_template = {
1185         "components": {
1186             "inputs_test.cwl": {
1187                 'runtime_constraints': {
1188                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1189                     'min_ram_mb_per_node': 1024
1190                 },
1191                 'script_parameters': {
1192                     'cwl:tool':
1193                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1194                     'optionalFloatInput': None,
1195                     'fileInput': {
1196                         'type': 'File',
1197                         'dataclass': 'File',
1198                         'required': True,
1199                         'title': "It's a file; we expect to find some characters in it.",
1200                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1201                     },
1202                     'floatInput': {
1203                         'type': 'float',
1204                         'dataclass': 'number',
1205                         'required': True,
1206                         'title': 'Floats like a duck',
1207                         'default': 0.1,
1208                         'value': 0.1,
1209                     },
1210                     'optionalFloatInput': {
1211                         'type': ['null', 'float'],
1212                         'dataclass': 'number',
1213                         'required': False,
1214                     },
1215                     'boolInput': {
1216                         'type': 'boolean',
1217                         'dataclass': 'boolean',
1218                         'required': True,
1219                         'title': 'True or false?',
1220                     },
1221                 },
1222                 'repository': 'arvados',
1223                 'script_version': 'master',
1224                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1225                 'script': 'cwl-runner',
1226             },
1227         },
1228         "name": "inputs_test.cwl",
1229     }
1230
1231     @stubs
1232     def test_inputs_empty(self, stubs):
1233         exited = arvados_cwl.main(
1234             ["--create-template",
1235              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1236             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1237         self.assertEqual(exited, 0)
1238
1239         stubs.api.pipeline_templates().create.assert_called_with(
1240             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1241
1242     @stubs
1243     def test_inputs(self, stubs):
1244         exited = arvados_cwl.main(
1245             ["--create-template",
1246              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1247             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1248         self.assertEqual(exited, 0)
1249
1250         expect_template = copy.deepcopy(self.expect_template)
1251         params = expect_template[
1252             "components"]["inputs_test.cwl"]["script_parameters"]
1253         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1254         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1255         params["floatInput"]["value"] = 1.234
1256         params["boolInput"]["value"] = True
1257
1258         stubs.api.pipeline_templates().create.assert_called_with(
1259             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)