sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 _rootDesc = None
  21
  22 def stubs(func):
  23     @functools.wraps(func)
  24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  25     @mock.patch("arvados.collection.KeepClient")
  26     @mock.patch("arvados.keep.KeepClient")
  27     @mock.patch("arvados.events.subscribe")
  28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  29         class Stubs:
  30             pass
  31         stubs = Stubs()
  32         stubs.events = events
  33         stubs.keepdocker = keepdocker
  34
  35
  36         def putstub(p, **kwargs):
  37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  38         keep_client1().put.side_effect = putstub
  39         keep_client1.put.side_effect = putstub
  40         keep_client2().put.side_effect = putstub
  41         keep_client2.put.side_effect = putstub
  42
  43         stubs.keep_client = keep_client2
  44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  46
  47         stubs.api = mock.MagicMock()
  48         stubs.api._rootDesc = get_rootDesc()
  49
  50         stubs.api.users().current().execute.return_value = {
  51             "uuid": stubs.fake_user_uuid,
  52         }
  53         stubs.api.collections().list().execute.return_value = {"items": []}
  54         stubs.api.collections().create().execute.side_effect = ({
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  56             "portable_data_hash": "99999999999999999999999999999991+99",
  57             "manifest_text": ""
  58         }, {
  59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  60             "portable_data_hash": "99999999999999999999999999999992+99",
  61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  62         },
  63         {
  64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  65             "portable_data_hash": "99999999999999999999999999999994+99",
  66             "manifest_text": ""
  67         },
  68         {
  69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  70             "portable_data_hash": "99999999999999999999999999999995+99",
  71             "manifest_text": ""
  72         },
  73         {
  74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  75             "portable_data_hash": "99999999999999999999999999999996+99",
  76             "manifest_text": ""
  77         }
  78         )
  79         stubs.api.collections().get().execute.return_value = {
  80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
 102             },
 103             'script_parameters': {
 104                 'x': {
 105                     'basename': 'blorp.txt',
 106                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 107                     'class': 'File'
 108                 },
 109                 'y': {
 110                     'basename': '99999999999999999999999999999998+99',
 111                     'location': 'keep:99999999999999999999999999999998+99',
 112                     'class': 'Directory'
 113                 },
 114                 'z': {
 115                     'basename': 'anonymous',
 116                     "listing": [{
 117                         "basename": "renamed.txt",
 118                         "class": "File",
 119                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 120                     }],
 121                     'class': 'Directory'
 122                 },
 123                 'cwl:tool':
 124                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 125             },
 126             'repository': 'arvados',
 127             'script_version': arvados_cwl.__version__,
 128             'script': 'cwl-runner'
 129         }
 130         stubs.pipeline_component = stubs.expect_job_spec.copy()
 131         stubs.expect_pipeline_instance = {
 132             'name': 'submit_wf.cwl',
 133             'state': 'RunningOnServer',
 134             'owner_uuid': None,
 135             "components": {
 136                 "cwl-runner": {
 137                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
 138                     'script_parameters': {
 139                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 140                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 141                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 142                               'listing': [
 143                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 144                               ]}},
 145                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 146                         'arv:enable_reuse': True
 147                     },
 148                     'repository': 'arvados',
 149                     'script_version': arvados_cwl.__version__,
 150                     'script': 'cwl-runner',
 151                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 152                 }
 153             }
 154         }
 155         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 156         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 157         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 158         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 159         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 160             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 161             "state": "Queued"
 162         }
 163         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 164         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 165
 166         stubs.expect_container_spec = {
 167             'priority': 1,
 168             'mounts': {
 169                 '/var/spool/cwl': {
 170                     'writable': True,
 171                     'kind': 'collection'
 172                 },
 173                 '/var/lib/cwl/workflow': {
 174                     'portable_data_hash': '99999999999999999999999999999991+99',
 175                     'kind': 'collection'
 176                 },
 177                 'stdout': {
 178                     'path': '/var/spool/cwl/cwl.output.json',
 179                     'kind': 'file'
 180                 },
 181                 '/var/lib/cwl/job/cwl.input.json': {
 182                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 183                     'kind': 'collection'
 184                 }
 185             },
 186             'state': 'Committed',
 187             'owner_uuid': None,
 188             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 189             'name': 'submit_wf.cwl',
 190             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 191             'output_path': '/var/spool/cwl',
 192             'cwd': '/var/spool/cwl',
 193             'runtime_constraints': {
 194                 'API': True,
 195                 'vcpus': 1,
 196                 'ram': 268435456
 197             }
 198         }
 199
 200         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 201         stubs.api.workflows().create().execute.return_value = {
 202             "uuid": stubs.expect_workflow_uuid,
 203         }
 204         def update_mock(**kwargs):
 205             stubs.updated_uuid = kwargs.get('uuid')
 206             return mock.DEFAULT
 207         stubs.api.workflows().update.side_effect = update_mock
 208         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 209             "uuid": stubs.updated_uuid,
 210         }
 211
 212         return func(self, stubs, *args, **kwargs)
 213     return wrapped
 214
 215
 216 class TestSubmit(unittest.TestCase):
 217     @mock.patch("time.sleep")
 218     @stubs
 219     def test_submit(self, stubs, tm):
 220         capture_stdout = cStringIO.StringIO()
 221         exited = arvados_cwl.main(
 222             ["--submit", "--no-wait", "--debug",
 223              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 224             capture_stdout, sys.stderr, api_client=stubs.api)
 225         self.assertEqual(exited, 0)
 226
 227         stubs.api.collections().create.assert_has_calls([
 228             mock.call(),
 229             mock.call(body={
 230                 'manifest_text':
 231                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 232                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 233                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 234                 'owner_uuid': None,
 235                 'name': 'submit_wf.cwl',
 236             }, ensure_unique_name=True),
 237             mock.call().execute(),
 238             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 239                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 240                             'replication_desired': None,
 241                             'name': 'New collection'
 242             }, ensure_unique_name=True),
 243             mock.call().execute(num_retries=4),
 244             mock.call(body={
 245                 'manifest_text':
 246                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 247                 'owner_uuid': None,
 248                 'name': '#',
 249             }, ensure_unique_name=True),
 250             mock.call().execute()])
 251
 252         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 253         stubs.api.pipeline_instances().create.assert_called_with(
 254             body=expect_pipeline)
 255         self.assertEqual(capture_stdout.getvalue(),
 256                          stubs.expect_pipeline_uuid + '\n')
 257
 258
 259     @mock.patch("time.sleep")
 260     @stubs
 261     def test_submit_no_reuse(self, stubs, tm):
 262         capture_stdout = cStringIO.StringIO()
 263         exited = arvados_cwl.main(
 264             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 265              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 266             capture_stdout, sys.stderr, api_client=stubs.api)
 267         self.assertEqual(exited, 0)
 268
 269         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 270
 271         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 272         stubs.api.pipeline_instances().create.assert_called_with(
 273             body=expect_pipeline)
 274         self.assertEqual(capture_stdout.getvalue(),
 275                          stubs.expect_pipeline_uuid + '\n')
 276
 277     @mock.patch("time.sleep")
 278     @stubs
 279     def test_submit_output_name(self, stubs, tm):
 280         output_name = "test_output_name"
 281
 282         capture_stdout = cStringIO.StringIO()
 283         exited = arvados_cwl.main(
 284             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 285              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 286             capture_stdout, sys.stderr, api_client=stubs.api)
 287         self.assertEqual(exited, 0)
 288
 289         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 290
 291         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 292         stubs.api.pipeline_instances().create.assert_called_with(
 293             body=expect_pipeline)
 294         self.assertEqual(capture_stdout.getvalue(),
 295                          stubs.expect_pipeline_uuid + '\n')
 296
 297     @mock.patch("time.sleep")
 298     @stubs
 299     def test_submit_output_tags(self, stubs, tm):
 300         output_tags = "tag0,tag1,tag2"
 301
 302         capture_stdout = cStringIO.StringIO()
 303         exited = arvados_cwl.main(
 304             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 305              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 306             capture_stdout, sys.stderr, api_client=stubs.api)
 307         self.assertEqual(exited, 0)
 308
 309         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 310
 311         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 312         stubs.api.pipeline_instances().create.assert_called_with(
 313             body=expect_pipeline)
 314         self.assertEqual(capture_stdout.getvalue(),
 315                          stubs.expect_pipeline_uuid + '\n')
 316
 317     @mock.patch("time.sleep")
 318     @stubs
 319     def test_submit_with_project_uuid(self, stubs, tm):
 320         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 321
 322         exited = arvados_cwl.main(
 323             ["--submit", "--no-wait",
 324              "--project-uuid", project_uuid,
 325              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 326             sys.stdout, sys.stderr, api_client=stubs.api)
 327         self.assertEqual(exited, 0)
 328
 329         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 330         expect_pipeline["owner_uuid"] = project_uuid
 331         stubs.api.pipeline_instances().create.assert_called_with(
 332             body=expect_pipeline)
 333
 334     @stubs
 335     def test_submit_container(self, stubs):
 336         capture_stdout = cStringIO.StringIO()
 337         try:
 338             exited = arvados_cwl.main(
 339                 ["--submit", "--no-wait", "--api=containers", "--debug",
 340                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 341                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 342             self.assertEqual(exited, 0)
 343         except:
 344             logging.exception("")
 345
 346         stubs.api.collections().create.assert_has_calls([
 347             mock.call(),
 348             mock.call(body={
 349                 'manifest_text':
 350                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 351                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 352                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 353                 'owner_uuid': None,
 354                 'name': 'submit_wf.cwl',
 355             }, ensure_unique_name=True),
 356             mock.call().execute(),
 357             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 358                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 359                             'name': 'New collection',
 360                             'replication_desired': None,
 361             }, ensure_unique_name=True),
 362             mock.call().execute(num_retries=4),
 363             mock.call(body={
 364                 'manifest_text':
 365                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 366                 'owner_uuid': None,
 367                 'name': '#',
 368             }, ensure_unique_name=True),
 369             mock.call().execute()])
 370
 371         expect_container = copy.deepcopy(stubs.expect_container_spec)
 372         stubs.api.container_requests().create.assert_called_with(
 373             body=expect_container)
 374         self.assertEqual(capture_stdout.getvalue(),
 375                          stubs.expect_container_request_uuid + '\n')
 376
 377     @stubs
 378     def test_submit_container_no_reuse(self, stubs):
 379         capture_stdout = cStringIO.StringIO()
 380         try:
 381             exited = arvados_cwl.main(
 382                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 383                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 384                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 385             self.assertEqual(exited, 0)
 386         except:
 387             logging.exception("")
 388
 389         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 390
 391         expect_container = copy.deepcopy(stubs.expect_container_spec)
 392         stubs.api.container_requests().create.assert_called_with(
 393             body=expect_container)
 394         self.assertEqual(capture_stdout.getvalue(),
 395                          stubs.expect_container_request_uuid + '\n')
 396
 397     @stubs
 398     def test_submit_container_output_name(self, stubs):
 399         output_name = "test_output_name"
 400
 401         capture_stdout = cStringIO.StringIO()
 402         try:
 403             exited = arvados_cwl.main(
 404                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 405                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 406                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 407             self.assertEqual(exited, 0)
 408         except:
 409             logging.exception("")
 410
 411         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-name="+output_name, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         stubs.api.container_requests().create.assert_called_with(
 415             body=expect_container)
 416         self.assertEqual(capture_stdout.getvalue(),
 417                          stubs.expect_container_request_uuid + '\n')
 418
 419     @stubs
 420     def test_submit_container_output_tags(self, stubs):
 421         output_tags = "tag0,tag1,tag2"
 422
 423         capture_stdout = cStringIO.StringIO()
 424         try:
 425             exited = arvados_cwl.main(
 426                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 427                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 428                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 429             self.assertEqual(exited, 0)
 430         except:
 431             logging.exception("")
 432
 433         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-tags="+output_tags, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 434
 435         expect_container = copy.deepcopy(stubs.expect_container_spec)
 436         stubs.api.container_requests().create.assert_called_with(
 437             body=expect_container)
 438         self.assertEqual(capture_stdout.getvalue(),
 439                          stubs.expect_container_request_uuid + '\n')
 440
 441     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 442     @mock.patch("cwltool.docker.get_image")
 443     @mock.patch("arvados.api")
 444     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 445         arvrunner = mock.MagicMock()
 446         arvrunner.project_uuid = ""
 447         api.return_value = mock.MagicMock()
 448         arvrunner.api = api.return_value
 449         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 450                                                             {"items": [], "items_available": 0, "offset": 0},
 451                                                             {"items": [], "items_available": 0, "offset": 0},
 452                                                             {"items": [{"created_at": "",
 453                                                                         "head_uuid": "",
 454                                                                         "link_class": "docker_image_hash",
 455                                                                         "name": "123456",
 456                                                                         "owner_uuid": "",
 457                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 458                                                             {"items": [], "items_available": 0, "offset": 0},
 459                                                             {"items": [{"created_at": "",
 460                                                                         "head_uuid": "",
 461                                                                         "link_class": "docker_image_repo+tag",
 462                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 463                                                                         "owner_uuid": "",
 464                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 465                                                             {"items": [{"created_at": "",
 466                                                                         "head_uuid": "",
 467                                                                         "link_class": "docker_image_hash",
 468                                                                         "name": "123456",
 469                                                                         "owner_uuid": "",
 470                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 471         )
 472         find_one_image_hash.return_value = "123456"
 473
 474         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 475                                                                   {"items": [{"uuid": "",
 476                                                                               "owner_uuid": "",
 477                                                                               "manifest_text": "",
 478                                                                               "properties": ""
 479                                                                           }], "items_available": 1, "offset": 0},
 480                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 481         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 482         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
 483
 484 class TestCreateTemplate(unittest.TestCase):
 485     @stubs
 486     def test_create(self, stubs):
 487         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 488
 489         capture_stdout = cStringIO.StringIO()
 490
 491         exited = arvados_cwl.main(
 492             ["--create-workflow", "--debug",
 493              "--project-uuid", project_uuid,
 494              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 495             capture_stdout, sys.stderr, api_client=stubs.api)
 496         self.assertEqual(exited, 0)
 497
 498         stubs.api.pipeline_instances().create.refute_called()
 499         stubs.api.jobs().create.refute_called()
 500
 501         expect_component = copy.deepcopy(stubs.expect_job_spec)
 502         expect_component['script_parameters']['x'] = {
 503             'dataclass': 'File',
 504             'required': True,
 505             'type': 'File',
 506             'value': '99999999999999999999999999999994+99/blorp.txt',
 507         }
 508         expect_component['script_parameters']['y'] = {
 509             'dataclass': 'Collection',
 510             'required': True,
 511             'type': 'Directory',
 512             'value': '99999999999999999999999999999998+99',
 513         }
 514         expect_component['script_parameters']['z'] = {
 515             'dataclass': 'Collection',
 516             'required': True,
 517             'type': 'Directory',
 518         }
 519         expect_template = {
 520             "components": {
 521                 "submit_wf.cwl": expect_component,
 522             },
 523             "name": "submit_wf.cwl",
 524             "owner_uuid": project_uuid,
 525         }
 526         stubs.api.pipeline_templates().create.assert_called_with(
 527             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 528
 529         self.assertEqual(capture_stdout.getvalue(),
 530                          stubs.expect_pipeline_template_uuid + '\n')
 531
 532
 533 class TestCreateWorkflow(unittest.TestCase):
 534     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 535     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 536
 537     @stubs
 538     def test_create(self, stubs):
 539         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 540
 541         capture_stdout = cStringIO.StringIO()
 542
 543         exited = arvados_cwl.main(
 544             ["--create-workflow", "--debug",
 545              "--api=containers",
 546              "--project-uuid", project_uuid,
 547              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 548             capture_stdout, sys.stderr, api_client=stubs.api)
 549         self.assertEqual(exited, 0)
 550
 551         stubs.api.pipeline_templates().create.refute_called()
 552         stubs.api.container_requests().create.refute_called()
 553
 554         body = {
 555             "workflow": {
 556                 "owner_uuid": project_uuid,
 557                 "name": "submit_wf.cwl",
 558                 "description": "",
 559                 "definition": self.expect_workflow,
 560             }
 561         }
 562         stubs.api.workflows().create.assert_called_with(
 563             body=JsonDiffMatcher(body))
 564
 565         self.assertEqual(capture_stdout.getvalue(),
 566                          stubs.expect_workflow_uuid + '\n')
 567
 568     @stubs
 569     def test_incompatible_api(self, stubs):
 570         capture_stderr = cStringIO.StringIO()
 571         logging.getLogger('arvados.cwl-runner').addHandler(
 572             logging.StreamHandler(capture_stderr))
 573
 574         exited = arvados_cwl.main(
 575             ["--update-workflow", self.existing_workflow_uuid,
 576              "--api=jobs",
 577              "--debug",
 578              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 579             sys.stderr, sys.stderr, api_client=stubs.api)
 580         self.assertEqual(exited, 1)
 581         self.assertRegexpMatches(
 582             capture_stderr.getvalue(),
 583             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
 584
 585     @stubs
 586     def test_update(self, stubs):
 587         capture_stdout = cStringIO.StringIO()
 588
 589         exited = arvados_cwl.main(
 590             ["--update-workflow", self.existing_workflow_uuid,
 591              "--debug",
 592              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 593             capture_stdout, sys.stderr, api_client=stubs.api)
 594         self.assertEqual(exited, 0)
 595
 596         body = {
 597             "workflow": {
 598                 "name": "submit_wf.cwl",
 599                 "description": "",
 600                 "definition": self.expect_workflow,
 601             }
 602         }
 603         stubs.api.workflows().update.assert_called_with(
 604             uuid=self.existing_workflow_uuid,
 605             body=JsonDiffMatcher(body))
 606         self.assertEqual(capture_stdout.getvalue(),
 607                          self.existing_workflow_uuid + '\n')
 608
 609
 610 class TestTemplateInputs(unittest.TestCase):
 611     expect_template = {
 612         "components": {
 613             "inputs_test.cwl": {
 614                 'runtime_constraints': {
 615                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 616                 },
 617                 'script_parameters': {
 618                     'cwl:tool':
 619                     '99999999999999999999999999999991+99/'
 620                     'wf/inputs_test.cwl',
 621                     'optionalFloatInput': None,
 622                     'fileInput': {
 623                         'type': 'File',
 624                         'dataclass': 'File',
 625                         'required': True,
 626                         'title': "It's a file; we expect to find some characters in it.",
 627                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 628                     },
 629                     'floatInput': {
 630                         'type': 'float',
 631                         'dataclass': 'number',
 632                         'required': True,
 633                         'title': 'Floats like a duck',
 634                         'default': 0.1,
 635                         'value': 0.1,
 636                     },
 637                     'optionalFloatInput': {
 638                         'type': ['null', 'float'],
 639                         'dataclass': 'number',
 640                         'required': False,
 641                     },
 642                     'boolInput': {
 643                         'type': 'boolean',
 644                         'dataclass': 'boolean',
 645                         'required': True,
 646                         'title': 'True or false?',
 647                     },
 648                 },
 649                 'repository': 'arvados',
 650                 'script_version': arvados_cwl.__version__,
 651                 'script': 'cwl-runner',
 652             },
 653         },
 654         "name": "inputs_test.cwl",
 655     }
 656
 657     @stubs
 658     def test_inputs_empty(self, stubs):
 659         exited = arvados_cwl.main(
 660             ["--create-template", "--no-wait",
 661              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 662             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 663         self.assertEqual(exited, 0)
 664
 665         stubs.api.pipeline_templates().create.assert_called_with(
 666             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
 667
 668     @stubs
 669     def test_inputs(self, stubs):
 670         exited = arvados_cwl.main(
 671             ["--create-template", "--no-wait",
 672              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 673             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 674         self.assertEqual(exited, 0)
 675
 676         expect_template = copy.deepcopy(self.expect_template)
 677         params = expect_template[
 678             "components"]["inputs_test.cwl"]["script_parameters"]
 679         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 680         params["floatInput"]["value"] = 1.234
 681         params["boolInput"]["value"] = True
 682
 683         stubs.api.pipeline_templates().create.assert_called_with(
 684             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)