sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 _rootDesc = None
  21
  22 def stubs(func):
  23     @functools.wraps(func)
  24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  25     @mock.patch("arvados.collection.KeepClient")
  26     @mock.patch("arvados.keep.KeepClient")
  27     @mock.patch("arvados.events.subscribe")
  28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  29         class Stubs:
  30             pass
  31         stubs = Stubs()
  32         stubs.events = events
  33         stubs.keepdocker = keepdocker
  34
  35
  36         def putstub(p, **kwargs):
  37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  38         keep_client1().put.side_effect = putstub
  39         keep_client1.put.side_effect = putstub
  40         keep_client2().put.side_effect = putstub
  41         keep_client2.put.side_effect = putstub
  42
  43         stubs.keep_client = keep_client2
  44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  46
  47         stubs.api = mock.MagicMock()
  48         stubs.api._rootDesc = get_rootDesc()
  49
  50         stubs.api.users().current().execute.return_value = {
  51             "uuid": stubs.fake_user_uuid,
  52         }
  53         stubs.api.collections().list().execute.return_value = {"items": []}
  54         stubs.api.collections().create().execute.side_effect = ({
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  56             "portable_data_hash": "99999999999999999999999999999991+99",
  57             "manifest_text": ""
  58         }, {
  59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  60             "portable_data_hash": "99999999999999999999999999999992+99",
  61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  62         },
  63         {
  64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  65             "portable_data_hash": "99999999999999999999999999999994+99",
  66             "manifest_text": ""
  67         },
  68         {
  69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  70             "portable_data_hash": "99999999999999999999999999999995+99",
  71             "manifest_text": ""
  72         },
  73         {
  74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  75             "portable_data_hash": "99999999999999999999999999999996+99",
  76             "manifest_text": ""
  77         }
  78         )
  79         stubs.api.collections().get().execute.return_value = {
  80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
 102             },
 103             'script_parameters': {
 104                 'x': {
 105                     'basename': 'blorp.txt',
 106                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 107                     'class': 'File'
 108                 },
 109                 'y': {
 110                     'basename': '99999999999999999999999999999998+99',
 111                     'location': 'keep:99999999999999999999999999999998+99',
 112                     'class': 'Directory'
 113                 },
 114                 'z': {
 115                     'basename': 'anonymous',
 116                     "listing": [{
 117                         "basename": "renamed.txt",
 118                         "class": "File",
 119                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 120                     }],
 121                     'class': 'Directory'
 122                 },
 123                 'cwl:tool':
 124                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 125             },
 126             'repository': 'arvados',
 127             'script_version': arvados_cwl.__version__,
 128             'script': 'cwl-runner'
 129         }
 130         stubs.pipeline_component = stubs.expect_job_spec.copy()
 131         stubs.expect_pipeline_instance = {
 132             'name': 'submit_wf.cwl',
 133             'state': 'RunningOnServer',
 134             "components": {
 135                 "cwl-runner": {
 136                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
 137                     'script_parameters': {
 138                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 139                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 140                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 141                               'listing': [
 142                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 143                               ]}},
 144                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 145                         'arv:enable_reuse': True
 146                     },
 147                     'repository': 'arvados',
 148                     'script_version': arvados_cwl.__version__,
 149                     'script': 'cwl-runner',
 150                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 151                 }
 152             }
 153         }
 154         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 155         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 156         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 157         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 158         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 159             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 160             "state": "Queued"
 161         }
 162         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 163         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 164
 165         stubs.expect_container_spec = {
 166             'priority': 1,
 167             'mounts': {
 168                 '/var/spool/cwl': {
 169                     'writable': True,
 170                     'kind': 'collection'
 171                 },
 172                 '/var/lib/cwl/workflow': {
 173                     'portable_data_hash': '99999999999999999999999999999991+99',
 174                     'kind': 'collection'
 175                 },
 176                 'stdout': {
 177                     'path': '/var/spool/cwl/cwl.output.json',
 178                     'kind': 'file'
 179                 },
 180                 '/var/lib/cwl/job/cwl.input.json': {
 181                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 182                     'kind': 'collection'
 183                 }
 184             },
 185             'state': 'Committed',
 186             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 187             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 188             'name': 'submit_wf.cwl',
 189             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 190             'output_path': '/var/spool/cwl',
 191             'cwd': '/var/spool/cwl',
 192             'runtime_constraints': {
 193                 'API': True,
 194                 'vcpus': 1,
 195                 'ram': 268435456
 196             }
 197         }
 198
 199         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 200         stubs.api.workflows().create().execute.return_value = {
 201             "uuid": stubs.expect_workflow_uuid,
 202         }
 203
 204         return func(self, stubs, *args, **kwargs)
 205     return wrapped
 206
 207
 208 class TestSubmit(unittest.TestCase):
 209     @mock.patch("time.sleep")
 210     @stubs
 211     def test_submit(self, stubs, tm):
 212         capture_stdout = cStringIO.StringIO()
 213         exited = arvados_cwl.main(
 214             ["--submit", "--no-wait", "--debug",
 215              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 216             capture_stdout, sys.stderr, api_client=stubs.api)
 217         self.assertEqual(exited, 0)
 218
 219         stubs.api.collections().create.assert_has_calls([
 220             mock.call(),
 221             mock.call(body={
 222                 'manifest_text':
 223                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 224                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 225                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 226                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 227                 'name': 'submit_wf.cwl',
 228             }, ensure_unique_name=True),
 229             mock.call().execute(),
 230             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 231                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 232                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 233                             'replication_desired': None,
 234                             'name': 'New collection'
 235             }, ensure_unique_name=True),
 236             mock.call().execute(num_retries=4),
 237             mock.call(body={
 238                 'manifest_text':
 239                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 240                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 241                 'name': '#',
 242             }, ensure_unique_name=True),
 243             mock.call().execute()])
 244
 245         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 246         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 247         stubs.api.pipeline_instances().create.assert_called_with(
 248             body=expect_pipeline)
 249         self.assertEqual(capture_stdout.getvalue(),
 250                          stubs.expect_pipeline_uuid + '\n')
 251
 252
 253     @mock.patch("time.sleep")
 254     @stubs
 255     def test_submit_no_reuse(self, stubs, tm):
 256         capture_stdout = cStringIO.StringIO()
 257         exited = arvados_cwl.main(
 258             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 259              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 260             capture_stdout, sys.stderr, api_client=stubs.api)
 261         self.assertEqual(exited, 0)
 262
 263         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 264
 265         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 266         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 267         stubs.api.pipeline_instances().create.assert_called_with(
 268             body=expect_pipeline)
 269         self.assertEqual(capture_stdout.getvalue(),
 270                          stubs.expect_pipeline_uuid + '\n')
 271
 272     @mock.patch("time.sleep")
 273     @stubs
 274     def test_submit_output_name(self, stubs, tm):
 275         output_name = "test_output_name"
 276
 277         capture_stdout = cStringIO.StringIO()
 278         exited = arvados_cwl.main(
 279             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 281             capture_stdout, sys.stderr, api_client=stubs.api)
 282         self.assertEqual(exited, 0)
 283
 284         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 285
 286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 287         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 288         stubs.api.pipeline_instances().create.assert_called_with(
 289             body=expect_pipeline)
 290         self.assertEqual(capture_stdout.getvalue(),
 291                          stubs.expect_pipeline_uuid + '\n')
 292
 293     @mock.patch("time.sleep")
 294     @stubs
 295     def test_submit_output_tags(self, stubs, tm):
 296         output_tags = "tag0,tag1,tag2"
 297
 298         capture_stdout = cStringIO.StringIO()
 299         exited = arvados_cwl.main(
 300             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 301              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 302             capture_stdout, sys.stderr, api_client=stubs.api)
 303         self.assertEqual(exited, 0)
 304
 305         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 306
 307         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 308         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 309         stubs.api.pipeline_instances().create.assert_called_with(
 310             body=expect_pipeline)
 311         self.assertEqual(capture_stdout.getvalue(),
 312                          stubs.expect_pipeline_uuid + '\n')
 313
 314     @mock.patch("time.sleep")
 315     @stubs
 316     def test_submit_with_project_uuid(self, stubs, tm):
 317         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 318
 319         exited = arvados_cwl.main(
 320             ["--submit", "--no-wait",
 321              "--project-uuid", project_uuid,
 322              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 323             sys.stdout, sys.stderr, api_client=stubs.api)
 324         self.assertEqual(exited, 0)
 325
 326         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 327         expect_pipeline["owner_uuid"] = project_uuid
 328         stubs.api.pipeline_instances().create.assert_called_with(
 329             body=expect_pipeline)
 330
 331     @stubs
 332     def test_submit_container(self, stubs):
 333         capture_stdout = cStringIO.StringIO()
 334         try:
 335             exited = arvados_cwl.main(
 336                 ["--submit", "--no-wait", "--api=containers", "--debug",
 337                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 338                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 339             self.assertEqual(exited, 0)
 340         except:
 341             logging.exception("")
 342
 343         stubs.api.collections().create.assert_has_calls([
 344             mock.call(),
 345             mock.call(body={
 346                 'manifest_text':
 347                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 348                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 349                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 350                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 351                 'name': 'submit_wf.cwl',
 352             }, ensure_unique_name=True),
 353             mock.call().execute(),
 354             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 355                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 356                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 357                             'name': 'New collection',
 358                             'replication_desired': None,
 359             }, ensure_unique_name=True),
 360             mock.call().execute(num_retries=4),
 361             mock.call(body={
 362                 'manifest_text':
 363                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 364                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 365                 'name': '#',
 366             }, ensure_unique_name=True),
 367             mock.call().execute()])
 368
 369         expect_container = copy.deepcopy(stubs.expect_container_spec)
 370         expect_container["owner_uuid"] = stubs.fake_user_uuid
 371         stubs.api.container_requests().create.assert_called_with(
 372             body=expect_container)
 373         self.assertEqual(capture_stdout.getvalue(),
 374                          stubs.expect_container_request_uuid + '\n')
 375
 376     @stubs
 377     def test_submit_container_no_reuse(self, stubs):
 378         capture_stdout = cStringIO.StringIO()
 379         try:
 380             exited = arvados_cwl.main(
 381                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 382                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 383                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 384             self.assertEqual(exited, 0)
 385         except:
 386             logging.exception("")
 387
 388         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 389
 390         expect_container = copy.deepcopy(stubs.expect_container_spec)
 391         expect_container["owner_uuid"] = stubs.fake_user_uuid
 392         stubs.api.container_requests().create.assert_called_with(
 393             body=expect_container)
 394         self.assertEqual(capture_stdout.getvalue(),
 395                          stubs.expect_container_request_uuid + '\n')
 396
 397     @stubs
 398     def test_submit_container_output_name(self, stubs):
 399         output_name = "test_output_name"
 400
 401         capture_stdout = cStringIO.StringIO()
 402         try:
 403             exited = arvados_cwl.main(
 404                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 405                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 406                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 407             self.assertEqual(exited, 0)
 408         except:
 409             logging.exception("")
 410
 411         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-name="+output_name, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         expect_container["owner_uuid"] = stubs.fake_user_uuid
 415         stubs.api.container_requests().create.assert_called_with(
 416             body=expect_container)
 417         self.assertEqual(capture_stdout.getvalue(),
 418                          stubs.expect_container_request_uuid + '\n')
 419
 420     @stubs
 421     def test_submit_container_output_tags(self, stubs):
 422         output_tags = "tag0,tag1,tag2"
 423
 424         capture_stdout = cStringIO.StringIO()
 425         try:
 426             exited = arvados_cwl.main(
 427                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 428                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 429                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 430             self.assertEqual(exited, 0)
 431         except:
 432             logging.exception("")
 433
 434         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-tags="+output_tags, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["owner_uuid"] = stubs.fake_user_uuid
 438         stubs.api.container_requests().create.assert_called_with(
 439             body=expect_container)
 440         self.assertEqual(capture_stdout.getvalue(),
 441                          stubs.expect_container_request_uuid + '\n')
 442
 443     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 444     @mock.patch("cwltool.docker.get_image")
 445     @mock.patch("arvados.api")
 446     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 447         arvrunner = mock.MagicMock()
 448         arvrunner.project_uuid = ""
 449         api.return_value = mock.MagicMock()
 450         arvrunner.api = api.return_value
 451         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 452                                                             {"items": [], "items_available": 0, "offset": 0},
 453                                                             {"items": [], "items_available": 0, "offset": 0},
 454                                                             {"items": [{"created_at": "",
 455                                                                         "head_uuid": "",
 456                                                                         "link_class": "docker_image_hash",
 457                                                                         "name": "123456",
 458                                                                         "owner_uuid": "",
 459                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 460                                                             {"items": [], "items_available": 0, "offset": 0},
 461                                                             {"items": [{"created_at": "",
 462                                                                         "head_uuid": "",
 463                                                                         "link_class": "docker_image_repo+tag",
 464                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 465                                                                         "owner_uuid": "",
 466                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 467                                                             {"items": [{"created_at": "",
 468                                                                         "head_uuid": "",
 469                                                                         "link_class": "docker_image_hash",
 470                                                                         "name": "123456",
 471                                                                         "owner_uuid": "",
 472                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 473         )
 474         find_one_image_hash.return_value = "123456"
 475
 476         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 477                                                                   {"items": [{"uuid": "",
 478                                                                               "owner_uuid": "",
 479                                                                               "manifest_text": "",
 480                                                                               "properties": ""
 481                                                                           }], "items_available": 1, "offset": 0},
 482                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 483         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 484         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
 485
 486 class TestCreateTemplate(unittest.TestCase):
 487     @stubs
 488     def test_create(self, stubs):
 489         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 490
 491         capture_stdout = cStringIO.StringIO()
 492
 493         exited = arvados_cwl.main(
 494             ["--create-template", "--debug",
 495              "--project-uuid", project_uuid,
 496              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 497             capture_stdout, sys.stderr, api_client=stubs.api)
 498         self.assertEqual(exited, 0)
 499
 500         stubs.api.pipeline_instances().create.refute_called()
 501         stubs.api.jobs().create.refute_called()
 502
 503         expect_component = copy.deepcopy(stubs.expect_job_spec)
 504         expect_component['script_parameters']['x'] = {
 505             'dataclass': 'File',
 506             'required': True,
 507             'type': 'File',
 508             'value': '99999999999999999999999999999994+99/blorp.txt',
 509         }
 510         expect_component['script_parameters']['y'] = {
 511             'dataclass': 'Collection',
 512             'required': True,
 513             'type': 'Directory',
 514             'value': '99999999999999999999999999999998+99',
 515         }
 516         expect_component['script_parameters']['z'] = {
 517             'dataclass': 'Collection',
 518             'required': True,
 519             'type': 'Directory',
 520         }
 521         expect_template = {
 522             "components": {
 523                 "submit_wf.cwl": expect_component,
 524             },
 525             "name": "submit_wf.cwl",
 526             "owner_uuid": project_uuid,
 527         }
 528         stubs.api.pipeline_templates().create.assert_called_with(
 529             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 530
 531         self.assertEqual(capture_stdout.getvalue(),
 532                          stubs.expect_pipeline_template_uuid + '\n')
 533
 534
 535 class TestCreateWorkflow(unittest.TestCase):
 536     @stubs
 537     def test_create(self, stubs):
 538         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 539
 540         capture_stdout = cStringIO.StringIO()
 541
 542         exited = arvados_cwl.main(
 543             ["--create-workflow", "--debug",
 544              "--project-uuid", project_uuid,
 545              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 546             capture_stdout, sys.stderr, api_client=stubs.api)
 547         self.assertEqual(exited, 0)
 548
 549         stubs.api.pipeline_templates().create.refute_called()
 550         stubs.api.container_requests().create.refute_called()
 551
 552         with open("tests/wf/expect_packed.cwl") as f:
 553             expect_workflow = f.read()
 554
 555         body = {
 556             "workflow": {
 557                 "owner_uuid": project_uuid,
 558                 "name": "submit_wf.cwl",
 559                 "description": "",
 560                 "definition": expect_workflow
 561                 }
 562         }
 563         stubs.api.workflows().create.assert_called_with(
 564             body=JsonDiffMatcher(body))
 565
 566         self.assertEqual(capture_stdout.getvalue(),
 567                          stubs.expect_workflow_uuid + '\n')
 568
 569
 570 class TestTemplateInputs(unittest.TestCase):
 571     expect_template = {
 572         "components": {
 573             "inputs_test.cwl": {
 574                 'runtime_constraints': {
 575                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 576                 },
 577                 'script_parameters': {
 578                     'cwl:tool':
 579                     '99999999999999999999999999999991+99/'
 580                     'wf/inputs_test.cwl',
 581                     'optionalFloatInput': None,
 582                     'fileInput': {
 583                         'type': 'File',
 584                         'dataclass': 'File',
 585                         'required': True,
 586                         'title': "It's a file; we expect to find some characters in it.",
 587                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 588                     },
 589                     'floatInput': {
 590                         'type': 'float',
 591                         'dataclass': 'number',
 592                         'required': True,
 593                         'title': 'Floats like a duck',
 594                         'default': 0.1,
 595                         'value': 0.1,
 596                     },
 597                     'optionalFloatInput': {
 598                         'type': ['null', 'float'],
 599                         'dataclass': 'number',
 600                         'required': False,
 601                     },
 602                     'boolInput': {
 603                         'type': 'boolean',
 604                         'dataclass': 'boolean',
 605                         'required': True,
 606                         'title': 'True or false?',
 607                     },
 608                 },
 609                 'repository': 'arvados',
 610                 'script_version': arvados_cwl.__version__,
 611                 'script': 'cwl-runner',
 612             },
 613         },
 614         "name": "inputs_test.cwl",
 615     }
 616
 617     @stubs
 618     def test_inputs_empty(self, stubs):
 619         exited = arvados_cwl.main(
 620             ["--create-template", "--no-wait",
 621              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 622             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 623         self.assertEqual(exited, 0)
 624
 625         expect_template = copy.deepcopy(self.expect_template)
 626         expect_template["owner_uuid"] = stubs.fake_user_uuid
 627
 628         stubs.api.pipeline_templates().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 630
 631     @stubs
 632     def test_inputs(self, stubs):
 633         exited = arvados_cwl.main(
 634             ["--create-template", "--no-wait",
 635              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 636             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 637         self.assertEqual(exited, 0)
 638
 639         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 640
 641         expect_template = copy.deepcopy(self.expect_template)
 642         expect_template["owner_uuid"] = stubs.fake_user_uuid
 643         params = expect_template[
 644             "components"]["inputs_test.cwl"]["script_parameters"]
 645         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 646         params["floatInput"]["value"] = 1.234
 647         params["boolInput"]["value"] = True
 648
 649         stubs.api.pipeline_templates().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)