Merge branch '9397-prepopulate-output-directory' refs #9397
[arvados.git] / sdk / cwl / tests / test_submit.py
1 import copy
2 import cStringIO
3 import functools
4 import hashlib
5 import json
6 import logging
7 import mock
8 import sys
9 import unittest
10
11 import arvados
12 import arvados.collection
13 import arvados_cwl
14 import arvados_cwl.runner
15 import arvados.keep
16
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
19
20 import ruamel.yaml as yaml
21
22 _rootDesc = None
23
24 def stubs(func):
25     @functools.wraps(func)
26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
27     @mock.patch("arvados.collection.KeepClient")
28     @mock.patch("arvados.keep.KeepClient")
29     @mock.patch("arvados.events.subscribe")
30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
31         class Stubs:
32             pass
33         stubs = Stubs()
34         stubs.events = events
35         stubs.keepdocker = keepdocker
36
37
38         def putstub(p, **kwargs):
39             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
40         keep_client1().put.side_effect = putstub
41         keep_client1.put.side_effect = putstub
42         keep_client2().put.side_effect = putstub
43         keep_client2.put.side_effect = putstub
44
45         stubs.keep_client = keep_client2
46         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
47         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
48
49         stubs.api = mock.MagicMock()
50         stubs.api._rootDesc = get_rootDesc()
51
52         stubs.api.users().current().execute.return_value = {
53             "uuid": stubs.fake_user_uuid,
54         }
55         stubs.api.collections().list().execute.return_value = {"items": []}
56         stubs.api.collections().create().execute.side_effect = ({
57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
58             "portable_data_hash": "99999999999999999999999999999991+99",
59             "manifest_text": ""
60         }, {
61             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
62             "portable_data_hash": "99999999999999999999999999999992+99",
63             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
64         },
65         {
66             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
67             "portable_data_hash": "99999999999999999999999999999994+99",
68             "manifest_text": ""
69         },
70         {
71             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
72             "portable_data_hash": "99999999999999999999999999999995+99",
73             "manifest_text": ""
74         },
75         {
76             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
77             "portable_data_hash": "99999999999999999999999999999996+99",
78             "manifest_text": ""
79         }
80         )
81         stubs.api.collections().get().execute.return_value = {
82             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
83
84         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
85         stubs.api.jobs().create().execute.return_value = {
86             "uuid": stubs.expect_job_uuid,
87             "state": "Queued",
88         }
89
90         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
91         stubs.api.container_requests().create().execute.return_value = {
92             "uuid": stubs.expect_container_request_uuid,
93             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
94             "state": "Queued"
95         }
96
97         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
98         stubs.api.pipeline_templates().create().execute.return_value = {
99             "uuid": stubs.expect_pipeline_template_uuid,
100         }
101         stubs.expect_job_spec = {
102             'runtime_constraints': {
103                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
104                 'min_ram_mb_per_node': 1024
105             },
106             'script_parameters': {
107                 'x': {
108                     'basename': 'blorp.txt',
109                     'location': 'keep:99999999999999999999999999999992+99/blorp.txt',
110                     'class': 'File'
111                 },
112                 'y': {
113                     'basename': '99999999999999999999999999999998+99',
114                     'location': 'keep:99999999999999999999999999999998+99',
115                     'class': 'Directory'
116                 },
117                 'z': {
118                     'basename': 'anonymous',
119                     "listing": [{
120                         "basename": "renamed.txt",
121                         "class": "File",
122                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
123                     }],
124                     'class': 'Directory'
125                 },
126                 'cwl:tool':
127                 '99999999999999999999999999999994+99/workflow.cwl#main'
128             },
129             'repository': 'arvados',
130             'script_version': 'master',
131             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
132             'script': 'cwl-runner'
133         }
134         stubs.pipeline_component = stubs.expect_job_spec.copy()
135         stubs.expect_pipeline_instance = {
136             'name': 'submit_wf.cwl',
137             'state': 'RunningOnServer',
138             'owner_uuid': None,
139             "components": {
140                 "cwl-runner": {
141                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
142                     'script_parameters': {
143                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
144                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999992+99/blorp.txt'}},
145                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
146                               'listing': [
147                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
148                               ]}},
149                         'cwl:tool': '99999999999999999999999999999994+99/workflow.cwl#main',
150                         'arv:enable_reuse': True,
151                         'arv:on_error': 'continue'
152                     },
153                     'repository': 'arvados',
154                     'script_version': 'master',
155                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
156                     'script': 'cwl-runner',
157                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
158                 }
159             }
160         }
161         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
162         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
163         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
164         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
165         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
166             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
167             "state": "Queued"
168         }
169         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
170         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
171
172         with open("tests/wf/submit_wf_packed.cwl") as f:
173             expect_packed_workflow = yaml.round_trip_load(f)
174
175         stubs.expect_container_spec = {
176             'priority': 1,
177             'mounts': {
178                 '/var/spool/cwl': {
179                     'writable': True,
180                     'kind': 'collection'
181                 },
182                 '/var/lib/cwl/workflow.json': {
183                     'content': expect_packed_workflow,
184                     'kind': 'json'
185                 },
186                 'stdout': {
187                     'path': '/var/spool/cwl/cwl.output.json',
188                     'kind': 'file'
189                 },
190                 '/var/lib/cwl/cwl.input.json': {
191                     'kind': 'json',
192                     'content': {
193                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
194                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999992+99/blorp.txt'},
195                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
196                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
197                         ]}
198                     },
199                     'kind': 'json'
200                 }
201             },
202             'state': 'Committed',
203             'owner_uuid': None,
204             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
205                         '--enable-reuse', '--on-error=continue',
206                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
207             'name': 'submit_wf.cwl',
208             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
209             'output_path': '/var/spool/cwl',
210             'cwd': '/var/spool/cwl',
211             'runtime_constraints': {
212                 'API': True,
213                 'vcpus': 1,
214                 'ram': 1024*1024*1024
215             },
216             "properties": {}
217         }
218
219         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
220         stubs.api.workflows().create().execute.return_value = {
221             "uuid": stubs.expect_workflow_uuid,
222         }
223         def update_mock(**kwargs):
224             stubs.updated_uuid = kwargs.get('uuid')
225             return mock.DEFAULT
226         stubs.api.workflows().update.side_effect = update_mock
227         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
228             "uuid": stubs.updated_uuid,
229         }
230
231         return func(self, stubs, *args, **kwargs)
232     return wrapped
233
234
235 class TestSubmit(unittest.TestCase):
236     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
237     @mock.patch("time.sleep")
238     @stubs
239     def test_submit(self, stubs, tm, arvdock):
240         capture_stdout = cStringIO.StringIO()
241         exited = arvados_cwl.main(
242             ["--submit", "--no-wait", "--api=jobs", "--debug",
243              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
244             capture_stdout, sys.stderr, api_client=stubs.api)
245         self.assertEqual(exited, 0)
246
247         stubs.api.collections().create.assert_has_calls([
248             mock.call(),
249             mock.call(body=JsonDiffMatcher({
250                 'manifest_text':
251                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
252                 'owner_uuid': None,
253                 'name': 'submit_tool.cwl dependencies',
254             }), ensure_unique_name=True),
255             mock.call().execute(),
256             mock.call(body=JsonDiffMatcher({
257                 'manifest_text':
258                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
259                 'owner_uuid': None,
260                 'name': 'submit_wf.cwl input',
261             }), ensure_unique_name=True),
262             mock.call().execute()])
263
264         arvdock.assert_has_calls([
265             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
266             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
267         ])
268
269         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
270         stubs.api.pipeline_instances().create.assert_called_with(
271             body=JsonDiffMatcher(expect_pipeline))
272         self.assertEqual(capture_stdout.getvalue(),
273                          stubs.expect_pipeline_uuid + '\n')
274
275
276     @mock.patch("time.sleep")
277     @stubs
278     def test_submit_no_reuse(self, stubs, tm):
279         capture_stdout = cStringIO.StringIO()
280         exited = arvados_cwl.main(
281             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
282              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
283             capture_stdout, sys.stderr, api_client=stubs.api)
284         self.assertEqual(exited, 0)
285
286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
287         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
288
289         stubs.api.pipeline_instances().create.assert_called_with(
290             body=JsonDiffMatcher(expect_pipeline))
291         self.assertEqual(capture_stdout.getvalue(),
292                          stubs.expect_pipeline_uuid + '\n')
293
294     @mock.patch("time.sleep")
295     @stubs
296     def test_submit_on_error(self, stubs, tm):
297         capture_stdout = cStringIO.StringIO()
298         exited = arvados_cwl.main(
299             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
301             capture_stdout, sys.stderr, api_client=stubs.api)
302         self.assertEqual(exited, 0)
303
304         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
305         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
306
307         stubs.api.pipeline_instances().create.assert_called_with(
308             body=JsonDiffMatcher(expect_pipeline))
309         self.assertEqual(capture_stdout.getvalue(),
310                          stubs.expect_pipeline_uuid + '\n')
311
312
313     @mock.patch("time.sleep")
314     @stubs
315     def test_submit_runner_ram(self, stubs, tm):
316         capture_stdout = cStringIO.StringIO()
317         exited = arvados_cwl.main(
318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
320             capture_stdout, sys.stderr, api_client=stubs.api)
321         self.assertEqual(exited, 0)
322
323         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
324         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
325
326         stubs.api.pipeline_instances().create.assert_called_with(
327             body=JsonDiffMatcher(expect_pipeline))
328         self.assertEqual(capture_stdout.getvalue(),
329                          stubs.expect_pipeline_uuid + '\n')
330
331
332     @mock.patch("time.sleep")
333     @stubs
334     def test_submit_invalid_runner_ram(self, stubs, tm):
335         capture_stdout = cStringIO.StringIO()
336         exited = arvados_cwl.main(
337             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
338              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
339             capture_stdout, sys.stderr, api_client=stubs.api)
340         self.assertEqual(exited, 1)
341
342     @mock.patch("time.sleep")
343     @stubs
344     def test_submit_output_name(self, stubs, tm):
345         output_name = "test_output_name"
346
347         capture_stdout = cStringIO.StringIO()
348         exited = arvados_cwl.main(
349             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
351             capture_stdout, sys.stderr, api_client=stubs.api)
352         self.assertEqual(exited, 0)
353
354         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
355         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
356
357         stubs.api.pipeline_instances().create.assert_called_with(
358             body=JsonDiffMatcher(expect_pipeline))
359         self.assertEqual(capture_stdout.getvalue(),
360                          stubs.expect_pipeline_uuid + '\n')
361
362
363     @mock.patch("time.sleep")
364     @stubs
365     def test_submit_pipeline_name(self, stubs, tm):
366         capture_stdout = cStringIO.StringIO()
367         exited = arvados_cwl.main(
368             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
369              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
370             capture_stdout, sys.stderr, api_client=stubs.api)
371         self.assertEqual(exited, 0)
372
373         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
374         expect_pipeline["name"] = "hello job 123"
375
376         stubs.api.pipeline_instances().create.assert_called_with(
377             body=expect_pipeline)
378         self.assertEqual(capture_stdout.getvalue(),
379                          stubs.expect_pipeline_uuid + '\n')
380
381     @mock.patch("time.sleep")
382     @stubs
383     def test_submit_output_tags(self, stubs, tm):
384         output_tags = "tag0,tag1,tag2"
385
386         capture_stdout = cStringIO.StringIO()
387         exited = arvados_cwl.main(
388             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
390             capture_stdout, sys.stderr, api_client=stubs.api)
391         self.assertEqual(exited, 0)
392
393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
395
396         stubs.api.pipeline_instances().create.assert_called_with(
397             body=JsonDiffMatcher(expect_pipeline))
398         self.assertEqual(capture_stdout.getvalue(),
399                          stubs.expect_pipeline_uuid + '\n')
400
401     @mock.patch("time.sleep")
402     @stubs
403     def test_submit_with_project_uuid(self, stubs, tm):
404         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
405
406         exited = arvados_cwl.main(
407             ["--submit", "--no-wait",
408              "--project-uuid", project_uuid,
409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
410             sys.stdout, sys.stderr, api_client=stubs.api)
411         self.assertEqual(exited, 0)
412
413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
414         expect_pipeline["owner_uuid"] = project_uuid
415         stubs.api.pipeline_instances().create.assert_called_with(
416             body=JsonDiffMatcher(expect_pipeline))
417
418     @stubs
419     def test_submit_container(self, stubs):
420         capture_stdout = cStringIO.StringIO()
421         try:
422             exited = arvados_cwl.main(
423                 ["--submit", "--no-wait", "--api=containers", "--debug",
424                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
425                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
426             self.assertEqual(exited, 0)
427         except:
428             logging.exception("")
429
430         stubs.api.collections().create.assert_has_calls([
431             mock.call(),
432             mock.call(body=JsonDiffMatcher({
433                 'manifest_text':
434                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
435                 'owner_uuid': None,
436                 'name': 'submit_tool.cwl dependencies',
437             }), ensure_unique_name=True),
438             mock.call().execute(),
439             mock.call(body=JsonDiffMatcher({
440                 'manifest_text':
441                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
442                 'owner_uuid': None,
443                 'name': 'submit_wf.cwl input',
444             }), ensure_unique_name=True),
445             mock.call().execute()])
446
447         expect_container = copy.deepcopy(stubs.expect_container_spec)
448         stubs.api.container_requests().create.assert_called_with(
449             body=JsonDiffMatcher(expect_container))
450         self.assertEqual(capture_stdout.getvalue(),
451                          stubs.expect_container_request_uuid + '\n')
452
453     @stubs
454     def test_submit_container_no_reuse(self, stubs):
455         capture_stdout = cStringIO.StringIO()
456         try:
457             exited = arvados_cwl.main(
458                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
459                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
460                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
461             self.assertEqual(exited, 0)
462         except:
463             logging.exception("")
464
465         expect_container = copy.deepcopy(stubs.expect_container_spec)
466         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
467                                                   '--disable-reuse', '--on-error=continue',
468                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
469
470         stubs.api.container_requests().create.assert_called_with(
471             body=JsonDiffMatcher(expect_container))
472         self.assertEqual(capture_stdout.getvalue(),
473                          stubs.expect_container_request_uuid + '\n')
474
475
476     @stubs
477     def test_submit_container_on_error(self, stubs):
478         capture_stdout = cStringIO.StringIO()
479         try:
480             exited = arvados_cwl.main(
481                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
482                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
483                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
484             self.assertEqual(exited, 0)
485         except:
486             logging.exception("")
487
488         expect_container = copy.deepcopy(stubs.expect_container_spec)
489         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
490                                                   '--enable-reuse', '--on-error=stop',
491                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
492
493         stubs.api.container_requests().create.assert_called_with(
494             body=JsonDiffMatcher(expect_container))
495         self.assertEqual(capture_stdout.getvalue(),
496                          stubs.expect_container_request_uuid + '\n')
497
498     @stubs
499     def test_submit_container_output_name(self, stubs):
500         output_name = "test_output_name"
501
502         capture_stdout = cStringIO.StringIO()
503         try:
504             exited = arvados_cwl.main(
505                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
506                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
507                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
508             self.assertEqual(exited, 0)
509         except:
510             logging.exception("")
511
512         expect_container = copy.deepcopy(stubs.expect_container_spec)
513         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
514                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
515                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
516         expect_container["output_name"] = output_name
517
518         stubs.api.container_requests().create.assert_called_with(
519             body=JsonDiffMatcher(expect_container))
520         self.assertEqual(capture_stdout.getvalue(),
521                          stubs.expect_container_request_uuid + '\n')
522
523     @stubs
524     def test_submit_container_output_tags(self, stubs):
525         output_tags = "tag0,tag1,tag2"
526
527         capture_stdout = cStringIO.StringIO()
528         try:
529             exited = arvados_cwl.main(
530                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
531                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
532                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
533             self.assertEqual(exited, 0)
534         except:
535             logging.exception("")
536
537         expect_container = copy.deepcopy(stubs.expect_container_spec)
538         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
539                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
540                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
541
542         stubs.api.container_requests().create.assert_called_with(
543             body=JsonDiffMatcher(expect_container))
544         self.assertEqual(capture_stdout.getvalue(),
545                          stubs.expect_container_request_uuid + '\n')
546
547     @stubs
548     def test_submit_container_runner_ram(self, stubs):
549         capture_stdout = cStringIO.StringIO()
550         try:
551             exited = arvados_cwl.main(
552                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
553                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
554                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
555             self.assertEqual(exited, 0)
556         except:
557             logging.exception("")
558
559         expect_container = copy.deepcopy(stubs.expect_container_spec)
560         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
561
562         stubs.api.container_requests().create.assert_called_with(
563             body=JsonDiffMatcher(expect_container))
564         self.assertEqual(capture_stdout.getvalue(),
565                          stubs.expect_container_request_uuid + '\n')
566
567     @mock.patch("arvados.collection.CollectionReader")
568     @mock.patch("time.sleep")
569     @stubs
570     def test_submit_file_keepref(self, stubs, tm, collectionReader):
571         capture_stdout = cStringIO.StringIO()
572         exited = arvados_cwl.main(
573             ["--submit", "--no-wait", "--api=containers", "--debug",
574              "tests/wf/submit_keepref_wf.cwl"],
575             capture_stdout, sys.stderr, api_client=stubs.api)
576         self.assertEqual(exited, 0)
577
578
579     @mock.patch("arvados.collection.CollectionReader")
580     @mock.patch("time.sleep")
581     @stubs
582     def test_submit_keepref(self, stubs, tm, reader):
583         capture_stdout = cStringIO.StringIO()
584
585         with open("tests/wf/expect_arvworkflow.cwl") as f:
586             reader().open().__enter__().read.return_value = f.read()
587
588         exited = arvados_cwl.main(
589             ["--submit", "--no-wait", "--api=containers", "--debug",
590              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
591             capture_stdout, sys.stderr, api_client=stubs.api)
592         self.assertEqual(exited, 0)
593
594         expect_container = {
595             'priority': 1,
596             'mounts': {
597                 '/var/spool/cwl': {
598                     'writable': True,
599                     'kind': 'collection'
600                 },
601                 'stdout': {
602                     'path': '/var/spool/cwl/cwl.output.json',
603                     'kind': 'file'
604                 },
605                 '/var/lib/cwl/workflow': {
606                     'portable_data_hash': '99999999999999999999999999999994+99',
607                     'kind': 'collection'
608                 },
609                 '/var/lib/cwl/cwl.input.json': {
610                     'content': {
611                         'x': 'XxX'
612                     },
613                     'kind': 'json'
614                 }
615             }, 'state': 'Committed',
616             'owner_uuid': None,
617             'output_path': '/var/spool/cwl',
618             'name': 'expect_arvworkflow.cwl#main',
619             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
620             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
621                         '--enable-reuse', '--on-error=continue',
622                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
623             'cwd': '/var/spool/cwl',
624             'runtime_constraints': {
625                 'API': True,
626                 'vcpus': 1,
627                 'ram': 1073741824
628             },
629             "properties": {}
630         }
631
632         stubs.api.container_requests().create.assert_called_with(
633             body=JsonDiffMatcher(expect_container))
634         self.assertEqual(capture_stdout.getvalue(),
635                          stubs.expect_container_request_uuid + '\n')
636
637
638     @mock.patch("arvados.collection.CollectionReader")
639     @mock.patch("time.sleep")
640     @stubs
641     def test_submit_jobs_keepref(self, stubs, tm, reader):
642         capture_stdout = cStringIO.StringIO()
643
644         with open("tests/wf/expect_arvworkflow.cwl") as f:
645             reader().open().__enter__().read.return_value = f.read()
646
647         exited = arvados_cwl.main(
648             ["--submit", "--no-wait", "--api=jobs", "--debug",
649              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
650             capture_stdout, sys.stderr, api_client=stubs.api)
651         self.assertEqual(exited, 0)
652
653         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
654         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
655         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
656         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
657         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
658         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
659         stubs.api.pipeline_instances().create.assert_called_with(
660             body=JsonDiffMatcher(expect_pipeline))
661
662     @mock.patch("time.sleep")
663     @stubs
664     def test_submit_arvworkflow(self, stubs, tm):
665         capture_stdout = cStringIO.StringIO()
666
667         with open("tests/wf/expect_arvworkflow.cwl") as f:
668             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
669
670         exited = arvados_cwl.main(
671             ["--submit", "--no-wait", "--api=containers", "--debug",
672              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
673             capture_stdout, sys.stderr, api_client=stubs.api)
674         self.assertEqual(exited, 0)
675
676         expect_container = {
677             'priority': 1,
678             'mounts': {
679                 '/var/spool/cwl': {
680                     'writable': True,
681                     'kind': 'collection'
682                 },
683                 'stdout': {
684                     'path': '/var/spool/cwl/cwl.output.json',
685                     'kind': 'file'
686                 },
687                 '/var/lib/cwl/workflow.json': {
688                     'kind': 'json',
689                     'content': {
690                         'cwlVersion': 'v1.0',
691                         '$graph': [
692                             {
693                                 'id': '#main',
694                                 'inputs': [
695                                     {'type': 'string', 'id': '#main/x'}
696                                 ],
697                                 'steps': [
698                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
699                                      'run': '#submit_tool.cwl',
700                                      'id': '#main/step1',
701                                      'out': []}
702                                 ],
703                                 'class': 'Workflow',
704                                 'outputs': []
705                             },
706                             {
707                                 'inputs': [
708                                     {
709                                         'inputBinding': {'position': 1},
710                                         'type': 'string',
711                                         'id': '#submit_tool.cwl/x'}
712                                 ],
713                                 'requirements': [
714                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
715                                 ],
716                                 'id': '#submit_tool.cwl',
717                                 'outputs': [],
718                                 'baseCommand': 'cat',
719                                 'class': 'CommandLineTool'
720                             }
721                         ]
722                     }
723                 },
724                 '/var/lib/cwl/cwl.input.json': {
725                     'content': {
726                         'x': 'XxX'
727                     },
728                     'kind': 'json'
729                 }
730             }, 'state': 'Committed',
731             'owner_uuid': None,
732             'output_path': '/var/spool/cwl',
733             'name': 'a test workflow',
734             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
735             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
736                         '--enable-reuse', '--on-error=continue',
737                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
738             'cwd': '/var/spool/cwl',
739             'runtime_constraints': {
740                 'API': True,
741                 'vcpus': 1,
742                 'ram': 1073741824
743             },
744             "properties": {
745                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
746             }
747         }
748
749         stubs.api.container_requests().create.assert_called_with(
750             body=JsonDiffMatcher(expect_container))
751         self.assertEqual(capture_stdout.getvalue(),
752                          stubs.expect_container_request_uuid + '\n')
753
754
755     @stubs
756     def test_submit_container_name(self, stubs):
757         capture_stdout = cStringIO.StringIO()
758         try:
759             exited = arvados_cwl.main(
760                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
761                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
762                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
763             self.assertEqual(exited, 0)
764         except:
765             logging.exception("")
766
767         expect_container = copy.deepcopy(stubs.expect_container_spec)
768         expect_container["name"] = "hello container 123"
769
770         stubs.api.container_requests().create.assert_called_with(
771             body=JsonDiffMatcher(expect_container))
772         self.assertEqual(capture_stdout.getvalue(),
773                          stubs.expect_container_request_uuid + '\n')
774
775
776     @stubs
777     def test_submit_job_runner_image(self, stubs):
778         capture_stdout = cStringIO.StringIO()
779         try:
780             exited = arvados_cwl.main(
781                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
782                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
783                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
784             self.assertEqual(exited, 0)
785         except:
786             logging.exception("")
787
788         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
789
790         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
791         stubs.api.pipeline_instances().create.assert_called_with(
792             body=JsonDiffMatcher(expect_pipeline))
793         self.assertEqual(capture_stdout.getvalue(),
794                          stubs.expect_pipeline_uuid + '\n')
795
796     @stubs
797     def test_submit_container_runner_image(self, stubs):
798         capture_stdout = cStringIO.StringIO()
799         try:
800             exited = arvados_cwl.main(
801                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
802                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
803                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
804             self.assertEqual(exited, 0)
805         except:
806             logging.exception("")
807
808         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
809
810         expect_container = copy.deepcopy(stubs.expect_container_spec)
811         stubs.api.container_requests().create.assert_called_with(
812             body=JsonDiffMatcher(expect_container))
813         self.assertEqual(capture_stdout.getvalue(),
814                          stubs.expect_container_request_uuid + '\n')
815
816
817     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
818     @mock.patch("cwltool.docker.get_image")
819     @mock.patch("arvados.api")
820     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
821         arvrunner = mock.MagicMock()
822         arvrunner.project_uuid = ""
823         api.return_value = mock.MagicMock()
824         arvrunner.api = api.return_value
825         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
826                                                             {"items": [], "items_available": 0, "offset": 0},
827                                                             {"items": [], "items_available": 0, "offset": 0},
828                                                             {"items": [{"created_at": "",
829                                                                         "head_uuid": "",
830                                                                         "link_class": "docker_image_hash",
831                                                                         "name": "123456",
832                                                                         "owner_uuid": "",
833                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
834                                                             {"items": [], "items_available": 0, "offset": 0},
835                                                             {"items": [{"created_at": "",
836                                                                         "head_uuid": "",
837                                                                         "link_class": "docker_image_repo+tag",
838                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
839                                                                         "owner_uuid": "",
840                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
841                                                             {"items": [{"created_at": "",
842                                                                         "head_uuid": "",
843                                                                         "link_class": "docker_image_hash",
844                                                                         "name": "123456",
845                                                                         "owner_uuid": "",
846                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
847         )
848         find_one_image_hash.return_value = "123456"
849
850         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
851                                                                   {"items": [{"uuid": "",
852                                                                               "owner_uuid": "",
853                                                                               "manifest_text": "",
854                                                                               "properties": ""
855                                                                           }], "items_available": 1, "offset": 0},
856                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
857         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
858         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
859
860 class TestCreateTemplate(unittest.TestCase):
861     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
862
863     def _adjust_script_params(self, expect_component):
864         expect_component['script_parameters']['x'] = {
865             'dataclass': 'File',
866             'required': True,
867             'type': 'File',
868             'value': '99999999999999999999999999999992+99/blorp.txt',
869         }
870         expect_component['script_parameters']['y'] = {
871             'dataclass': 'Collection',
872             'required': True,
873             'type': 'Directory',
874             'value': '99999999999999999999999999999998+99',
875         }
876         expect_component['script_parameters']['z'] = {
877             'dataclass': 'Collection',
878             'required': True,
879             'type': 'Directory',
880         }
881
882     @stubs
883     def test_create(self, stubs):
884         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
885
886         capture_stdout = cStringIO.StringIO()
887
888         exited = arvados_cwl.main(
889             ["--create-workflow", "--debug",
890              "--api=jobs",
891              "--project-uuid", project_uuid,
892              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
893             capture_stdout, sys.stderr, api_client=stubs.api)
894         self.assertEqual(exited, 0)
895
896         stubs.api.pipeline_instances().create.refute_called()
897         stubs.api.jobs().create.refute_called()
898
899         expect_component = copy.deepcopy(stubs.expect_job_spec)
900         self._adjust_script_params(expect_component)
901         expect_template = {
902             "components": {
903                 "submit_wf.cwl": expect_component,
904             },
905             "name": "submit_wf.cwl",
906             "owner_uuid": project_uuid,
907         }
908         stubs.api.pipeline_templates().create.assert_called_with(
909             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
910
911         self.assertEqual(capture_stdout.getvalue(),
912                          stubs.expect_pipeline_template_uuid + '\n')
913
914
915     @stubs
916     def test_create_name(self, stubs):
917         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
918
919         capture_stdout = cStringIO.StringIO()
920
921         exited = arvados_cwl.main(
922             ["--create-workflow", "--debug",
923              "--project-uuid", project_uuid,
924              "--api=jobs",
925              "--name", "testing 123",
926              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
927             capture_stdout, sys.stderr, api_client=stubs.api)
928         self.assertEqual(exited, 0)
929
930         stubs.api.pipeline_instances().create.refute_called()
931         stubs.api.jobs().create.refute_called()
932
933         expect_component = copy.deepcopy(stubs.expect_job_spec)
934         self._adjust_script_params(expect_component)
935         expect_template = {
936             "components": {
937                 "testing 123": expect_component,
938             },
939             "name": "testing 123",
940             "owner_uuid": project_uuid,
941         }
942         stubs.api.pipeline_templates().create.assert_called_with(
943             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
944
945         self.assertEqual(capture_stdout.getvalue(),
946                          stubs.expect_pipeline_template_uuid + '\n')
947
948
949     @stubs
950     def test_update_name(self, stubs):
951         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
952
953         capture_stdout = cStringIO.StringIO()
954
955         exited = arvados_cwl.main(
956             ["--update-workflow", self.existing_template_uuid,
957              "--debug",
958              "--project-uuid", project_uuid,
959              "--api=jobs",
960              "--name", "testing 123",
961              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
962             capture_stdout, sys.stderr, api_client=stubs.api)
963         self.assertEqual(exited, 0)
964
965         stubs.api.pipeline_instances().create.refute_called()
966         stubs.api.jobs().create.refute_called()
967
968         expect_component = copy.deepcopy(stubs.expect_job_spec)
969         self._adjust_script_params(expect_component)
970         expect_template = {
971             "components": {
972                 "testing 123": expect_component,
973             },
974             "name": "testing 123",
975             "owner_uuid": project_uuid,
976         }
977         stubs.api.pipeline_templates().create.refute_called()
978         stubs.api.pipeline_templates().update.assert_called_with(
979             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
980
981         self.assertEqual(capture_stdout.getvalue(),
982                          self.existing_template_uuid + '\n')
983
984
985 class TestCreateWorkflow(unittest.TestCase):
986     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
987     expect_workflow = open("tests/wf/expect_packed.cwl").read()
988
989     @stubs
990     def test_create(self, stubs):
991         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
992
993         capture_stdout = cStringIO.StringIO()
994
995         exited = arvados_cwl.main(
996             ["--create-workflow", "--debug",
997              "--api=containers",
998              "--project-uuid", project_uuid,
999              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1000             capture_stdout, sys.stderr, api_client=stubs.api)
1001         self.assertEqual(exited, 0)
1002
1003         stubs.api.pipeline_templates().create.refute_called()
1004         stubs.api.container_requests().create.refute_called()
1005
1006         body = {
1007             "workflow": {
1008                 "owner_uuid": project_uuid,
1009                 "name": "submit_wf.cwl",
1010                 "description": "",
1011                 "definition": self.expect_workflow,
1012             }
1013         }
1014         stubs.api.workflows().create.assert_called_with(
1015             body=JsonDiffMatcher(body))
1016
1017         self.assertEqual(capture_stdout.getvalue(),
1018                          stubs.expect_workflow_uuid + '\n')
1019
1020
1021     @stubs
1022     def test_create_name(self, stubs):
1023         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1024
1025         capture_stdout = cStringIO.StringIO()
1026
1027         exited = arvados_cwl.main(
1028             ["--create-workflow", "--debug",
1029              "--api=containers",
1030              "--project-uuid", project_uuid,
1031              "--name", "testing 123",
1032              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1033             capture_stdout, sys.stderr, api_client=stubs.api)
1034         self.assertEqual(exited, 0)
1035
1036         stubs.api.pipeline_templates().create.refute_called()
1037         stubs.api.container_requests().create.refute_called()
1038
1039         body = {
1040             "workflow": {
1041                 "owner_uuid": project_uuid,
1042                 "name": "testing 123",
1043                 "description": "",
1044                 "definition": self.expect_workflow,
1045             }
1046         }
1047         stubs.api.workflows().create.assert_called_with(
1048             body=JsonDiffMatcher(body))
1049
1050         self.assertEqual(capture_stdout.getvalue(),
1051                          stubs.expect_workflow_uuid + '\n')
1052
1053     @stubs
1054     def test_incompatible_api(self, stubs):
1055         capture_stderr = cStringIO.StringIO()
1056         logging.getLogger('arvados.cwl-runner').addHandler(
1057             logging.StreamHandler(capture_stderr))
1058
1059         exited = arvados_cwl.main(
1060             ["--update-workflow", self.existing_workflow_uuid,
1061              "--api=jobs",
1062              "--debug",
1063              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1064             sys.stderr, sys.stderr, api_client=stubs.api)
1065         self.assertEqual(exited, 1)
1066         self.assertRegexpMatches(
1067             capture_stderr.getvalue(),
1068             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1069
1070     @stubs
1071     def test_update(self, stubs):
1072         capture_stdout = cStringIO.StringIO()
1073
1074         exited = arvados_cwl.main(
1075             ["--update-workflow", self.existing_workflow_uuid,
1076              "--debug",
1077              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1078             capture_stdout, sys.stderr, api_client=stubs.api)
1079         self.assertEqual(exited, 0)
1080
1081         body = {
1082             "workflow": {
1083                 "name": "submit_wf.cwl",
1084                 "description": "",
1085                 "definition": self.expect_workflow,
1086             }
1087         }
1088         stubs.api.workflows().update.assert_called_with(
1089             uuid=self.existing_workflow_uuid,
1090             body=JsonDiffMatcher(body))
1091         self.assertEqual(capture_stdout.getvalue(),
1092                          self.existing_workflow_uuid + '\n')
1093
1094
1095     @stubs
1096     def test_update_name(self, stubs):
1097         capture_stdout = cStringIO.StringIO()
1098
1099         exited = arvados_cwl.main(
1100             ["--update-workflow", self.existing_workflow_uuid,
1101              "--debug", "--name", "testing 123",
1102              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1103             capture_stdout, sys.stderr, api_client=stubs.api)
1104         self.assertEqual(exited, 0)
1105
1106         body = {
1107             "workflow": {
1108                 "name": "testing 123",
1109                 "description": "",
1110                 "definition": self.expect_workflow,
1111             }
1112         }
1113         stubs.api.workflows().update.assert_called_with(
1114             uuid=self.existing_workflow_uuid,
1115             body=JsonDiffMatcher(body))
1116         self.assertEqual(capture_stdout.getvalue(),
1117                          self.existing_workflow_uuid + '\n')
1118
1119
1120 class TestTemplateInputs(unittest.TestCase):
1121     expect_template = {
1122         "components": {
1123             "inputs_test.cwl": {
1124                 'runtime_constraints': {
1125                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1126                     'min_ram_mb_per_node': 1024
1127                 },
1128                 'script_parameters': {
1129                     'cwl:tool':
1130                     '99999999999999999999999999999992+99/workflow.cwl#main',
1131                     'optionalFloatInput': None,
1132                     'fileInput': {
1133                         'type': 'File',
1134                         'dataclass': 'File',
1135                         'required': True,
1136                         'title': "It's a file; we expect to find some characters in it.",
1137                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1138                     },
1139                     'floatInput': {
1140                         'type': 'float',
1141                         'dataclass': 'number',
1142                         'required': True,
1143                         'title': 'Floats like a duck',
1144                         'default': 0.1,
1145                         'value': 0.1,
1146                     },
1147                     'optionalFloatInput': {
1148                         'type': ['null', 'float'],
1149                         'dataclass': 'number',
1150                         'required': False,
1151                     },
1152                     'boolInput': {
1153                         'type': 'boolean',
1154                         'dataclass': 'boolean',
1155                         'required': True,
1156                         'title': 'True or false?',
1157                     },
1158                 },
1159                 'repository': 'arvados',
1160                 'script_version': 'master',
1161                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1162                 'script': 'cwl-runner',
1163             },
1164         },
1165         "name": "inputs_test.cwl",
1166     }
1167
1168     @stubs
1169     def test_inputs_empty(self, stubs):
1170         exited = arvados_cwl.main(
1171             ["--create-template",
1172              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1173             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1174         self.assertEqual(exited, 0)
1175
1176         stubs.api.pipeline_templates().create.assert_called_with(
1177             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1178
1179     @stubs
1180     def test_inputs(self, stubs):
1181         exited = arvados_cwl.main(
1182             ["--create-template",
1183              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1184             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1185         self.assertEqual(exited, 0)
1186
1187         expect_template = copy.deepcopy(self.expect_template)
1188         params = expect_template[
1189             "components"]["inputs_test.cwl"]["script_parameters"]
1190         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
1191         params["cwl:tool"] = '99999999999999999999999999999994+99/workflow.cwl#main'
1192         params["floatInput"]["value"] = 1.234
1193         params["boolInput"]["value"] = True
1194
1195         stubs.api.pipeline_templates().create.assert_called_with(
1196             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)