Merge branch '10259-cwl-pipeline-value' refs #10259
[arvados.git] / sdk / cwl / tests / test_submit.py
1 import arvados
2 import arvados.keep
3 import arvados.collection
4 import arvados_cwl
5 import copy
6 import cStringIO
7 import functools
8 import hashlib
9 import mock
10 import sys
11 import unittest
12 import json
13 import logging
14
15 from .matcher import JsonDiffMatcher
16
17
18 def stubs(func):
19     @functools.wraps(func)
20     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
21     @mock.patch("arvados.collection.KeepClient")
22     @mock.patch("arvados.events.subscribe")
23     def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
24         class Stubs:
25             pass
26         stubs = Stubs()
27         stubs.events = events
28         stubs.keepdocker = keepdocker
29         stubs.keep_client = keep_client
30
31         def putstub(p, **kwargs):
32             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
33         stubs.keep_client().put.side_effect = putstub
34         stubs.keep_client.put.side_effect = putstub
35
36         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
37         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
38
39
40         stubs.api = mock.MagicMock()
41         stubs.api.users().current().execute.return_value = {
42             "uuid": stubs.fake_user_uuid,
43         }
44         stubs.api.collections().list().execute.return_value = {"items": []}
45         stubs.api.collections().create().execute.side_effect = ({
46             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
47             "portable_data_hash": "99999999999999999999999999999991+99",
48             "manifest_text": ""
49         }, {
50             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
51             "portable_data_hash": "99999999999999999999999999999992+99",
52             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
53         },
54         {
55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
56             "portable_data_hash": "99999999999999999999999999999994+99",
57             "manifest_text": ""
58         },
59         {
60             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
61             "portable_data_hash": "99999999999999999999999999999995+99",
62             "manifest_text": ""
63         },
64         {
65             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
66             "portable_data_hash": "99999999999999999999999999999996+99",
67             "manifest_text": ""
68         }
69         )
70         stubs.api.collections().get().execute.return_value = {
71             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
72
73         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
74         stubs.api.jobs().create().execute.return_value = {
75             "uuid": stubs.expect_job_uuid,
76             "state": "Queued",
77         }
78
79         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
80         stubs.api.container_requests().create().execute.return_value = {
81             "uuid": stubs.expect_container_request_uuid,
82             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
83             "state": "Queued"
84         }
85
86         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
87         stubs.api.pipeline_templates().create().execute.return_value = {
88             "uuid": stubs.expect_pipeline_template_uuid,
89         }
90         stubs.expect_job_spec = {
91             'runtime_constraints': {
92                 'docker_image': 'arvados/jobs'
93             },
94             'script_parameters': {
95                 'x': {
96                     'basename': 'blorp.txt',
97                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
98                     'class': 'File'
99                 },
100                 'y': {
101                     'basename': '99999999999999999999999999999998+99',
102                     'location': 'keep:99999999999999999999999999999998+99',
103                     'class': 'Directory'
104                 },
105                 'z': {
106                     'basename': 'anonymous',
107                     "listing": [{
108                         "basename": "renamed.txt",
109                         "class": "File",
110                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
111                     }],
112                     'class': 'Directory'
113                 },
114                 'cwl:tool':
115                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
116             },
117             'repository': 'arvados',
118             'script_version': 'master',
119             'script': 'cwl-runner'
120         }
121         stubs.pipeline_component = stubs.expect_job_spec.copy()
122         stubs.expect_pipeline_instance = {
123             'name': 'submit_wf.cwl',
124             'state': 'RunningOnServer',
125             "components": {
126                 "cwl-runner": {
127                     'runtime_constraints': {'docker_image': 'arvados/jobs'},
128                     'script_parameters': {
129                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
130                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
131                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
132                               'listing': [
133                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
134                               ]}},
135                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
136                     },
137                     'repository': 'arvados',
138                     'script_version': 'master',
139                     'script': 'cwl-runner'
140                 }
141             }
142         }
143         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
144         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
145         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
146         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
147         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
148             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
149             "state": "Queued"
150         }
151         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
152         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
153
154         stubs.expect_container_spec = {
155             'priority': 1,
156             'mounts': {
157                 '/var/spool/cwl': {
158                     'writable': True,
159                     'kind': 'collection'
160                 },
161                 '/var/lib/cwl/workflow': {
162                     'portable_data_hash': '99999999999999999999999999999991+99',
163                     'kind': 'collection'
164                 },
165                 'stdout': {
166                     'path': '/var/spool/cwl/cwl.output.json',
167                     'kind': 'file'
168                 },
169                 '/var/lib/cwl/job/cwl.input.json': {
170                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
171                     'kind': 'collection'
172                 }
173             },
174             'state': 'Committed',
175             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
176             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
177             'name': 'submit_wf.cwl',
178             'container_image': '99999999999999999999999999999993+99',
179             'output_path': '/var/spool/cwl',
180             'cwd': '/var/spool/cwl',
181             'runtime_constraints': {
182                 'API': True,
183                 'vcpus': 1,
184                 'ram': 268435456
185             }
186         }
187
188         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
189         stubs.api.workflows().create().execute.return_value = {
190             "uuid": stubs.expect_workflow_uuid,
191         }
192
193         return func(self, stubs, *args, **kwargs)
194     return wrapped
195
196
197 class TestSubmit(unittest.TestCase):
198     @mock.patch("time.sleep")
199     @stubs
200     def test_submit(self, stubs, tm):
201         capture_stdout = cStringIO.StringIO()
202         exited = arvados_cwl.main(
203             ["--submit", "--no-wait", "--debug",
204              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
205             capture_stdout, sys.stderr, api_client=stubs.api)
206         self.assertEqual(exited, 0)
207
208         stubs.api.collections().create.assert_has_calls([
209             mock.call(),
210             mock.call(body={
211                 'manifest_text':
212                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
213                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
214                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
215                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
216                 'name': 'submit_wf.cwl',
217             }, ensure_unique_name=True),
218             mock.call().execute(),
219             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
220                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
221                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
222                             'replication_desired': None,
223                             'name': 'New collection'
224             }, ensure_unique_name=True),
225             mock.call().execute(num_retries=4),
226             mock.call(body={
227                 'manifest_text':
228                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
229                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
230                 'name': '#',
231             }, ensure_unique_name=True),
232             mock.call().execute()])
233
234         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
235         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
236         stubs.api.pipeline_instances().create.assert_called_with(
237             body=expect_pipeline)
238         self.assertEqual(capture_stdout.getvalue(),
239                          stubs.expect_pipeline_uuid + '\n')
240
241     @mock.patch("time.sleep")
242     @stubs
243     def test_submit_with_project_uuid(self, stubs, tm):
244         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
245
246         exited = arvados_cwl.main(
247             ["--submit", "--no-wait",
248              "--project-uuid", project_uuid,
249              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
250             sys.stdout, sys.stderr, api_client=stubs.api)
251         self.assertEqual(exited, 0)
252
253         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
254         expect_pipeline["owner_uuid"] = project_uuid
255         stubs.api.pipeline_instances().create.assert_called_with(
256             body=expect_pipeline)
257
258     @stubs
259     def test_submit_container(self, stubs):
260         capture_stdout = cStringIO.StringIO()
261         try:
262             exited = arvados_cwl.main(
263                 ["--submit", "--no-wait", "--api=containers", "--debug",
264                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
265                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
266             self.assertEqual(exited, 0)
267         except:
268             logging.exception("")
269
270         stubs.api.collections().create.assert_has_calls([
271             mock.call(),
272             mock.call(body={
273                 'manifest_text':
274                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
275                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
276                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
277                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
278                 'name': 'submit_wf.cwl',
279             }, ensure_unique_name=True),
280             mock.call().execute(),
281             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
282                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
283                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
284                             'name': 'New collection',
285                             'replication_desired': None,
286             }, ensure_unique_name=True),
287             mock.call().execute(num_retries=4),
288             mock.call(body={
289                 'manifest_text':
290                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
291                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
292                 'name': '#',
293             }, ensure_unique_name=True),
294             mock.call().execute()])
295
296         expect_container = copy.deepcopy(stubs.expect_container_spec)
297         expect_container["owner_uuid"] = stubs.fake_user_uuid
298         stubs.api.container_requests().create.assert_called_with(
299             body=expect_container)
300         self.assertEqual(capture_stdout.getvalue(),
301                          stubs.expect_container_request_uuid + '\n')
302
303
304 class TestCreateTemplate(unittest.TestCase):
305     @stubs
306     def test_create(self, stubs):
307         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
308
309         capture_stdout = cStringIO.StringIO()
310
311         exited = arvados_cwl.main(
312             ["--create-template", "--debug",
313              "--project-uuid", project_uuid,
314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
315             capture_stdout, sys.stderr, api_client=stubs.api)
316         self.assertEqual(exited, 0)
317
318         stubs.api.pipeline_instances().create.refute_called()
319         stubs.api.jobs().create.refute_called()
320
321         expect_component = copy.deepcopy(stubs.expect_job_spec)
322         expect_component['script_parameters']['x'] = {
323             'dataclass': 'File',
324             'required': True,
325             'type': 'File',
326             'value': '99999999999999999999999999999994+99/blorp.txt',
327         }
328         expect_component['script_parameters']['y'] = {
329             'dataclass': 'Collection',
330             'required': True,
331             'type': 'Directory',
332             'value': '99999999999999999999999999999998+99',
333         }
334         expect_component['script_parameters']['z'] = {
335             'dataclass': 'Collection',
336             'required': True,
337             'type': 'Directory',
338         }
339         expect_template = {
340             "components": {
341                 "submit_wf.cwl": expect_component,
342             },
343             "name": "submit_wf.cwl",
344             "owner_uuid": project_uuid,
345         }
346         stubs.api.pipeline_templates().create.assert_called_with(
347             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
348
349         self.assertEqual(capture_stdout.getvalue(),
350                          stubs.expect_pipeline_template_uuid + '\n')
351
352
353 class TestCreateWorkflow(unittest.TestCase):
354     @stubs
355     def test_create(self, stubs):
356         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
357
358         capture_stdout = cStringIO.StringIO()
359
360         exited = arvados_cwl.main(
361             ["--create-workflow", "--debug",
362              "--project-uuid", project_uuid,
363              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
364             capture_stdout, sys.stderr, api_client=stubs.api)
365         self.assertEqual(exited, 0)
366
367         stubs.api.pipeline_templates().create.refute_called()
368         stubs.api.container_requests().create.refute_called()
369
370         with open("tests/wf/expect_packed.cwl") as f:
371             expect_workflow = f.read()
372
373         body = {
374             "workflow": {
375                 "owner_uuid": project_uuid,
376                 "name": "submit_wf.cwl",
377                 "description": "",
378                 "definition": expect_workflow
379                 }
380         }
381         stubs.api.workflows().create.assert_called_with(
382             body=JsonDiffMatcher(body))
383
384         self.assertEqual(capture_stdout.getvalue(),
385                          stubs.expect_workflow_uuid + '\n')
386
387
388 class TestTemplateInputs(unittest.TestCase):
389     expect_template = {
390         "components": {
391             "inputs_test.cwl": {
392                 'runtime_constraints': {
393                     'docker_image': 'arvados/jobs',
394                 },
395                 'script_parameters': {
396                     'cwl:tool':
397                     '99999999999999999999999999999991+99/'
398                     'wf/inputs_test.cwl',
399                     'optionalFloatInput': None,
400                     'fileInput': {
401                         'type': 'File',
402                         'dataclass': 'File',
403                         'required': True,
404                         'title': "It's a file; we expect to find some characters in it.",
405                         'description': 'If there were anything further to say, it would be said here,\nor here.'
406                     },
407                     'floatInput': {
408                         'type': 'float',
409                         'dataclass': 'number',
410                         'required': True,
411                         'title': 'Floats like a duck',
412                         'default': 0.1,
413                         'value': 0.1,
414                     },
415                     'optionalFloatInput': {
416                         'type': ['null', 'float'],
417                         'dataclass': 'number',
418                         'required': False,
419                     },
420                     'boolInput': {
421                         'type': 'boolean',
422                         'dataclass': 'boolean',
423                         'required': True,
424                         'title': 'True or false?',
425                     },
426                 },
427                 'repository': 'arvados',
428                 'script_version': 'master',
429                 'script': 'cwl-runner',
430             },
431         },
432         "name": "inputs_test.cwl",
433     }
434
435     @stubs
436     def test_inputs_empty(self, stubs):
437         exited = arvados_cwl.main(
438             ["--create-template", "--no-wait",
439              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
440             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
441         self.assertEqual(exited, 0)
442
443         expect_template = copy.deepcopy(self.expect_template)
444         expect_template["owner_uuid"] = stubs.fake_user_uuid
445
446         stubs.api.pipeline_templates().create.assert_called_with(
447             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
448
449     @stubs
450     def test_inputs(self, stubs):
451         exited = arvados_cwl.main(
452             ["--create-template", "--no-wait",
453              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
454             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
455         self.assertEqual(exited, 0)
456
457         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
458
459         expect_template = copy.deepcopy(self.expect_template)
460         expect_template["owner_uuid"] = stubs.fake_user_uuid
461         params = expect_template[
462             "components"]["inputs_test.cwl"]["script_parameters"]
463         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
464         params["floatInput"]["value"] = 1.234
465         params["boolInput"]["value"] = True
466
467         stubs.api.pipeline_templates().create.assert_called_with(
468             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)