Merge branch 'master' into wtsi-hgi-8087-arv-cli-request-body-from-file
[arvados.git] / sdk / cwl / tests / test_submit.py
1 import arvados
2 import arvados.keep
3 import arvados.collection
4 import arvados_cwl
5 import copy
6 import cStringIO
7 import functools
8 import hashlib
9 import mock
10 import sys
11 import unittest
12
13 from .matcher import JsonDiffMatcher
14
15
16 def stubs(func):
17     @functools.wraps(func)
18     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
19     @mock.patch("arvados.collection.KeepClient")
20     @mock.patch("arvados.events.subscribe")
21     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
22         class Stubs:
23             pass
24         stubs = Stubs()
25         stubs.events = events
26         stubs.KeepClient = KeepClient
27         stubs.keepdocker = keepdocker
28
29         def putstub(p, **kwargs):
30             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
31         stubs.KeepClient().put.side_effect = putstub
32
33         stubs.keepdocker.return_value = True
34         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
35
36         stubs.api = mock.MagicMock()
37         stubs.api.users().current().execute.return_value = {
38             "uuid": stubs.fake_user_uuid,
39         }
40         stubs.api.collections().list().execute.return_value = {"items": []}
41         stubs.api.collections().create().execute.side_effect = ({
42             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
43             "portable_data_hash": "99999999999999999999999999999991+99",
44         }, {
45             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
46             "portable_data_hash": "99999999999999999999999999999992+99",
47         })
48         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
49         stubs.api.jobs().create().execute.return_value = {
50             "uuid": stubs.expect_job_uuid,
51             "state": "Queued",
52         }
53         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
54         stubs.api.pipeline_templates().create().execute.return_value = {
55             "uuid": stubs.expect_pipeline_template_uuid,
56         }
57         stubs.expect_job_spec = {
58             'runtime_constraints': {
59                 'docker_image': 'arvados/jobs'
60             },
61             'script_parameters': {
62                 'x': {
63                     'path': '99999999999999999999999999999992+99/blorp.txt',
64                     'class': 'File'
65                 },
66                 'cwl:tool':
67                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
68             },
69             'repository': 'arvados',
70             'script_version': 'master',
71             'script': 'cwl-runner'
72         }
73         return func(self, stubs, *args, **kwargs)
74     return wrapped
75
76
77 class TestSubmit(unittest.TestCase):
78     @stubs
79     def test_submit(self, stubs):
80         capture_stdout = cStringIO.StringIO()
81         exited = arvados_cwl.main(
82             ["--submit", "--no-wait",
83              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
84             capture_stdout, sys.stderr, api_client=stubs.api)
85         self.assertEqual(exited, 0)
86
87         stubs.api.collections().create.assert_has_calls([
88             mock.call(),
89             mock.call(body={
90                 'manifest_text':
91                 './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
92                 '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
93                 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
94                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
95                 'name': 'submit_wf.cwl',
96             }, ensure_unique_name=True),
97             mock.call().execute(),
98             mock.call(body={
99                 'manifest_text':
100                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
101                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
102                 'name': '#',
103             }, ensure_unique_name=True),
104             mock.call().execute()])
105
106         expect_job = copy.deepcopy(stubs.expect_job_spec)
107         expect_job["owner_uuid"] = stubs.fake_user_uuid
108         stubs.api.jobs().create.assert_called_with(
109             body=expect_job,
110             find_or_create=True)
111         self.assertEqual(capture_stdout.getvalue(),
112                          stubs.expect_job_uuid + '\n')
113
114     @stubs
115     def test_submit_with_project_uuid(self, stubs):
116         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
117
118         exited = arvados_cwl.main(
119             ["--submit", "--no-wait",
120              "--project-uuid", project_uuid,
121              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
122             sys.stdout, sys.stderr, api_client=stubs.api)
123         self.assertEqual(exited, 0)
124
125         expect_body = copy.deepcopy(stubs.expect_job_spec)
126         expect_body["owner_uuid"] = project_uuid
127         stubs.api.jobs().create.assert_called_with(
128             body=expect_body,
129             find_or_create=True)
130
131
132 class TestCreateTemplate(unittest.TestCase):
133     @stubs
134     def test_create(self, stubs):
135         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
136
137         capture_stdout = cStringIO.StringIO()
138
139         exited = arvados_cwl.main(
140             ["--create-template", "--no-wait",
141              "--project-uuid", project_uuid,
142              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
143             capture_stdout, sys.stderr, api_client=stubs.api)
144         self.assertEqual(exited, 0)
145
146         stubs.api.pipeline_instances().create.refute_called()
147         stubs.api.jobs().create.refute_called()
148
149         expect_component = copy.deepcopy(stubs.expect_job_spec)
150         expect_component['script_parameters']['x'] = {
151             'dataclass': 'File',
152             'required': True,
153             'type': 'File',
154             'value': '99999999999999999999999999999992+99/blorp.txt',
155         }
156         expect_template = {
157             "components": {
158                 "submit_wf.cwl": expect_component,
159             },
160             "name": "submit_wf.cwl",
161             "owner_uuid": project_uuid,
162         }
163         stubs.api.pipeline_templates().create.assert_called_with(
164             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
165
166         self.assertEqual(capture_stdout.getvalue(),
167                          stubs.expect_pipeline_template_uuid + '\n')
168
169
170 class TestTemplateInputs(unittest.TestCase):
171     expect_template = {
172         "components": {
173             "inputs_test.cwl": {
174                 'runtime_constraints': {
175                     'docker_image': 'arvados/jobs',
176                 },
177                 'script_parameters': {
178                     'cwl:tool':
179                     '99999999999999999999999999999991+99/'
180                     'wf/inputs_test.cwl',
181                     'optionalFloatInput': None,
182                     'fileInput': {
183                         'type': 'File',
184                         'dataclass': 'File',
185                         'required': True,
186                         'title': "It's a file; we expect to find some characters in it.",
187                         'description': 'If there were anything further to say, it would be said here,\nor here.'
188                     },
189                     'floatInput': {
190                         'type': 'float',
191                         'dataclass': 'number',
192                         'required': True,
193                         'title': 'Floats like a duck',
194                         'default': 0.1,
195                         'value': 0.1,
196                     },
197                     'optionalFloatInput': {
198                         'type': ['null', 'float'],
199                         'dataclass': 'number',
200                         'required': False,
201                     },
202                     'boolInput': {
203                         'type': 'boolean',
204                         'dataclass': 'boolean',
205                         'required': True,
206                         'title': 'True or false?',
207                     },
208                 },
209                 'repository': 'arvados',
210                 'script_version': 'master',
211                 'script': 'cwl-runner',
212             },
213         },
214         "name": "inputs_test.cwl",
215     }
216
217     @stubs
218     def test_inputs_empty(self, stubs):
219         exited = arvados_cwl.main(
220             ["--create-template", "--no-wait",
221              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
222             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
223         self.assertEqual(exited, 0)
224
225         expect_template = copy.deepcopy(self.expect_template)
226         expect_template["owner_uuid"] = stubs.fake_user_uuid
227
228         stubs.api.pipeline_templates().create.assert_called_with(
229             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
230
231     @stubs
232     def test_inputs(self, stubs):
233         exited = arvados_cwl.main(
234             ["--create-template", "--no-wait",
235              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
236             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
237         self.assertEqual(exited, 0)
238
239         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
240
241         expect_template = copy.deepcopy(self.expect_template)
242         expect_template["owner_uuid"] = stubs.fake_user_uuid
243         params = expect_template[
244             "components"]["inputs_test.cwl"]["script_parameters"]
245         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
246         params["floatInput"]["value"] = 1.234
247         params["boolInput"]["value"] = True
248
249         stubs.api.pipeline_templates().create.assert_called_with(
250             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)