3 import arvados.collection
13 from .matcher import JsonDiffMatcher
17 @functools.wraps(func)
18 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
19 @mock.patch("arvados.collection.KeepClient")
20 @mock.patch("arvados.events.subscribe")
21 def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
26 stubs.KeepClient = KeepClient
27 stubs.keepdocker = keepdocker
29 def putstub(p, **kwargs):
30 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
31 stubs.KeepClient().put.side_effect = putstub
33 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
34 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
36 stubs.api = mock.MagicMock()
37 stubs.api.users().current().execute.return_value = {
38 "uuid": stubs.fake_user_uuid,
40 stubs.api.collections().list().execute.return_value = {"items": []}
41 stubs.api.collections().create().execute.side_effect = ({
42 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
43 "portable_data_hash": "99999999999999999999999999999991+99",
45 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
46 "portable_data_hash": "99999999999999999999999999999992+99",
49 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
50 "portable_data_hash": "99999999999999999999999999999994+99",
53 stubs.api.collections().get().execute.return_value = {
54 "portable_data_hash": "99999999999999999999999999999993+99"}
56 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
57 stubs.api.jobs().create().execute.return_value = {
58 "uuid": stubs.expect_job_uuid,
62 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
63 stubs.api.container_requests().create().execute.return_value = {
64 "uuid": stubs.expect_container_request_uuid,
65 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
69 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
70 stubs.api.pipeline_templates().create().execute.return_value = {
71 "uuid": stubs.expect_pipeline_template_uuid,
73 stubs.expect_job_spec = {
74 'runtime_constraints': {
75 'docker_image': 'arvados/jobs'
77 'script_parameters': {
79 'path': '99999999999999999999999999999992+99/blorp.txt',
83 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
85 'repository': 'arvados',
86 'script_version': 'master',
87 'script': 'cwl-runner'
90 stubs.expect_container_spec = {
94 'path': '/var/spool/cwl/cwl.output.json',
97 '/var/lib/cwl/workflow/submit_wf.cwl': {
98 'portable_data_hash': '999999999999999999999999991+99',
101 '/var/lib/cwl/job/cwl.input.json': {
102 'portable_data_hash': '102435082199e5229f99b01165b67096+60/cwl.input.json',
106 'state': 'Committed',
107 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
108 'command': ['arvados-cwl-runner', '--local', '--crunch2', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
109 'name': 'submit_wf.cwl',
110 'container_image': '99999999999999999999999999999993+99',
111 'output_path': '/var/spool/cwl',
112 'cwd': '/var/spool/cwl',
113 'runtime_constraints': {
118 return func(self, stubs, *args, **kwargs)
122 class TestSubmit(unittest.TestCase):
124 def test_submit(self, stubs):
125 capture_stdout = cStringIO.StringIO()
126 exited = arvados_cwl.main(
127 ["--submit", "--no-wait",
128 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
129 capture_stdout, sys.stderr, api_client=stubs.api)
130 self.assertEqual(exited, 0)
132 stubs.api.collections().create.assert_has_calls([
136 './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
137 '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
138 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
139 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
140 'name': 'submit_wf.cwl',
141 }, ensure_unique_name=True),
142 mock.call().execute(),
145 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
146 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
148 }, ensure_unique_name=True),
149 mock.call().execute()])
151 expect_job = copy.deepcopy(stubs.expect_job_spec)
152 expect_job["owner_uuid"] = stubs.fake_user_uuid
153 stubs.api.jobs().create.assert_called_with(
156 self.assertEqual(capture_stdout.getvalue(),
157 stubs.expect_job_uuid + '\n')
160 def test_submit_with_project_uuid(self, stubs):
161 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
163 exited = arvados_cwl.main(
164 ["--submit", "--no-wait",
165 "--project-uuid", project_uuid,
166 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
167 sys.stdout, sys.stderr, api_client=stubs.api)
168 self.assertEqual(exited, 0)
170 expect_body = copy.deepcopy(stubs.expect_job_spec)
171 expect_body["owner_uuid"] = project_uuid
172 stubs.api.jobs().create.assert_called_with(
177 def test_submit_container(self, stubs):
178 capture_stdout = cStringIO.StringIO()
179 exited = arvados_cwl.main(
180 ["--submit", "--no-wait", "--crunch2", "--debug",
181 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
182 capture_stdout, sys.stderr, api_client=stubs.api)
183 self.assertEqual(exited, 0)
185 stubs.api.collections().create.assert_has_calls([
189 './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
190 '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
191 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
192 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
193 'name': 'submit_wf.cwl',
194 }, ensure_unique_name=True),
195 mock.call().execute(),
198 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
199 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
201 }, ensure_unique_name=True),
202 mock.call().execute()])
204 expect_container = copy.deepcopy(stubs.expect_container_spec)
205 expect_container["owner_uuid"] = stubs.fake_user_uuid
206 stubs.api.container_requests().create.assert_called_with(
207 body=expect_container)
208 self.assertEqual(capture_stdout.getvalue(),
209 stubs.expect_container_request_uuid + '\n')
212 class TestCreateTemplate(unittest.TestCase):
214 def test_create(self, stubs):
215 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
217 capture_stdout = cStringIO.StringIO()
219 exited = arvados_cwl.main(
220 ["--create-template", "--no-wait",
221 "--project-uuid", project_uuid,
222 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
223 capture_stdout, sys.stderr, api_client=stubs.api)
224 self.assertEqual(exited, 0)
226 stubs.api.pipeline_instances().create.refute_called()
227 stubs.api.jobs().create.refute_called()
229 expect_component = copy.deepcopy(stubs.expect_job_spec)
230 expect_component['script_parameters']['x'] = {
234 'value': '99999999999999999999999999999992+99/blorp.txt',
238 "submit_wf.cwl": expect_component,
240 "name": "submit_wf.cwl",
241 "owner_uuid": project_uuid,
243 stubs.api.pipeline_templates().create.assert_called_with(
244 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
246 self.assertEqual(capture_stdout.getvalue(),
247 stubs.expect_pipeline_template_uuid + '\n')
250 class TestTemplateInputs(unittest.TestCase):
254 'runtime_constraints': {
255 'docker_image': 'arvados/jobs',
257 'script_parameters': {
259 '99999999999999999999999999999991+99/'
260 'wf/inputs_test.cwl',
261 'optionalFloatInput': None,
266 'title': "It's a file; we expect to find some characters in it.",
267 'description': 'If there were anything further to say, it would be said here,\nor here.'
271 'dataclass': 'number',
273 'title': 'Floats like a duck',
277 'optionalFloatInput': {
278 'type': ['null', 'float'],
279 'dataclass': 'number',
284 'dataclass': 'boolean',
286 'title': 'True or false?',
289 'repository': 'arvados',
290 'script_version': 'master',
291 'script': 'cwl-runner',
294 "name": "inputs_test.cwl",
298 def test_inputs_empty(self, stubs):
299 exited = arvados_cwl.main(
300 ["--create-template", "--no-wait",
301 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
302 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
303 self.assertEqual(exited, 0)
305 expect_template = copy.deepcopy(self.expect_template)
306 expect_template["owner_uuid"] = stubs.fake_user_uuid
308 stubs.api.pipeline_templates().create.assert_called_with(
309 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
312 def test_inputs(self, stubs):
313 exited = arvados_cwl.main(
314 ["--create-template", "--no-wait",
315 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
316 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
317 self.assertEqual(exited, 0)
319 self.expect_template["owner_uuid"] = stubs.fake_user_uuid
321 expect_template = copy.deepcopy(self.expect_template)
322 expect_template["owner_uuid"] = stubs.fake_user_uuid
323 params = expect_template[
324 "components"]["inputs_test.cwl"]["script_parameters"]
325 params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
326 params["floatInput"]["value"] = 1.234
327 params["boolInput"]["value"] = True
329 stubs.api.pipeline_templates().create.assert_called_with(
330 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)