3 import arvados.collection
14 from .matcher import JsonDiffMatcher
18 @functools.wraps(func)
19 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
20 @mock.patch("arvados.collection.KeepClient")
21 @mock.patch("arvados.events.subscribe")
22 def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
27 stubs.KeepClient = KeepClient
28 stubs.keepdocker = keepdocker
30 def putstub(p, **kwargs):
31 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
32 stubs.KeepClient().put.side_effect = putstub
34 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
35 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
37 stubs.api = mock.MagicMock()
38 stubs.api.users().current().execute.return_value = {
39 "uuid": stubs.fake_user_uuid,
41 stubs.api.collections().list().execute.return_value = {"items": []}
42 stubs.api.collections().create().execute.side_effect = ({
43 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
44 "portable_data_hash": "99999999999999999999999999999991+99",
47 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
48 "portable_data_hash": "99999999999999999999999999999992+99",
49 "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
52 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
53 "portable_data_hash": "99999999999999999999999999999994+99",
57 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
58 "portable_data_hash": "99999999999999999999999999999995+99",
61 stubs.api.collections().get().execute.return_value = {
62 "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
64 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
65 stubs.api.jobs().create().execute.return_value = {
66 "uuid": stubs.expect_job_uuid,
70 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
71 stubs.api.container_requests().create().execute.return_value = {
72 "uuid": stubs.expect_container_request_uuid,
73 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
77 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
78 stubs.api.pipeline_templates().create().execute.return_value = {
79 "uuid": stubs.expect_pipeline_template_uuid,
81 stubs.expect_job_spec = {
82 'runtime_constraints': {
83 'docker_image': 'arvados/jobs'
85 'script_parameters': {
87 'basename': 'blorp.txt',
88 'location': '99999999999999999999999999999994+99/blorp.txt',
92 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
94 'repository': 'arvados',
95 'script_version': 'master',
96 'script': 'cwl-runner'
99 stubs.expect_container_spec = {
106 '/var/lib/cwl/workflow': {
107 'portable_data_hash': '99999999999999999999999999999991+99',
111 'path': '/var/spool/cwl/cwl.output.json',
114 '/var/lib/cwl/job/cwl.input.json': {
115 'portable_data_hash': '765fda0d9897729ff467a4609879c00a+60/cwl.input.json',
119 'state': 'Committed',
120 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
121 'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
122 'name': 'submit_wf.cwl',
123 'container_image': '99999999999999999999999999999993+99',
124 'output_path': '/var/spool/cwl',
125 'cwd': '/var/spool/cwl',
126 'runtime_constraints': {
133 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
134 stubs.api.workflows().create().execute.return_value = {
135 "uuid": stubs.expect_workflow_uuid,
138 return func(self, stubs, *args, **kwargs)
142 class TestSubmit(unittest.TestCase):
144 def test_submit(self, stubs):
145 capture_stdout = cStringIO.StringIO()
146 exited = arvados_cwl.main(
147 ["--submit", "--no-wait",
148 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
149 capture_stdout, sys.stderr, api_client=stubs.api)
150 self.assertEqual(exited, 0)
152 stubs.api.collections().create.assert_has_calls([
156 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
157 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
158 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
159 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
160 'name': 'submit_wf.cwl',
161 }, ensure_unique_name=True),
162 mock.call().execute(),
163 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
164 '0:0:blub.txt 0:0:submit_tool.cwl\n',
165 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
166 'name': 'New collection'
167 }, ensure_unique_name=True),
168 mock.call().execute(num_retries=4),
171 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
172 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
174 }, ensure_unique_name=True),
175 mock.call().execute()])
177 expect_job = copy.deepcopy(stubs.expect_job_spec)
178 expect_job["owner_uuid"] = stubs.fake_user_uuid
179 stubs.api.jobs().create.assert_called_with(
182 self.assertEqual(capture_stdout.getvalue(),
183 stubs.expect_job_uuid + '\n')
186 def test_submit_with_project_uuid(self, stubs):
187 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
189 exited = arvados_cwl.main(
190 ["--submit", "--no-wait",
191 "--project-uuid", project_uuid,
192 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
193 sys.stdout, sys.stderr, api_client=stubs.api)
194 self.assertEqual(exited, 0)
196 expect_body = copy.deepcopy(stubs.expect_job_spec)
197 expect_body["owner_uuid"] = project_uuid
198 stubs.api.jobs().create.assert_called_with(
203 def test_submit_container(self, stubs):
204 capture_stdout = cStringIO.StringIO()
205 exited = arvados_cwl.main(
206 ["--submit", "--no-wait", "--api=containers", "--debug",
207 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
208 capture_stdout, sys.stderr, api_client=stubs.api)
209 self.assertEqual(exited, 0)
211 stubs.api.collections().create.assert_has_calls([
215 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
216 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
217 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
218 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
219 'name': 'submit_wf.cwl',
220 }, ensure_unique_name=True),
221 mock.call().execute(),
222 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
223 '0:0:blub.txt 0:0:submit_tool.cwl\n',
224 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
225 'name': 'New collection',
226 }, ensure_unique_name=True),
227 mock.call().execute(num_retries=4),
230 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
231 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
233 }, ensure_unique_name=True),
234 mock.call().execute()])
236 expect_container = copy.deepcopy(stubs.expect_container_spec)
237 expect_container["owner_uuid"] = stubs.fake_user_uuid
238 stubs.api.container_requests().create.assert_called_with(
239 body=expect_container)
240 self.assertEqual(capture_stdout.getvalue(),
241 stubs.expect_container_request_uuid + '\n')
244 class TestCreateTemplate(unittest.TestCase):
246 def test_create(self, stubs):
247 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
249 capture_stdout = cStringIO.StringIO()
251 exited = arvados_cwl.main(
252 ["--create-template", "--debug",
253 "--project-uuid", project_uuid,
254 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
255 capture_stdout, sys.stderr, api_client=stubs.api)
256 self.assertEqual(exited, 0)
258 stubs.api.pipeline_instances().create.refute_called()
259 stubs.api.jobs().create.refute_called()
261 expect_component = copy.deepcopy(stubs.expect_job_spec)
262 expect_component['script_parameters']['x'] = {
266 'value': '99999999999999999999999999999994+99/blorp.txt',
270 "submit_wf.cwl": expect_component,
272 "name": "submit_wf.cwl",
273 "owner_uuid": project_uuid,
275 stubs.api.pipeline_templates().create.assert_called_with(
276 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
278 self.assertEqual(capture_stdout.getvalue(),
279 stubs.expect_pipeline_template_uuid + '\n')
282 class TestCreateWorkflow(unittest.TestCase):
284 def test_create(self, stubs):
285 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
287 capture_stdout = cStringIO.StringIO()
289 exited = arvados_cwl.main(
290 ["--create-workflow", "--debug",
291 "--project-uuid", project_uuid,
292 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
293 capture_stdout, sys.stderr, api_client=stubs.api)
294 self.assertEqual(exited, 0)
296 stubs.api.pipeline_templates().create.refute_called()
297 stubs.api.container_requests().create.refute_called()
299 with open("tests/wf/expect_packed.cwl") as f:
300 expect_workflow = f.read()
304 "owner_uuid": project_uuid,
305 "name": "submit_wf.cwl",
307 "workflow": expect_workflow
310 stubs.api.workflows().create.assert_called_with(
311 body=JsonDiffMatcher(body))
313 self.assertEqual(capture_stdout.getvalue(),
314 stubs.expect_workflow_uuid + '\n')
317 class TestTemplateInputs(unittest.TestCase):
321 'runtime_constraints': {
322 'docker_image': 'arvados/jobs',
324 'script_parameters': {
326 '99999999999999999999999999999991+99/'
327 'wf/inputs_test.cwl',
328 'optionalFloatInput': None,
333 'title': "It's a file; we expect to find some characters in it.",
334 'description': 'If there were anything further to say, it would be said here,\nor here.'
338 'dataclass': 'number',
340 'title': 'Floats like a duck',
344 'optionalFloatInput': {
345 'type': ['null', 'float'],
346 'dataclass': 'number',
351 'dataclass': 'boolean',
353 'title': 'True or false?',
356 'repository': 'arvados',
357 'script_version': 'master',
358 'script': 'cwl-runner',
361 "name": "inputs_test.cwl",
365 def test_inputs_empty(self, stubs):
366 exited = arvados_cwl.main(
367 ["--create-template", "--no-wait",
368 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
369 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
370 self.assertEqual(exited, 0)
372 expect_template = copy.deepcopy(self.expect_template)
373 expect_template["owner_uuid"] = stubs.fake_user_uuid
375 stubs.api.pipeline_templates().create.assert_called_with(
376 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
379 def test_inputs(self, stubs):
380 exited = arvados_cwl.main(
381 ["--create-template", "--no-wait",
382 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
383 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
384 self.assertEqual(exited, 0)
386 self.expect_template["owner_uuid"] = stubs.fake_user_uuid
388 expect_template = copy.deepcopy(self.expect_template)
389 expect_template["owner_uuid"] = stubs.fake_user_uuid
390 params = expect_template[
391 "components"]["inputs_test.cwl"]["script_parameters"]
392 params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
393 params["floatInput"]["value"] = 1.234
394 params["boolInput"]["value"] = True
396 stubs.api.pipeline_templates().create.assert_called_with(
397 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)