12 import arvados.collection
14 import arvados_cwl.runner
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
23 @functools.wraps(func)
24 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
25 @mock.patch("arvados.collection.KeepClient")
26 @mock.patch("arvados.keep.KeepClient")
27 @mock.patch("arvados.events.subscribe")
28 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
33 stubs.keepdocker = keepdocker
36 def putstub(p, **kwargs):
37 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
38 keep_client1().put.side_effect = putstub
39 keep_client1.put.side_effect = putstub
40 keep_client2().put.side_effect = putstub
41 keep_client2.put.side_effect = putstub
43 stubs.keep_client = keep_client2
44 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
45 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
47 stubs.api = mock.MagicMock()
48 stubs.api._rootDesc = get_rootDesc()
50 stubs.api.users().current().execute.return_value = {
51 "uuid": stubs.fake_user_uuid,
53 stubs.api.collections().list().execute.return_value = {"items": []}
54 stubs.api.collections().create().execute.side_effect = ({
55 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
56 "portable_data_hash": "99999999999999999999999999999991+99",
59 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
60 "portable_data_hash": "99999999999999999999999999999992+99",
61 "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
64 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
65 "portable_data_hash": "99999999999999999999999999999994+99",
69 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
70 "portable_data_hash": "99999999999999999999999999999995+99",
74 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
75 "portable_data_hash": "99999999999999999999999999999996+99",
79 stubs.api.collections().get().execute.return_value = {
80 "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
82 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
83 stubs.api.jobs().create().execute.return_value = {
84 "uuid": stubs.expect_job_uuid,
88 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
89 stubs.api.container_requests().create().execute.return_value = {
90 "uuid": stubs.expect_container_request_uuid,
91 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
95 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
96 stubs.api.pipeline_templates().create().execute.return_value = {
97 "uuid": stubs.expect_pipeline_template_uuid,
99 stubs.expect_job_spec = {
100 'runtime_constraints': {
101 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
103 'script_parameters': {
105 'basename': 'blorp.txt',
106 'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
110 'basename': '99999999999999999999999999999998+99',
111 'location': 'keep:99999999999999999999999999999998+99',
115 'basename': 'anonymous',
117 "basename": "renamed.txt",
119 "location": "keep:99999999999999999999999999999998+99/file1.txt"
124 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
126 'repository': 'arvados',
127 'script_version': arvados_cwl.__version__,
128 'script': 'cwl-runner'
130 stubs.pipeline_component = stubs.expect_job_spec.copy()
131 stubs.expect_pipeline_instance = {
132 'name': 'submit_wf.cwl',
133 'state': 'RunningOnServer',
136 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
137 'script_parameters': {
138 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
139 'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
140 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
142 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
144 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
145 'arv:enable_reuse': True
147 'repository': 'arvados',
148 'script_version': arvados_cwl.__version__,
149 'script': 'cwl-runner',
150 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
154 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
155 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
156 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
157 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
158 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
159 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
162 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
163 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
165 stubs.expect_container_spec = {
172 '/var/lib/cwl/workflow': {
173 'portable_data_hash': '99999999999999999999999999999991+99',
177 'path': '/var/spool/cwl/cwl.output.json',
180 '/var/lib/cwl/job/cwl.input.json': {
181 'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
185 'state': 'Committed',
186 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
187 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
188 'name': 'submit_wf.cwl',
189 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
190 'output_path': '/var/spool/cwl',
191 'cwd': '/var/spool/cwl',
192 'runtime_constraints': {
199 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
200 stubs.api.workflows().create().execute.return_value = {
201 "uuid": stubs.expect_workflow_uuid,
204 return func(self, stubs, *args, **kwargs)
208 class TestSubmit(unittest.TestCase):
209 @mock.patch("time.sleep")
211 def test_submit(self, stubs, tm):
212 capture_stdout = cStringIO.StringIO()
213 exited = arvados_cwl.main(
214 ["--submit", "--no-wait", "--debug",
215 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
216 capture_stdout, sys.stderr, api_client=stubs.api)
217 self.assertEqual(exited, 0)
219 stubs.api.collections().create.assert_has_calls([
223 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
224 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
225 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
226 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
227 'name': 'submit_wf.cwl',
228 }, ensure_unique_name=True),
229 mock.call().execute(),
230 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
231 '0:0:blub.txt 0:0:submit_tool.cwl\n',
232 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
233 'replication_desired': None,
234 'name': 'New collection'
235 }, ensure_unique_name=True),
236 mock.call().execute(num_retries=4),
239 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
240 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
242 }, ensure_unique_name=True),
243 mock.call().execute()])
245 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
246 expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
247 stubs.api.pipeline_instances().create.assert_called_with(
248 body=expect_pipeline)
249 self.assertEqual(capture_stdout.getvalue(),
250 stubs.expect_pipeline_uuid + '\n')
253 @mock.patch("time.sleep")
255 def test_submit_no_reuse(self, stubs, tm):
256 capture_stdout = cStringIO.StringIO()
257 exited = arvados_cwl.main(
258 ["--submit", "--no-wait", "--debug", "--disable-reuse",
259 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
260 capture_stdout, sys.stderr, api_client=stubs.api)
261 self.assertEqual(exited, 0)
263 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
265 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
266 expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
267 stubs.api.pipeline_instances().create.assert_called_with(
268 body=expect_pipeline)
269 self.assertEqual(capture_stdout.getvalue(),
270 stubs.expect_pipeline_uuid + '\n')
272 @mock.patch("time.sleep")
274 def test_submit_with_project_uuid(self, stubs, tm):
275 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
277 exited = arvados_cwl.main(
278 ["--submit", "--no-wait",
279 "--project-uuid", project_uuid,
280 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
281 sys.stdout, sys.stderr, api_client=stubs.api)
282 self.assertEqual(exited, 0)
284 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
285 expect_pipeline["owner_uuid"] = project_uuid
286 stubs.api.pipeline_instances().create.assert_called_with(
287 body=expect_pipeline)
290 def test_submit_container(self, stubs):
291 capture_stdout = cStringIO.StringIO()
293 exited = arvados_cwl.main(
294 ["--submit", "--no-wait", "--api=containers", "--debug",
295 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
296 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
297 self.assertEqual(exited, 0)
299 logging.exception("")
301 stubs.api.collections().create.assert_has_calls([
305 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
306 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
307 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
308 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
309 'name': 'submit_wf.cwl',
310 }, ensure_unique_name=True),
311 mock.call().execute(),
312 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
313 '0:0:blub.txt 0:0:submit_tool.cwl\n',
314 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
315 'name': 'New collection',
316 'replication_desired': None,
317 }, ensure_unique_name=True),
318 mock.call().execute(num_retries=4),
321 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
322 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
324 }, ensure_unique_name=True),
325 mock.call().execute()])
327 expect_container = copy.deepcopy(stubs.expect_container_spec)
328 expect_container["owner_uuid"] = stubs.fake_user_uuid
329 stubs.api.container_requests().create.assert_called_with(
330 body=expect_container)
331 self.assertEqual(capture_stdout.getvalue(),
332 stubs.expect_container_request_uuid + '\n')
335 def test_submit_container_no_reuse(self, stubs):
336 capture_stdout = cStringIO.StringIO()
338 exited = arvados_cwl.main(
339 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
340 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
341 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
342 self.assertEqual(exited, 0)
344 logging.exception("")
346 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
348 expect_container = copy.deepcopy(stubs.expect_container_spec)
349 expect_container["owner_uuid"] = stubs.fake_user_uuid
350 stubs.api.container_requests().create.assert_called_with(
351 body=expect_container)
352 self.assertEqual(capture_stdout.getvalue(),
353 stubs.expect_container_request_uuid + '\n')
355 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
356 @mock.patch("cwltool.docker.get_image")
357 @mock.patch("arvados.api")
358 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
359 arvrunner = mock.MagicMock()
360 arvrunner.project_uuid = ""
361 api.return_value = mock.MagicMock()
362 arvrunner.api = api.return_value
363 arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
364 {"items": [], "items_available": 0, "offset": 0},
365 {"items": [], "items_available": 0, "offset": 0},
366 {"items": [{"created_at": "",
368 "link_class": "docker_image_hash",
371 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
372 {"items": [], "items_available": 0, "offset": 0},
373 {"items": [{"created_at": "",
375 "link_class": "docker_image_repo+tag",
376 "name": "arvados/jobs:"+arvados_cwl.__version__,
378 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
379 {"items": [{"created_at": "",
381 "link_class": "docker_image_hash",
384 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
386 find_one_image_hash.return_value = "123456"
388 arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
389 {"items": [{"uuid": "",
393 }], "items_available": 1, "offset": 0},
394 {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
395 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
396 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
398 class TestCreateTemplate(unittest.TestCase):
400 def test_create(self, stubs):
401 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
403 capture_stdout = cStringIO.StringIO()
405 exited = arvados_cwl.main(
406 ["--create-template", "--debug",
407 "--project-uuid", project_uuid,
408 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
409 capture_stdout, sys.stderr, api_client=stubs.api)
410 self.assertEqual(exited, 0)
412 stubs.api.pipeline_instances().create.refute_called()
413 stubs.api.jobs().create.refute_called()
415 expect_component = copy.deepcopy(stubs.expect_job_spec)
416 expect_component['script_parameters']['x'] = {
420 'value': '99999999999999999999999999999994+99/blorp.txt',
422 expect_component['script_parameters']['y'] = {
423 'dataclass': 'Collection',
426 'value': '99999999999999999999999999999998+99',
428 expect_component['script_parameters']['z'] = {
429 'dataclass': 'Collection',
435 "submit_wf.cwl": expect_component,
437 "name": "submit_wf.cwl",
438 "owner_uuid": project_uuid,
440 stubs.api.pipeline_templates().create.assert_called_with(
441 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
443 self.assertEqual(capture_stdout.getvalue(),
444 stubs.expect_pipeline_template_uuid + '\n')
447 class TestCreateWorkflow(unittest.TestCase):
449 def test_create(self, stubs):
450 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
452 capture_stdout = cStringIO.StringIO()
454 exited = arvados_cwl.main(
455 ["--create-workflow", "--debug",
456 "--project-uuid", project_uuid,
457 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
458 capture_stdout, sys.stderr, api_client=stubs.api)
459 self.assertEqual(exited, 0)
461 stubs.api.pipeline_templates().create.refute_called()
462 stubs.api.container_requests().create.refute_called()
464 with open("tests/wf/expect_packed.cwl") as f:
465 expect_workflow = f.read()
469 "owner_uuid": project_uuid,
470 "name": "submit_wf.cwl",
472 "definition": expect_workflow
475 stubs.api.workflows().create.assert_called_with(
476 body=JsonDiffMatcher(body))
478 self.assertEqual(capture_stdout.getvalue(),
479 stubs.expect_workflow_uuid + '\n')
482 class TestTemplateInputs(unittest.TestCase):
486 'runtime_constraints': {
487 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
489 'script_parameters': {
491 '99999999999999999999999999999991+99/'
492 'wf/inputs_test.cwl',
493 'optionalFloatInput': None,
498 'title': "It's a file; we expect to find some characters in it.",
499 'description': 'If there were anything further to say, it would be said here,\nor here.'
503 'dataclass': 'number',
505 'title': 'Floats like a duck',
509 'optionalFloatInput': {
510 'type': ['null', 'float'],
511 'dataclass': 'number',
516 'dataclass': 'boolean',
518 'title': 'True or false?',
521 'repository': 'arvados',
522 'script_version': arvados_cwl.__version__,
523 'script': 'cwl-runner',
526 "name": "inputs_test.cwl",
530 def test_inputs_empty(self, stubs):
531 exited = arvados_cwl.main(
532 ["--create-template", "--no-wait",
533 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
534 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
535 self.assertEqual(exited, 0)
537 expect_template = copy.deepcopy(self.expect_template)
538 expect_template["owner_uuid"] = stubs.fake_user_uuid
540 stubs.api.pipeline_templates().create.assert_called_with(
541 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
544 def test_inputs(self, stubs):
545 exited = arvados_cwl.main(
546 ["--create-template", "--no-wait",
547 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
548 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
549 self.assertEqual(exited, 0)
551 self.expect_template["owner_uuid"] = stubs.fake_user_uuid
553 expect_template = copy.deepcopy(self.expect_template)
554 expect_template["owner_uuid"] = stubs.fake_user_uuid
555 params = expect_template[
556 "components"]["inputs_test.cwl"]["script_parameters"]
557 params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
558 params["floatInput"]["value"] = 1.234
559 params["boolInput"]["value"] = True
561 stubs.api.pipeline_templates().create.assert_called_with(
562 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)