12 import arvados.collection
14 import arvados_cwl.runner
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
23 @functools.wraps(func)
24 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
25 @mock.patch("arvados.collection.KeepClient")
26 @mock.patch("arvados.keep.KeepClient")
27 @mock.patch("arvados.events.subscribe")
28 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
33 stubs.keepdocker = keepdocker
36 def putstub(p, **kwargs):
37 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
38 keep_client1().put.side_effect = putstub
39 keep_client1.put.side_effect = putstub
40 keep_client2().put.side_effect = putstub
41 keep_client2.put.side_effect = putstub
43 stubs.keep_client = keep_client2
44 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
45 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
47 stubs.api = mock.MagicMock()
48 stubs.api._rootDesc = get_rootDesc()
50 stubs.api.users().current().execute.return_value = {
51 "uuid": stubs.fake_user_uuid,
53 stubs.api.collections().list().execute.return_value = {"items": []}
54 stubs.api.collections().create().execute.side_effect = ({
55 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
56 "portable_data_hash": "99999999999999999999999999999991+99",
59 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
60 "portable_data_hash": "99999999999999999999999999999992+99",
61 "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
64 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
65 "portable_data_hash": "99999999999999999999999999999994+99",
69 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
70 "portable_data_hash": "99999999999999999999999999999995+99",
74 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
75 "portable_data_hash": "99999999999999999999999999999996+99",
79 stubs.api.collections().get().execute.return_value = {
80 "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
82 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
83 stubs.api.jobs().create().execute.return_value = {
84 "uuid": stubs.expect_job_uuid,
88 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
89 stubs.api.container_requests().create().execute.return_value = {
90 "uuid": stubs.expect_container_request_uuid,
91 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
95 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
96 stubs.api.pipeline_templates().create().execute.return_value = {
97 "uuid": stubs.expect_pipeline_template_uuid,
99 stubs.expect_job_spec = {
100 'runtime_constraints': {
101 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
102 'min_ram_mb_per_node': 1024
104 'script_parameters': {
106 'basename': 'blorp.txt',
107 'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
111 'basename': '99999999999999999999999999999998+99',
112 'location': 'keep:99999999999999999999999999999998+99',
116 'basename': 'anonymous',
118 "basename": "renamed.txt",
120 "location": "keep:99999999999999999999999999999998+99/file1.txt"
125 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
127 'repository': 'arvados',
128 'script_version': 'master',
129 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
130 'script': 'cwl-runner'
132 stubs.pipeline_component = stubs.expect_job_spec.copy()
133 stubs.expect_pipeline_instance = {
134 'name': 'submit_wf.cwl',
135 'state': 'RunningOnServer',
139 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
140 'script_parameters': {
141 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
142 'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
143 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
145 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
147 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
148 'arv:enable_reuse': True,
149 'arv:on_error': 'continue'
151 'repository': 'arvados',
152 'script_version': 'master',
153 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
154 'script': 'cwl-runner',
155 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
159 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
160 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
161 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
162 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
163 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
164 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
167 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
168 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
170 stubs.expect_container_spec = {
177 '/var/lib/cwl/workflow': {
178 'portable_data_hash': '99999999999999999999999999999991+99',
182 'path': '/var/spool/cwl/cwl.output.json',
185 '/var/lib/cwl/cwl.input.json': {
188 'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
189 'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999994+99/blorp.txt'},
190 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
191 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
197 'state': 'Committed',
199 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
200 '--enable-reuse', '--on-error=continue',
201 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json'],
202 'name': 'submit_wf.cwl',
203 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
204 'output_path': '/var/spool/cwl',
205 'cwd': '/var/spool/cwl',
206 'runtime_constraints': {
209 'ram': 1024*1024*1024
214 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
215 stubs.api.workflows().create().execute.return_value = {
216 "uuid": stubs.expect_workflow_uuid,
218 def update_mock(**kwargs):
219 stubs.updated_uuid = kwargs.get('uuid')
221 stubs.api.workflows().update.side_effect = update_mock
222 stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
223 "uuid": stubs.updated_uuid,
226 return func(self, stubs, *args, **kwargs)
230 class TestSubmit(unittest.TestCase):
231 @mock.patch("time.sleep")
233 def test_submit(self, stubs, tm):
234 capture_stdout = cStringIO.StringIO()
235 exited = arvados_cwl.main(
236 ["--submit", "--no-wait", "--debug",
237 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
238 capture_stdout, sys.stderr, api_client=stubs.api)
239 self.assertEqual(exited, 0)
241 stubs.api.collections().create.assert_has_calls([
245 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
246 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
247 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
249 'name': 'submit_wf.cwl',
250 }, ensure_unique_name=True),
251 mock.call().execute(),
252 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
253 '0:0:blub.txt 0:0:submit_tool.cwl\n',
254 'replication_desired': None,
255 'name': 'New collection'
256 }, ensure_unique_name=True),
257 mock.call().execute(num_retries=4),
260 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
263 }, ensure_unique_name=True),
264 mock.call().execute()])
266 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
267 stubs.api.pipeline_instances().create.assert_called_with(
268 body=expect_pipeline)
269 self.assertEqual(capture_stdout.getvalue(),
270 stubs.expect_pipeline_uuid + '\n')
273 @mock.patch("time.sleep")
275 def test_submit_no_reuse(self, stubs, tm):
276 capture_stdout = cStringIO.StringIO()
277 exited = arvados_cwl.main(
278 ["--submit", "--no-wait", "--debug", "--disable-reuse",
279 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
280 capture_stdout, sys.stderr, api_client=stubs.api)
281 self.assertEqual(exited, 0)
283 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
285 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
286 stubs.api.pipeline_instances().create.assert_called_with(
287 body=JsonDiffMatcher(expect_pipeline))
288 self.assertEqual(capture_stdout.getvalue(),
289 stubs.expect_pipeline_uuid + '\n')
291 @mock.patch("time.sleep")
293 def test_submit_on_error(self, stubs, tm):
294 capture_stdout = cStringIO.StringIO()
295 exited = arvados_cwl.main(
296 ["--submit", "--no-wait", "--debug", "--on-error=stop",
297 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
298 capture_stdout, sys.stderr, api_client=stubs.api)
299 self.assertEqual(exited, 0)
301 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
303 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
304 stubs.api.pipeline_instances().create.assert_called_with(
305 body=JsonDiffMatcher(expect_pipeline))
306 self.assertEqual(capture_stdout.getvalue(),
307 stubs.expect_pipeline_uuid + '\n')
310 @mock.patch("time.sleep")
312 def test_submit_runner_ram(self, stubs, tm):
313 capture_stdout = cStringIO.StringIO()
314 exited = arvados_cwl.main(
315 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
316 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
317 capture_stdout, sys.stderr, api_client=stubs.api)
318 self.assertEqual(exited, 0)
320 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
322 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
323 stubs.api.pipeline_instances().create.assert_called_with(
324 body=JsonDiffMatcher(expect_pipeline))
325 self.assertEqual(capture_stdout.getvalue(),
326 stubs.expect_pipeline_uuid + '\n')
329 @mock.patch("time.sleep")
331 def test_submit_invalid_runner_ram(self, stubs, tm):
332 capture_stdout = cStringIO.StringIO()
333 exited = arvados_cwl.main(
334 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
335 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
336 capture_stdout, sys.stderr, api_client=stubs.api)
337 self.assertEqual(exited, 1)
339 @mock.patch("time.sleep")
341 def test_submit_output_name(self, stubs, tm):
342 output_name = "test_output_name"
344 capture_stdout = cStringIO.StringIO()
345 exited = arvados_cwl.main(
346 ["--submit", "--no-wait", "--debug", "--output-name", output_name,
347 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
348 capture_stdout, sys.stderr, api_client=stubs.api)
349 self.assertEqual(exited, 0)
351 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
353 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
354 stubs.api.pipeline_instances().create.assert_called_with(
355 body=JsonDiffMatcher(expect_pipeline))
356 self.assertEqual(capture_stdout.getvalue(),
357 stubs.expect_pipeline_uuid + '\n')
360 @mock.patch("time.sleep")
362 def test_submit_pipeline_name(self, stubs, tm):
363 capture_stdout = cStringIO.StringIO()
364 exited = arvados_cwl.main(
365 ["--submit", "--no-wait", "--debug", "--name=hello job 123",
366 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
367 capture_stdout, sys.stderr, api_client=stubs.api)
368 self.assertEqual(exited, 0)
370 stubs.expect_pipeline_instance["name"] = "hello job 123"
372 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
373 stubs.api.pipeline_instances().create.assert_called_with(
374 body=expect_pipeline)
375 self.assertEqual(capture_stdout.getvalue(),
376 stubs.expect_pipeline_uuid + '\n')
378 @mock.patch("time.sleep")
380 def test_submit_output_tags(self, stubs, tm):
381 output_tags = "tag0,tag1,tag2"
383 capture_stdout = cStringIO.StringIO()
384 exited = arvados_cwl.main(
385 ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
386 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
387 capture_stdout, sys.stderr, api_client=stubs.api)
388 self.assertEqual(exited, 0)
390 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
392 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
393 stubs.api.pipeline_instances().create.assert_called_with(
394 body=JsonDiffMatcher(expect_pipeline))
395 self.assertEqual(capture_stdout.getvalue(),
396 stubs.expect_pipeline_uuid + '\n')
398 @mock.patch("time.sleep")
400 def test_submit_with_project_uuid(self, stubs, tm):
401 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
403 exited = arvados_cwl.main(
404 ["--submit", "--no-wait",
405 "--project-uuid", project_uuid,
406 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
407 sys.stdout, sys.stderr, api_client=stubs.api)
408 self.assertEqual(exited, 0)
410 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
411 expect_pipeline["owner_uuid"] = project_uuid
412 stubs.api.pipeline_instances().create.assert_called_with(
413 body=JsonDiffMatcher(expect_pipeline))
416 def test_submit_container(self, stubs):
417 capture_stdout = cStringIO.StringIO()
419 exited = arvados_cwl.main(
420 ["--submit", "--no-wait", "--api=containers", "--debug",
421 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
422 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
423 self.assertEqual(exited, 0)
425 logging.exception("")
427 stubs.api.collections().create.assert_has_calls([
431 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
432 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
433 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
435 'name': 'submit_wf.cwl',
436 }, ensure_unique_name=True),
437 mock.call().execute(),
438 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
439 '0:0:blub.txt 0:0:submit_tool.cwl\n',
440 'name': 'New collection',
441 'replication_desired': None,
442 }, ensure_unique_name=True),
443 mock.call().execute(num_retries=4),
446 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
449 }, ensure_unique_name=True),
450 mock.call().execute()])
452 expect_container = copy.deepcopy(stubs.expect_container_spec)
453 stubs.api.container_requests().create.assert_called_with(
454 body=JsonDiffMatcher(expect_container))
455 self.assertEqual(capture_stdout.getvalue(),
456 stubs.expect_container_request_uuid + '\n')
459 def test_submit_container_no_reuse(self, stubs):
460 capture_stdout = cStringIO.StringIO()
462 exited = arvados_cwl.main(
463 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
464 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
465 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
466 self.assertEqual(exited, 0)
468 logging.exception("")
470 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
471 '--disable-reuse', '--on-error=continue',
472 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
474 expect_container = copy.deepcopy(stubs.expect_container_spec)
475 stubs.api.container_requests().create.assert_called_with(
476 body=JsonDiffMatcher(expect_container))
477 self.assertEqual(capture_stdout.getvalue(),
478 stubs.expect_container_request_uuid + '\n')
482 def test_submit_container_on_error(self, stubs):
483 capture_stdout = cStringIO.StringIO()
485 exited = arvados_cwl.main(
486 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
487 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
488 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
489 self.assertEqual(exited, 0)
491 logging.exception("")
493 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
494 '--enable-reuse', '--on-error=stop',
495 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
497 expect_container = copy.deepcopy(stubs.expect_container_spec)
498 stubs.api.container_requests().create.assert_called_with(
499 body=JsonDiffMatcher(expect_container))
500 self.assertEqual(capture_stdout.getvalue(),
501 stubs.expect_container_request_uuid + '\n')
504 def test_submit_container_output_name(self, stubs):
505 output_name = "test_output_name"
507 capture_stdout = cStringIO.StringIO()
509 exited = arvados_cwl.main(
510 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
511 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
512 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
513 self.assertEqual(exited, 0)
515 logging.exception("")
517 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
518 "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
519 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
521 expect_container = copy.deepcopy(stubs.expect_container_spec)
522 stubs.api.container_requests().create.assert_called_with(
523 body=JsonDiffMatcher(expect_container))
524 self.assertEqual(capture_stdout.getvalue(),
525 stubs.expect_container_request_uuid + '\n')
528 def test_submit_container_output_tags(self, stubs):
529 output_tags = "tag0,tag1,tag2"
531 capture_stdout = cStringIO.StringIO()
533 exited = arvados_cwl.main(
534 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
535 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
536 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
537 self.assertEqual(exited, 0)
539 logging.exception("")
541 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
542 "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
543 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
545 expect_container = copy.deepcopy(stubs.expect_container_spec)
546 stubs.api.container_requests().create.assert_called_with(
547 body=JsonDiffMatcher(expect_container))
548 self.assertEqual(capture_stdout.getvalue(),
549 stubs.expect_container_request_uuid + '\n')
552 def test_submit_container_runner_ram(self, stubs):
553 capture_stdout = cStringIO.StringIO()
555 exited = arvados_cwl.main(
556 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
557 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
558 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
559 self.assertEqual(exited, 0)
561 logging.exception("")
563 stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
565 expect_container = copy.deepcopy(stubs.expect_container_spec)
566 stubs.api.container_requests().create.assert_called_with(
567 body=JsonDiffMatcher(expect_container))
568 self.assertEqual(capture_stdout.getvalue(),
569 stubs.expect_container_request_uuid + '\n')
571 @mock.patch("arvados.collection.CollectionReader")
572 @mock.patch("time.sleep")
574 def test_submit_file_keepref(self, stubs, tm, collectionReader):
575 capture_stdout = cStringIO.StringIO()
576 exited = arvados_cwl.main(
577 ["--submit", "--no-wait", "--api=containers", "--debug",
578 "tests/wf/submit_keepref_wf.cwl"],
579 capture_stdout, sys.stderr, api_client=stubs.api)
580 self.assertEqual(exited, 0)
583 @mock.patch("arvados.collection.CollectionReader")
584 @mock.patch("time.sleep")
586 def test_submit_keepref(self, stubs, tm, reader):
587 capture_stdout = cStringIO.StringIO()
589 with open("tests/wf/expect_arvworkflow.cwl") as f:
590 reader().open().__enter__().read.return_value = f.read()
592 exited = arvados_cwl.main(
593 ["--submit", "--no-wait", "--api=containers", "--debug",
594 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
595 capture_stdout, sys.stderr, api_client=stubs.api)
596 self.assertEqual(exited, 0)
606 'path': '/var/spool/cwl/cwl.output.json',
609 '/var/lib/cwl/workflow': {
610 'portable_data_hash': '99999999999999999999999999999994+99',
613 '/var/lib/cwl/cwl.input.json': {
619 }, 'state': 'Committed',
621 'output_path': '/var/spool/cwl',
622 'name': 'expect_arvworkflow.cwl#main',
623 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
624 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
625 '--enable-reuse', '--on-error=continue',
626 '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
627 'cwd': '/var/spool/cwl',
628 'runtime_constraints': {
636 stubs.api.container_requests().create.assert_called_with(
637 body=JsonDiffMatcher(expect_container))
638 self.assertEqual(capture_stdout.getvalue(),
639 stubs.expect_container_request_uuid + '\n')
641 @mock.patch("time.sleep")
643 def test_submit_arvworkflow(self, stubs, tm):
644 capture_stdout = cStringIO.StringIO()
646 with open("tests/wf/expect_arvworkflow.cwl") as f:
647 stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
649 exited = arvados_cwl.main(
650 ["--submit", "--no-wait", "--api=containers", "--debug",
651 "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
652 capture_stdout, sys.stderr, api_client=stubs.api)
653 self.assertEqual(exited, 0)
663 'path': '/var/spool/cwl/cwl.output.json',
666 '/var/lib/cwl/workflow.json': {
669 'cwlVersion': 'v1.0',
674 'inputBinding': {'position': 1},
676 'id': '#submit_tool.cwl/x'}
679 {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
681 'id': '#submit_tool.cwl',
683 'baseCommand': 'cat',
684 'class': 'CommandLineTool'
688 {'type': 'string', 'id': '#main/x'}
691 {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
692 'run': '#submit_tool.cwl',
702 '/var/lib/cwl/cwl.input.json': {
708 }, 'state': 'Committed',
710 'output_path': '/var/spool/cwl',
711 'name': 'a test workflow',
712 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
713 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
714 '--enable-reuse', '--on-error=continue',
715 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
716 'cwd': '/var/spool/cwl',
717 'runtime_constraints': {
723 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
727 stubs.api.container_requests().create.assert_called_with(
728 body=JsonDiffMatcher(expect_container))
729 self.assertEqual(capture_stdout.getvalue(),
730 stubs.expect_container_request_uuid + '\n')
734 def test_submit_container_name(self, stubs):
735 capture_stdout = cStringIO.StringIO()
737 exited = arvados_cwl.main(
738 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
739 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
740 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
741 self.assertEqual(exited, 0)
743 logging.exception("")
745 stubs.expect_container_spec["name"] = "hello container 123"
747 expect_container = copy.deepcopy(stubs.expect_container_spec)
748 stubs.api.container_requests().create.assert_called_with(
749 body=expect_container)
750 self.assertEqual(capture_stdout.getvalue(),
751 stubs.expect_container_request_uuid + '\n')
754 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
755 @mock.patch("cwltool.docker.get_image")
756 @mock.patch("arvados.api")
757 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
758 arvrunner = mock.MagicMock()
759 arvrunner.project_uuid = ""
760 api.return_value = mock.MagicMock()
761 arvrunner.api = api.return_value
762 arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
763 {"items": [], "items_available": 0, "offset": 0},
764 {"items": [], "items_available": 0, "offset": 0},
765 {"items": [{"created_at": "",
767 "link_class": "docker_image_hash",
770 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
771 {"items": [], "items_available": 0, "offset": 0},
772 {"items": [{"created_at": "",
774 "link_class": "docker_image_repo+tag",
775 "name": "arvados/jobs:"+arvados_cwl.__version__,
777 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
778 {"items": [{"created_at": "",
780 "link_class": "docker_image_hash",
783 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
785 find_one_image_hash.return_value = "123456"
787 arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
788 {"items": [{"uuid": "",
792 }], "items_available": 1, "offset": 0},
793 {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
794 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
795 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
797 class TestCreateTemplate(unittest.TestCase):
798 existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
800 def _adjust_script_params(self, expect_component):
801 expect_component['script_parameters']['x'] = {
805 'value': '99999999999999999999999999999994+99/blorp.txt',
807 expect_component['script_parameters']['y'] = {
808 'dataclass': 'Collection',
811 'value': '99999999999999999999999999999998+99',
813 expect_component['script_parameters']['z'] = {
814 'dataclass': 'Collection',
820 def test_create(self, stubs):
821 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
823 capture_stdout = cStringIO.StringIO()
825 exited = arvados_cwl.main(
826 ["--create-workflow", "--debug",
828 "--project-uuid", project_uuid,
829 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
830 capture_stdout, sys.stderr, api_client=stubs.api)
831 self.assertEqual(exited, 0)
833 stubs.api.pipeline_instances().create.refute_called()
834 stubs.api.jobs().create.refute_called()
836 expect_component = copy.deepcopy(stubs.expect_job_spec)
837 self._adjust_script_params(expect_component)
840 "submit_wf.cwl": expect_component,
842 "name": "submit_wf.cwl",
843 "owner_uuid": project_uuid,
845 stubs.api.pipeline_templates().create.assert_called_with(
846 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
848 self.assertEqual(capture_stdout.getvalue(),
849 stubs.expect_pipeline_template_uuid + '\n')
853 def test_create_name(self, stubs):
854 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
856 capture_stdout = cStringIO.StringIO()
858 exited = arvados_cwl.main(
859 ["--create-workflow", "--debug",
860 "--project-uuid", project_uuid,
862 "--name", "testing 123",
863 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
864 capture_stdout, sys.stderr, api_client=stubs.api)
865 self.assertEqual(exited, 0)
867 stubs.api.pipeline_instances().create.refute_called()
868 stubs.api.jobs().create.refute_called()
870 expect_component = copy.deepcopy(stubs.expect_job_spec)
871 self._adjust_script_params(expect_component)
874 "testing 123": expect_component,
876 "name": "testing 123",
877 "owner_uuid": project_uuid,
879 stubs.api.pipeline_templates().create.assert_called_with(
880 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
882 self.assertEqual(capture_stdout.getvalue(),
883 stubs.expect_pipeline_template_uuid + '\n')
887 def test_update_name(self, stubs):
888 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
890 capture_stdout = cStringIO.StringIO()
892 exited = arvados_cwl.main(
893 ["--update-workflow", self.existing_template_uuid,
895 "--project-uuid", project_uuid,
897 "--name", "testing 123",
898 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
899 capture_stdout, sys.stderr, api_client=stubs.api)
900 self.assertEqual(exited, 0)
902 stubs.api.pipeline_instances().create.refute_called()
903 stubs.api.jobs().create.refute_called()
905 expect_component = copy.deepcopy(stubs.expect_job_spec)
906 self._adjust_script_params(expect_component)
909 "testing 123": expect_component,
911 "name": "testing 123",
912 "owner_uuid": project_uuid,
914 stubs.api.pipeline_templates().create.refute_called()
915 stubs.api.pipeline_templates().update.assert_called_with(
916 body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
918 self.assertEqual(capture_stdout.getvalue(),
919 self.existing_template_uuid + '\n')
922 class TestCreateWorkflow(unittest.TestCase):
923 existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
924 expect_workflow = open("tests/wf/expect_packed.cwl").read()
927 def test_create(self, stubs):
928 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
930 capture_stdout = cStringIO.StringIO()
932 exited = arvados_cwl.main(
933 ["--create-workflow", "--debug",
935 "--project-uuid", project_uuid,
936 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
937 capture_stdout, sys.stderr, api_client=stubs.api)
938 self.assertEqual(exited, 0)
940 stubs.api.pipeline_templates().create.refute_called()
941 stubs.api.container_requests().create.refute_called()
945 "owner_uuid": project_uuid,
946 "name": "submit_wf.cwl",
948 "definition": self.expect_workflow,
951 stubs.api.workflows().create.assert_called_with(
952 body=JsonDiffMatcher(body))
954 self.assertEqual(capture_stdout.getvalue(),
955 stubs.expect_workflow_uuid + '\n')
959 def test_create_name(self, stubs):
960 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
962 capture_stdout = cStringIO.StringIO()
964 exited = arvados_cwl.main(
965 ["--create-workflow", "--debug",
967 "--project-uuid", project_uuid,
968 "--name", "testing 123",
969 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
970 capture_stdout, sys.stderr, api_client=stubs.api)
971 self.assertEqual(exited, 0)
973 stubs.api.pipeline_templates().create.refute_called()
974 stubs.api.container_requests().create.refute_called()
978 "owner_uuid": project_uuid,
979 "name": "testing 123",
981 "definition": self.expect_workflow,
984 stubs.api.workflows().create.assert_called_with(
985 body=JsonDiffMatcher(body))
987 self.assertEqual(capture_stdout.getvalue(),
988 stubs.expect_workflow_uuid + '\n')
991 def test_incompatible_api(self, stubs):
992 capture_stderr = cStringIO.StringIO()
993 logging.getLogger('arvados.cwl-runner').addHandler(
994 logging.StreamHandler(capture_stderr))
996 exited = arvados_cwl.main(
997 ["--update-workflow", self.existing_workflow_uuid,
1000 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1001 sys.stderr, sys.stderr, api_client=stubs.api)
1002 self.assertEqual(exited, 1)
1003 self.assertRegexpMatches(
1004 capture_stderr.getvalue(),
1005 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1008 def test_update(self, stubs):
1009 capture_stdout = cStringIO.StringIO()
1011 exited = arvados_cwl.main(
1012 ["--update-workflow", self.existing_workflow_uuid,
1014 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1015 capture_stdout, sys.stderr, api_client=stubs.api)
1016 self.assertEqual(exited, 0)
1020 "name": "submit_wf.cwl",
1022 "definition": self.expect_workflow,
1025 stubs.api.workflows().update.assert_called_with(
1026 uuid=self.existing_workflow_uuid,
1027 body=JsonDiffMatcher(body))
1028 self.assertEqual(capture_stdout.getvalue(),
1029 self.existing_workflow_uuid + '\n')
1033 def test_update_name(self, stubs):
1034 capture_stdout = cStringIO.StringIO()
1036 exited = arvados_cwl.main(
1037 ["--update-workflow", self.existing_workflow_uuid,
1038 "--debug", "--name", "testing 123",
1039 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1040 capture_stdout, sys.stderr, api_client=stubs.api)
1041 self.assertEqual(exited, 0)
1045 "name": "testing 123",
1047 "definition": self.expect_workflow,
1050 stubs.api.workflows().update.assert_called_with(
1051 uuid=self.existing_workflow_uuid,
1052 body=JsonDiffMatcher(body))
1053 self.assertEqual(capture_stdout.getvalue(),
1054 self.existing_workflow_uuid + '\n')
1057 class TestTemplateInputs(unittest.TestCase):
1060 "inputs_test.cwl": {
1061 'runtime_constraints': {
1062 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1063 'min_ram_mb_per_node': 1024
1065 'script_parameters': {
1067 '99999999999999999999999999999991+99/'
1068 'wf/inputs_test.cwl',
1069 'optionalFloatInput': None,
1072 'dataclass': 'File',
1074 'title': "It's a file; we expect to find some characters in it.",
1075 'description': 'If there were anything further to say, it would be said here,\nor here.'
1079 'dataclass': 'number',
1081 'title': 'Floats like a duck',
1085 'optionalFloatInput': {
1086 'type': ['null', 'float'],
1087 'dataclass': 'number',
1092 'dataclass': 'boolean',
1094 'title': 'True or false?',
1097 'repository': 'arvados',
1098 'script_version': 'master',
1099 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1100 'script': 'cwl-runner',
1103 "name": "inputs_test.cwl",
1107 def test_inputs_empty(self, stubs):
1108 exited = arvados_cwl.main(
1109 ["--create-template",
1110 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1111 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1112 self.assertEqual(exited, 0)
1114 stubs.api.pipeline_templates().create.assert_called_with(
1115 body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1118 def test_inputs(self, stubs):
1119 exited = arvados_cwl.main(
1120 ["--create-template",
1121 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1122 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1123 self.assertEqual(exited, 0)
1125 expect_template = copy.deepcopy(self.expect_template)
1126 params = expect_template[
1127 "components"]["inputs_test.cwl"]["script_parameters"]
1128 params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
1129 params["floatInput"]["value"] = 1.234
1130 params["boolInput"]["value"] = True
1132 stubs.api.pipeline_templates().create.assert_called_with(
1133 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)