12 import arvados.collection
14 import arvados_cwl.runner
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
23 @functools.wraps(func)
24 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
25 @mock.patch("arvados.collection.KeepClient")
26 @mock.patch("arvados.keep.KeepClient")
27 @mock.patch("arvados.events.subscribe")
28 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
33 stubs.keepdocker = keepdocker
36 def putstub(p, **kwargs):
37 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
38 keep_client1().put.side_effect = putstub
39 keep_client1.put.side_effect = putstub
40 keep_client2().put.side_effect = putstub
41 keep_client2.put.side_effect = putstub
43 stubs.keep_client = keep_client2
44 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
45 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
47 stubs.api = mock.MagicMock()
48 stubs.api._rootDesc = get_rootDesc()
50 stubs.api.users().current().execute.return_value = {
51 "uuid": stubs.fake_user_uuid,
53 stubs.api.collections().list().execute.return_value = {"items": []}
54 stubs.api.collections().create().execute.side_effect = ({
55 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
56 "portable_data_hash": "99999999999999999999999999999991+99",
59 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
60 "portable_data_hash": "99999999999999999999999999999992+99",
61 "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
64 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
65 "portable_data_hash": "99999999999999999999999999999994+99",
69 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
70 "portable_data_hash": "99999999999999999999999999999995+99",
74 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
75 "portable_data_hash": "99999999999999999999999999999996+99",
79 stubs.api.collections().get().execute.return_value = {
80 "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
82 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
83 stubs.api.jobs().create().execute.return_value = {
84 "uuid": stubs.expect_job_uuid,
88 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
89 stubs.api.container_requests().create().execute.return_value = {
90 "uuid": stubs.expect_container_request_uuid,
91 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
95 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
96 stubs.api.pipeline_templates().create().execute.return_value = {
97 "uuid": stubs.expect_pipeline_template_uuid,
99 stubs.expect_job_spec = {
100 'runtime_constraints': {
101 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
102 'min_ram_mb_per_node': 1024
104 'script_parameters': {
106 'basename': 'blorp.txt',
107 'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
111 'basename': '99999999999999999999999999999998+99',
112 'location': 'keep:99999999999999999999999999999998+99',
116 'basename': 'anonymous',
118 "basename": "renamed.txt",
120 "location": "keep:99999999999999999999999999999998+99/file1.txt"
125 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
127 'repository': 'arvados',
128 'script_version': 'master',
129 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
130 'script': 'cwl-runner'
132 stubs.pipeline_component = stubs.expect_job_spec.copy()
133 stubs.expect_pipeline_instance = {
134 'name': 'submit_wf.cwl',
135 'state': 'RunningOnServer',
139 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
140 'script_parameters': {
141 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
142 'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
143 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
145 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
147 'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
148 'arv:enable_reuse': True,
149 'arv:on_error': 'continue'
151 'repository': 'arvados',
152 'script_version': 'master',
153 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
154 'script': 'cwl-runner',
155 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
159 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
160 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
161 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
162 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
163 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
164 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
167 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
168 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
170 stubs.expect_container_spec = {
177 '/var/lib/cwl/workflow': {
178 'portable_data_hash': '99999999999999999999999999999991+99',
182 'path': '/var/spool/cwl/cwl.output.json',
185 '/var/lib/cwl/cwl.input.json': {
188 'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
189 'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999994+99/blorp.txt'},
190 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
191 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
197 'state': 'Committed',
199 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
200 '--enable-reuse', '--on-error=continue',
201 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json'],
202 'name': 'submit_wf.cwl',
203 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
204 'output_path': '/var/spool/cwl',
205 'cwd': '/var/spool/cwl',
206 'runtime_constraints': {
209 'ram': 1024*1024*1024
214 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
215 stubs.api.workflows().create().execute.return_value = {
216 "uuid": stubs.expect_workflow_uuid,
218 def update_mock(**kwargs):
219 stubs.updated_uuid = kwargs.get('uuid')
221 stubs.api.workflows().update.side_effect = update_mock
222 stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
223 "uuid": stubs.updated_uuid,
226 return func(self, stubs, *args, **kwargs)
230 class TestSubmit(unittest.TestCase):
231 @mock.patch("time.sleep")
233 def test_submit(self, stubs, tm):
234 capture_stdout = cStringIO.StringIO()
235 exited = arvados_cwl.main(
236 ["--submit", "--no-wait", "--debug",
237 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
238 capture_stdout, sys.stderr, api_client=stubs.api)
239 self.assertEqual(exited, 0)
241 stubs.api.collections().create.assert_has_calls([
245 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
246 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
247 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
249 'name': 'submit_wf.cwl',
250 }, ensure_unique_name=True),
251 mock.call().execute(),
252 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
253 '0:0:blub.txt 0:0:submit_tool.cwl\n',
254 'replication_desired': None,
255 'name': 'New collection'
256 }, ensure_unique_name=True),
257 mock.call().execute(num_retries=4),
260 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
263 }, ensure_unique_name=True),
264 mock.call().execute()])
266 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
267 stubs.api.pipeline_instances().create.assert_called_with(
268 body=expect_pipeline)
269 self.assertEqual(capture_stdout.getvalue(),
270 stubs.expect_pipeline_uuid + '\n')
273 @mock.patch("time.sleep")
275 def test_submit_no_reuse(self, stubs, tm):
276 capture_stdout = cStringIO.StringIO()
277 exited = arvados_cwl.main(
278 ["--submit", "--no-wait", "--debug", "--disable-reuse",
279 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
280 capture_stdout, sys.stderr, api_client=stubs.api)
281 self.assertEqual(exited, 0)
283 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
285 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
286 stubs.api.pipeline_instances().create.assert_called_with(
287 body=JsonDiffMatcher(expect_pipeline))
288 self.assertEqual(capture_stdout.getvalue(),
289 stubs.expect_pipeline_uuid + '\n')
291 @mock.patch("time.sleep")
293 def test_submit_on_error(self, stubs, tm):
294 capture_stdout = cStringIO.StringIO()
295 exited = arvados_cwl.main(
296 ["--submit", "--no-wait", "--debug", "--on-error=stop",
297 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
298 capture_stdout, sys.stderr, api_client=stubs.api)
299 self.assertEqual(exited, 0)
301 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
303 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
304 stubs.api.pipeline_instances().create.assert_called_with(
305 body=JsonDiffMatcher(expect_pipeline))
306 self.assertEqual(capture_stdout.getvalue(),
307 stubs.expect_pipeline_uuid + '\n')
310 @mock.patch("time.sleep")
312 def test_submit_runner_ram(self, stubs, tm):
313 capture_stdout = cStringIO.StringIO()
314 exited = arvados_cwl.main(
315 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
316 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
317 capture_stdout, sys.stderr, api_client=stubs.api)
318 self.assertEqual(exited, 0)
320 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
322 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
323 stubs.api.pipeline_instances().create.assert_called_with(
324 body=JsonDiffMatcher(expect_pipeline))
325 self.assertEqual(capture_stdout.getvalue(),
326 stubs.expect_pipeline_uuid + '\n')
329 @mock.patch("time.sleep")
331 def test_submit_invalid_runner_ram(self, stubs, tm):
332 capture_stdout = cStringIO.StringIO()
333 exited = arvados_cwl.main(
334 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
335 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
336 capture_stdout, sys.stderr, api_client=stubs.api)
337 self.assertEqual(exited, 1)
339 @mock.patch("time.sleep")
341 def test_submit_output_name(self, stubs, tm):
342 output_name = "test_output_name"
344 capture_stdout = cStringIO.StringIO()
345 exited = arvados_cwl.main(
346 ["--submit", "--no-wait", "--debug", "--output-name", output_name,
347 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
348 capture_stdout, sys.stderr, api_client=stubs.api)
349 self.assertEqual(exited, 0)
351 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
353 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
354 stubs.api.pipeline_instances().create.assert_called_with(
355 body=JsonDiffMatcher(expect_pipeline))
356 self.assertEqual(capture_stdout.getvalue(),
357 stubs.expect_pipeline_uuid + '\n')
360 @mock.patch("time.sleep")
362 def test_submit_pipeline_name(self, stubs, tm):
363 capture_stdout = cStringIO.StringIO()
364 exited = arvados_cwl.main(
365 ["--submit", "--no-wait", "--debug", "--name=hello job 123",
366 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
367 capture_stdout, sys.stderr, api_client=stubs.api)
368 self.assertEqual(exited, 0)
370 stubs.expect_pipeline_instance["name"] = "hello job 123"
372 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
373 stubs.api.pipeline_instances().create.assert_called_with(
374 body=expect_pipeline)
375 self.assertEqual(capture_stdout.getvalue(),
376 stubs.expect_pipeline_uuid + '\n')
378 @mock.patch("time.sleep")
380 def test_submit_output_tags(self, stubs, tm):
381 output_tags = "tag0,tag1,tag2"
383 capture_stdout = cStringIO.StringIO()
384 exited = arvados_cwl.main(
385 ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
386 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
387 capture_stdout, sys.stderr, api_client=stubs.api)
388 self.assertEqual(exited, 0)
390 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
392 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
393 stubs.api.pipeline_instances().create.assert_called_with(
394 body=JsonDiffMatcher(expect_pipeline))
395 self.assertEqual(capture_stdout.getvalue(),
396 stubs.expect_pipeline_uuid + '\n')
398 @mock.patch("time.sleep")
400 def test_submit_with_project_uuid(self, stubs, tm):
401 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
403 exited = arvados_cwl.main(
404 ["--submit", "--no-wait",
405 "--project-uuid", project_uuid,
406 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
407 sys.stdout, sys.stderr, api_client=stubs.api)
408 self.assertEqual(exited, 0)
410 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
411 expect_pipeline["owner_uuid"] = project_uuid
412 stubs.api.pipeline_instances().create.assert_called_with(
413 body=JsonDiffMatcher(expect_pipeline))
416 def test_submit_container(self, stubs):
417 capture_stdout = cStringIO.StringIO()
419 exited = arvados_cwl.main(
420 ["--submit", "--no-wait", "--api=containers", "--debug",
421 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
422 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
423 self.assertEqual(exited, 0)
425 logging.exception("")
427 stubs.api.collections().create.assert_has_calls([
431 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
432 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
433 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
435 'name': 'submit_wf.cwl',
436 }, ensure_unique_name=True),
437 mock.call().execute(),
438 mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
439 '0:0:blub.txt 0:0:submit_tool.cwl\n',
440 'name': 'New collection',
441 'replication_desired': None,
442 }, ensure_unique_name=True),
443 mock.call().execute(num_retries=4),
446 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
449 }, ensure_unique_name=True),
450 mock.call().execute()])
452 expect_container = copy.deepcopy(stubs.expect_container_spec)
453 stubs.api.container_requests().create.assert_called_with(
454 body=JsonDiffMatcher(expect_container))
455 self.assertEqual(capture_stdout.getvalue(),
456 stubs.expect_container_request_uuid + '\n')
459 def test_submit_container_no_reuse(self, stubs):
460 capture_stdout = cStringIO.StringIO()
462 exited = arvados_cwl.main(
463 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
464 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
465 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
466 self.assertEqual(exited, 0)
468 logging.exception("")
470 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
471 '--disable-reuse', '--on-error=continue',
472 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
474 expect_container = copy.deepcopy(stubs.expect_container_spec)
475 stubs.api.container_requests().create.assert_called_with(
476 body=JsonDiffMatcher(expect_container))
477 self.assertEqual(capture_stdout.getvalue(),
478 stubs.expect_container_request_uuid + '\n')
482 def test_submit_container_on_error(self, stubs):
483 capture_stdout = cStringIO.StringIO()
485 exited = arvados_cwl.main(
486 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
487 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
488 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
489 self.assertEqual(exited, 0)
491 logging.exception("")
493 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
494 '--enable-reuse', '--on-error=stop',
495 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
497 expect_container = copy.deepcopy(stubs.expect_container_spec)
498 stubs.api.container_requests().create.assert_called_with(
499 body=JsonDiffMatcher(expect_container))
500 self.assertEqual(capture_stdout.getvalue(),
501 stubs.expect_container_request_uuid + '\n')
504 def test_submit_container_output_name(self, stubs):
505 output_name = "test_output_name"
507 capture_stdout = cStringIO.StringIO()
509 exited = arvados_cwl.main(
510 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
511 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
512 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
513 self.assertEqual(exited, 0)
515 logging.exception("")
517 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
518 "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
519 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
520 stubs.expect_container_spec["output_name"] = output_name
522 expect_container = copy.deepcopy(stubs.expect_container_spec)
523 stubs.api.container_requests().create.assert_called_with(
524 body=JsonDiffMatcher(expect_container))
525 self.assertEqual(capture_stdout.getvalue(),
526 stubs.expect_container_request_uuid + '\n')
529 def test_submit_container_output_tags(self, stubs):
530 output_tags = "tag0,tag1,tag2"
532 capture_stdout = cStringIO.StringIO()
534 exited = arvados_cwl.main(
535 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
536 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
537 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
538 self.assertEqual(exited, 0)
540 logging.exception("")
542 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
543 "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
544 '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
546 expect_container = copy.deepcopy(stubs.expect_container_spec)
547 stubs.api.container_requests().create.assert_called_with(
548 body=JsonDiffMatcher(expect_container))
549 self.assertEqual(capture_stdout.getvalue(),
550 stubs.expect_container_request_uuid + '\n')
553 def test_submit_container_runner_ram(self, stubs):
554 capture_stdout = cStringIO.StringIO()
556 exited = arvados_cwl.main(
557 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
558 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
559 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
560 self.assertEqual(exited, 0)
562 logging.exception("")
564 stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
566 expect_container = copy.deepcopy(stubs.expect_container_spec)
567 stubs.api.container_requests().create.assert_called_with(
568 body=JsonDiffMatcher(expect_container))
569 self.assertEqual(capture_stdout.getvalue(),
570 stubs.expect_container_request_uuid + '\n')
572 @mock.patch("arvados.collection.CollectionReader")
573 @mock.patch("time.sleep")
575 def test_submit_file_keepref(self, stubs, tm, collectionReader):
576 capture_stdout = cStringIO.StringIO()
577 exited = arvados_cwl.main(
578 ["--submit", "--no-wait", "--api=containers", "--debug",
579 "tests/wf/submit_keepref_wf.cwl"],
580 capture_stdout, sys.stderr, api_client=stubs.api)
581 self.assertEqual(exited, 0)
584 @mock.patch("arvados.collection.CollectionReader")
585 @mock.patch("time.sleep")
587 def test_submit_keepref(self, stubs, tm, reader):
588 capture_stdout = cStringIO.StringIO()
590 with open("tests/wf/expect_arvworkflow.cwl") as f:
591 reader().open().__enter__().read.return_value = f.read()
593 exited = arvados_cwl.main(
594 ["--submit", "--no-wait", "--api=containers", "--debug",
595 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
596 capture_stdout, sys.stderr, api_client=stubs.api)
597 self.assertEqual(exited, 0)
607 'path': '/var/spool/cwl/cwl.output.json',
610 '/var/lib/cwl/workflow': {
611 'portable_data_hash': '99999999999999999999999999999994+99',
614 '/var/lib/cwl/cwl.input.json': {
620 }, 'state': 'Committed',
622 'output_path': '/var/spool/cwl',
623 'name': 'expect_arvworkflow.cwl#main',
624 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
625 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
626 '--enable-reuse', '--on-error=continue',
627 '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
628 'cwd': '/var/spool/cwl',
629 'runtime_constraints': {
637 stubs.api.container_requests().create.assert_called_with(
638 body=JsonDiffMatcher(expect_container))
639 self.assertEqual(capture_stdout.getvalue(),
640 stubs.expect_container_request_uuid + '\n')
642 @mock.patch("time.sleep")
644 def test_submit_arvworkflow(self, stubs, tm):
645 capture_stdout = cStringIO.StringIO()
647 with open("tests/wf/expect_arvworkflow.cwl") as f:
648 stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
650 exited = arvados_cwl.main(
651 ["--submit", "--no-wait", "--api=containers", "--debug",
652 "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
653 capture_stdout, sys.stderr, api_client=stubs.api)
654 self.assertEqual(exited, 0)
664 'path': '/var/spool/cwl/cwl.output.json',
667 '/var/lib/cwl/workflow.json': {
670 'cwlVersion': 'v1.0',
675 'inputBinding': {'position': 1},
677 'id': '#submit_tool.cwl/x'}
680 {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
682 'id': '#submit_tool.cwl',
684 'baseCommand': 'cat',
685 'class': 'CommandLineTool'
689 {'type': 'string', 'id': '#main/x'}
692 {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
693 'run': '#submit_tool.cwl',
703 '/var/lib/cwl/cwl.input.json': {
709 }, 'state': 'Committed',
711 'output_path': '/var/spool/cwl',
712 'name': 'a test workflow',
713 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
714 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
715 '--enable-reuse', '--on-error=continue',
716 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
717 'cwd': '/var/spool/cwl',
718 'runtime_constraints': {
724 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
728 stubs.api.container_requests().create.assert_called_with(
729 body=JsonDiffMatcher(expect_container))
730 self.assertEqual(capture_stdout.getvalue(),
731 stubs.expect_container_request_uuid + '\n')
735 def test_submit_container_name(self, stubs):
736 capture_stdout = cStringIO.StringIO()
738 exited = arvados_cwl.main(
739 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
740 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
741 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
742 self.assertEqual(exited, 0)
744 logging.exception("")
746 stubs.expect_container_spec["name"] = "hello container 123"
748 expect_container = copy.deepcopy(stubs.expect_container_spec)
749 stubs.api.container_requests().create.assert_called_with(
750 body=expect_container)
751 self.assertEqual(capture_stdout.getvalue(),
752 stubs.expect_container_request_uuid + '\n')
756 def test_submit_job_runner_image(self, stubs):
757 capture_stdout = cStringIO.StringIO()
759 exited = arvados_cwl.main(
760 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
761 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
762 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
763 self.assertEqual(exited, 0)
765 logging.exception("")
767 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
769 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
770 stubs.api.pipeline_instances().create.assert_called_with(
771 body=JsonDiffMatcher(expect_pipeline))
772 self.assertEqual(capture_stdout.getvalue(),
773 stubs.expect_pipeline_uuid + '\n')
776 def test_submit_container_runner_image(self, stubs):
777 capture_stdout = cStringIO.StringIO()
779 exited = arvados_cwl.main(
780 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
781 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
782 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
783 self.assertEqual(exited, 0)
785 logging.exception("")
787 stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
789 expect_container = copy.deepcopy(stubs.expect_container_spec)
790 stubs.api.container_requests().create.assert_called_with(
791 body=JsonDiffMatcher(expect_container))
792 self.assertEqual(capture_stdout.getvalue(),
793 stubs.expect_container_request_uuid + '\n')
796 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
797 @mock.patch("cwltool.docker.get_image")
798 @mock.patch("arvados.api")
799 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
800 arvrunner = mock.MagicMock()
801 arvrunner.project_uuid = ""
802 api.return_value = mock.MagicMock()
803 arvrunner.api = api.return_value
804 arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
805 {"items": [], "items_available": 0, "offset": 0},
806 {"items": [], "items_available": 0, "offset": 0},
807 {"items": [{"created_at": "",
809 "link_class": "docker_image_hash",
812 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
813 {"items": [], "items_available": 0, "offset": 0},
814 {"items": [{"created_at": "",
816 "link_class": "docker_image_repo+tag",
817 "name": "arvados/jobs:"+arvados_cwl.__version__,
819 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
820 {"items": [{"created_at": "",
822 "link_class": "docker_image_hash",
825 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
827 find_one_image_hash.return_value = "123456"
829 arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
830 {"items": [{"uuid": "",
834 }], "items_available": 1, "offset": 0},
835 {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
836 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
837 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
839 class TestCreateTemplate(unittest.TestCase):
840 existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
842 def _adjust_script_params(self, expect_component):
843 expect_component['script_parameters']['x'] = {
847 'value': '99999999999999999999999999999994+99/blorp.txt',
849 expect_component['script_parameters']['y'] = {
850 'dataclass': 'Collection',
853 'value': '99999999999999999999999999999998+99',
855 expect_component['script_parameters']['z'] = {
856 'dataclass': 'Collection',
862 def test_create(self, stubs):
863 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
865 capture_stdout = cStringIO.StringIO()
867 exited = arvados_cwl.main(
868 ["--create-workflow", "--debug",
870 "--project-uuid", project_uuid,
871 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
872 capture_stdout, sys.stderr, api_client=stubs.api)
873 self.assertEqual(exited, 0)
875 stubs.api.pipeline_instances().create.refute_called()
876 stubs.api.jobs().create.refute_called()
878 expect_component = copy.deepcopy(stubs.expect_job_spec)
879 self._adjust_script_params(expect_component)
882 "submit_wf.cwl": expect_component,
884 "name": "submit_wf.cwl",
885 "owner_uuid": project_uuid,
887 stubs.api.pipeline_templates().create.assert_called_with(
888 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
890 self.assertEqual(capture_stdout.getvalue(),
891 stubs.expect_pipeline_template_uuid + '\n')
895 def test_create_name(self, stubs):
896 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
898 capture_stdout = cStringIO.StringIO()
900 exited = arvados_cwl.main(
901 ["--create-workflow", "--debug",
902 "--project-uuid", project_uuid,
904 "--name", "testing 123",
905 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
906 capture_stdout, sys.stderr, api_client=stubs.api)
907 self.assertEqual(exited, 0)
909 stubs.api.pipeline_instances().create.refute_called()
910 stubs.api.jobs().create.refute_called()
912 expect_component = copy.deepcopy(stubs.expect_job_spec)
913 self._adjust_script_params(expect_component)
916 "testing 123": expect_component,
918 "name": "testing 123",
919 "owner_uuid": project_uuid,
921 stubs.api.pipeline_templates().create.assert_called_with(
922 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
924 self.assertEqual(capture_stdout.getvalue(),
925 stubs.expect_pipeline_template_uuid + '\n')
929 def test_update_name(self, stubs):
930 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
932 capture_stdout = cStringIO.StringIO()
934 exited = arvados_cwl.main(
935 ["--update-workflow", self.existing_template_uuid,
937 "--project-uuid", project_uuid,
939 "--name", "testing 123",
940 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
941 capture_stdout, sys.stderr, api_client=stubs.api)
942 self.assertEqual(exited, 0)
944 stubs.api.pipeline_instances().create.refute_called()
945 stubs.api.jobs().create.refute_called()
947 expect_component = copy.deepcopy(stubs.expect_job_spec)
948 self._adjust_script_params(expect_component)
951 "testing 123": expect_component,
953 "name": "testing 123",
954 "owner_uuid": project_uuid,
956 stubs.api.pipeline_templates().create.refute_called()
957 stubs.api.pipeline_templates().update.assert_called_with(
958 body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
960 self.assertEqual(capture_stdout.getvalue(),
961 self.existing_template_uuid + '\n')
964 class TestCreateWorkflow(unittest.TestCase):
965 existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
966 expect_workflow = open("tests/wf/expect_packed.cwl").read()
969 def test_create(self, stubs):
970 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
972 capture_stdout = cStringIO.StringIO()
974 exited = arvados_cwl.main(
975 ["--create-workflow", "--debug",
977 "--project-uuid", project_uuid,
978 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
979 capture_stdout, sys.stderr, api_client=stubs.api)
980 self.assertEqual(exited, 0)
982 stubs.api.pipeline_templates().create.refute_called()
983 stubs.api.container_requests().create.refute_called()
987 "owner_uuid": project_uuid,
988 "name": "submit_wf.cwl",
990 "definition": self.expect_workflow,
993 stubs.api.workflows().create.assert_called_with(
994 body=JsonDiffMatcher(body))
996 self.assertEqual(capture_stdout.getvalue(),
997 stubs.expect_workflow_uuid + '\n')
1001 def test_create_name(self, stubs):
1002 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1004 capture_stdout = cStringIO.StringIO()
1006 exited = arvados_cwl.main(
1007 ["--create-workflow", "--debug",
1009 "--project-uuid", project_uuid,
1010 "--name", "testing 123",
1011 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1012 capture_stdout, sys.stderr, api_client=stubs.api)
1013 self.assertEqual(exited, 0)
1015 stubs.api.pipeline_templates().create.refute_called()
1016 stubs.api.container_requests().create.refute_called()
1020 "owner_uuid": project_uuid,
1021 "name": "testing 123",
1023 "definition": self.expect_workflow,
1026 stubs.api.workflows().create.assert_called_with(
1027 body=JsonDiffMatcher(body))
1029 self.assertEqual(capture_stdout.getvalue(),
1030 stubs.expect_workflow_uuid + '\n')
1033 def test_incompatible_api(self, stubs):
1034 capture_stderr = cStringIO.StringIO()
1035 logging.getLogger('arvados.cwl-runner').addHandler(
1036 logging.StreamHandler(capture_stderr))
1038 exited = arvados_cwl.main(
1039 ["--update-workflow", self.existing_workflow_uuid,
1042 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1043 sys.stderr, sys.stderr, api_client=stubs.api)
1044 self.assertEqual(exited, 1)
1045 self.assertRegexpMatches(
1046 capture_stderr.getvalue(),
1047 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1050 def test_update(self, stubs):
1051 capture_stdout = cStringIO.StringIO()
1053 exited = arvados_cwl.main(
1054 ["--update-workflow", self.existing_workflow_uuid,
1056 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1057 capture_stdout, sys.stderr, api_client=stubs.api)
1058 self.assertEqual(exited, 0)
1062 "name": "submit_wf.cwl",
1064 "definition": self.expect_workflow,
1067 stubs.api.workflows().update.assert_called_with(
1068 uuid=self.existing_workflow_uuid,
1069 body=JsonDiffMatcher(body))
1070 self.assertEqual(capture_stdout.getvalue(),
1071 self.existing_workflow_uuid + '\n')
1075 def test_update_name(self, stubs):
1076 capture_stdout = cStringIO.StringIO()
1078 exited = arvados_cwl.main(
1079 ["--update-workflow", self.existing_workflow_uuid,
1080 "--debug", "--name", "testing 123",
1081 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1082 capture_stdout, sys.stderr, api_client=stubs.api)
1083 self.assertEqual(exited, 0)
1087 "name": "testing 123",
1089 "definition": self.expect_workflow,
1092 stubs.api.workflows().update.assert_called_with(
1093 uuid=self.existing_workflow_uuid,
1094 body=JsonDiffMatcher(body))
1095 self.assertEqual(capture_stdout.getvalue(),
1096 self.existing_workflow_uuid + '\n')
1099 class TestTemplateInputs(unittest.TestCase):
1102 "inputs_test.cwl": {
1103 'runtime_constraints': {
1104 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1105 'min_ram_mb_per_node': 1024
1107 'script_parameters': {
1109 '99999999999999999999999999999991+99/'
1110 'wf/inputs_test.cwl',
1111 'optionalFloatInput': None,
1114 'dataclass': 'File',
1116 'title': "It's a file; we expect to find some characters in it.",
1117 'description': 'If there were anything further to say, it would be said here,\nor here.'
1121 'dataclass': 'number',
1123 'title': 'Floats like a duck',
1127 'optionalFloatInput': {
1128 'type': ['null', 'float'],
1129 'dataclass': 'number',
1134 'dataclass': 'boolean',
1136 'title': 'True or false?',
1139 'repository': 'arvados',
1140 'script_version': 'master',
1141 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1142 'script': 'cwl-runner',
1145 "name": "inputs_test.cwl",
1149 def test_inputs_empty(self, stubs):
1150 exited = arvados_cwl.main(
1151 ["--create-template",
1152 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1153 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1154 self.assertEqual(exited, 0)
1156 stubs.api.pipeline_templates().create.assert_called_with(
1157 body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1160 def test_inputs(self, stubs):
1161 exited = arvados_cwl.main(
1162 ["--create-template",
1163 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1164 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1165 self.assertEqual(exited, 0)
1167 expect_template = copy.deepcopy(self.expect_template)
1168 params = expect_template[
1169 "components"]["inputs_test.cwl"]["script_parameters"]
1170 params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
1171 params["floatInput"]["value"] = 1.234
1172 params["boolInput"]["value"] = True
1174 stubs.api.pipeline_templates().create.assert_called_with(
1175 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)