12 import arvados.collection
14 import arvados_cwl.runner
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
20 import ruamel.yaml as yaml
25 @functools.wraps(func)
26 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
27 @mock.patch("arvados.collection.KeepClient")
28 @mock.patch("arvados.keep.KeepClient")
29 @mock.patch("arvados.events.subscribe")
30 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
35 stubs.keepdocker = keepdocker
38 def putstub(p, **kwargs):
39 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
40 keep_client1().put.side_effect = putstub
41 keep_client1.put.side_effect = putstub
42 keep_client2().put.side_effect = putstub
43 keep_client2.put.side_effect = putstub
45 stubs.keep_client = keep_client2
46 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
47 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
49 stubs.api = mock.MagicMock()
50 stubs.api._rootDesc = get_rootDesc()
52 stubs.api.users().current().execute.return_value = {
53 "uuid": stubs.fake_user_uuid,
55 stubs.api.collections().list().execute.return_value = {"items": []}
56 stubs.api.collections().create().execute.side_effect = ({
57 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
58 "portable_data_hash": "99999999999999999999999999999991+99",
61 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
62 "portable_data_hash": "99999999999999999999999999999992+99",
63 "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
66 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
67 "portable_data_hash": "99999999999999999999999999999994+99",
71 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
72 "portable_data_hash": "99999999999999999999999999999995+99",
76 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
77 "portable_data_hash": "99999999999999999999999999999996+99",
81 stubs.api.collections().get().execute.return_value = {
82 "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
84 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
85 stubs.api.jobs().create().execute.return_value = {
86 "uuid": stubs.expect_job_uuid,
90 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
91 stubs.api.container_requests().create().execute.return_value = {
92 "uuid": stubs.expect_container_request_uuid,
93 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
97 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
98 stubs.api.pipeline_templates().create().execute.return_value = {
99 "uuid": stubs.expect_pipeline_template_uuid,
101 stubs.expect_job_spec = {
102 'runtime_constraints': {
103 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
104 'min_ram_mb_per_node': 1024
106 'script_parameters': {
108 'basename': 'blorp.txt',
109 'location': 'keep:99999999999999999999999999999992+99/blorp.txt',
113 'basename': '99999999999999999999999999999998+99',
114 'location': 'keep:99999999999999999999999999999998+99',
118 'basename': 'anonymous',
120 "basename": "renamed.txt",
122 "location": "keep:99999999999999999999999999999998+99/file1.txt"
127 'ef2c299cb4e5a565a46d94887eafdc62+58/workflow.cwl'
129 'repository': 'arvados',
130 'script_version': 'master',
131 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
132 'script': 'cwl-runner'
134 stubs.pipeline_component = stubs.expect_job_spec.copy()
135 stubs.expect_pipeline_instance = {
136 'name': 'submit_wf.cwl',
137 'state': 'RunningOnServer',
141 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
142 'script_parameters': {
143 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
144 'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999992+99/blorp.txt'}},
145 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
147 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
149 'cwl:tool': 'f57578d5cfda7f70fef00cbc4b621e6b+58/workflow.cwl',
150 'arv:enable_reuse': True,
151 'arv:on_error': 'continue'
153 'repository': 'arvados',
154 'script_version': 'master',
155 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
156 'script': 'cwl-runner',
157 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
161 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
162 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
163 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
164 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
165 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
166 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
169 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
170 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
172 with open("tests/wf/submit_wf_packed.cwl") as f:
173 expect_packed_workflow = yaml.round_trip_load(f)
175 stubs.expect_container_spec = {
182 '/var/lib/cwl/workflow.json': {
183 'json': expect_packed_workflow,
187 'path': '/var/spool/cwl/cwl.output.json',
190 '/var/lib/cwl/cwl.input.json': {
193 'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
194 'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999992+99/blorp.txt'},
195 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
196 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
202 'state': 'Committed',
204 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
205 '--enable-reuse', '--on-error=continue',
206 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
207 'name': 'submit_wf.cwl',
208 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
209 'output_path': '/var/spool/cwl',
210 'cwd': '/var/spool/cwl',
211 'runtime_constraints': {
214 'ram': 1024*1024*1024
219 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
220 stubs.api.workflows().create().execute.return_value = {
221 "uuid": stubs.expect_workflow_uuid,
223 def update_mock(**kwargs):
224 stubs.updated_uuid = kwargs.get('uuid')
226 stubs.api.workflows().update.side_effect = update_mock
227 stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
228 "uuid": stubs.updated_uuid,
231 return func(self, stubs, *args, **kwargs)
235 class TestSubmit(unittest.TestCase):
236 @mock.patch("time.sleep")
238 def test_submit(self, stubs, tm):
239 capture_stdout = cStringIO.StringIO()
240 exited = arvados_cwl.main(
241 ["--submit", "--no-wait", "--debug",
242 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
243 capture_stdout, sys.stderr, api_client=stubs.api)
244 self.assertEqual(exited, 0)
246 stubs.api.collections().create.assert_has_calls([
248 mock.call(body=JsonDiffMatcher({
250 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
252 'name': 'submit_tool.cwl dependencies',
253 }), ensure_unique_name=True),
254 mock.call().execute(),
255 mock.call(body=JsonDiffMatcher({
257 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
259 'name': 'submit_wf.cwl input',
260 }), ensure_unique_name=True),
261 mock.call().execute()])
263 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
264 stubs.api.pipeline_instances().create.assert_called_with(
265 body=JsonDiffMatcher(expect_pipeline))
266 self.assertEqual(capture_stdout.getvalue(),
267 stubs.expect_pipeline_uuid + '\n')
270 @mock.patch("time.sleep")
272 def test_submit_no_reuse(self, stubs, tm):
273 capture_stdout = cStringIO.StringIO()
274 exited = arvados_cwl.main(
275 ["--submit", "--no-wait", "--debug", "--disable-reuse",
276 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
277 capture_stdout, sys.stderr, api_client=stubs.api)
278 self.assertEqual(exited, 0)
280 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
282 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
283 stubs.api.pipeline_instances().create.assert_called_with(
284 body=JsonDiffMatcher(expect_pipeline))
285 self.assertEqual(capture_stdout.getvalue(),
286 stubs.expect_pipeline_uuid + '\n')
288 @mock.patch("time.sleep")
290 def test_submit_on_error(self, stubs, tm):
291 capture_stdout = cStringIO.StringIO()
292 exited = arvados_cwl.main(
293 ["--submit", "--no-wait", "--debug", "--on-error=stop",
294 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
295 capture_stdout, sys.stderr, api_client=stubs.api)
296 self.assertEqual(exited, 0)
298 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
300 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
301 stubs.api.pipeline_instances().create.assert_called_with(
302 body=JsonDiffMatcher(expect_pipeline))
303 self.assertEqual(capture_stdout.getvalue(),
304 stubs.expect_pipeline_uuid + '\n')
307 @mock.patch("time.sleep")
309 def test_submit_runner_ram(self, stubs, tm):
310 capture_stdout = cStringIO.StringIO()
311 exited = arvados_cwl.main(
312 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
313 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
314 capture_stdout, sys.stderr, api_client=stubs.api)
315 self.assertEqual(exited, 0)
317 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
319 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
320 stubs.api.pipeline_instances().create.assert_called_with(
321 body=JsonDiffMatcher(expect_pipeline))
322 self.assertEqual(capture_stdout.getvalue(),
323 stubs.expect_pipeline_uuid + '\n')
326 @mock.patch("time.sleep")
328 def test_submit_invalid_runner_ram(self, stubs, tm):
329 capture_stdout = cStringIO.StringIO()
330 exited = arvados_cwl.main(
331 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
332 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
333 capture_stdout, sys.stderr, api_client=stubs.api)
334 self.assertEqual(exited, 1)
336 @mock.patch("time.sleep")
338 def test_submit_output_name(self, stubs, tm):
339 output_name = "test_output_name"
341 capture_stdout = cStringIO.StringIO()
342 exited = arvados_cwl.main(
343 ["--submit", "--no-wait", "--debug", "--output-name", output_name,
344 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
345 capture_stdout, sys.stderr, api_client=stubs.api)
346 self.assertEqual(exited, 0)
348 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
350 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
351 stubs.api.pipeline_instances().create.assert_called_with(
352 body=JsonDiffMatcher(expect_pipeline))
353 self.assertEqual(capture_stdout.getvalue(),
354 stubs.expect_pipeline_uuid + '\n')
357 @mock.patch("time.sleep")
359 def test_submit_pipeline_name(self, stubs, tm):
360 capture_stdout = cStringIO.StringIO()
361 exited = arvados_cwl.main(
362 ["--submit", "--no-wait", "--debug", "--name=hello job 123",
363 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
364 capture_stdout, sys.stderr, api_client=stubs.api)
365 self.assertEqual(exited, 0)
367 stubs.expect_pipeline_instance["name"] = "hello job 123"
369 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
370 stubs.api.pipeline_instances().create.assert_called_with(
371 body=expect_pipeline)
372 self.assertEqual(capture_stdout.getvalue(),
373 stubs.expect_pipeline_uuid + '\n')
375 @mock.patch("time.sleep")
377 def test_submit_output_tags(self, stubs, tm):
378 output_tags = "tag0,tag1,tag2"
380 capture_stdout = cStringIO.StringIO()
381 exited = arvados_cwl.main(
382 ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
383 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
384 capture_stdout, sys.stderr, api_client=stubs.api)
385 self.assertEqual(exited, 0)
387 stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
389 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
390 stubs.api.pipeline_instances().create.assert_called_with(
391 body=JsonDiffMatcher(expect_pipeline))
392 self.assertEqual(capture_stdout.getvalue(),
393 stubs.expect_pipeline_uuid + '\n')
395 @mock.patch("time.sleep")
397 def test_submit_with_project_uuid(self, stubs, tm):
398 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
400 exited = arvados_cwl.main(
401 ["--submit", "--no-wait",
402 "--project-uuid", project_uuid,
403 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
404 sys.stdout, sys.stderr, api_client=stubs.api)
405 self.assertEqual(exited, 0)
407 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
408 expect_pipeline["owner_uuid"] = project_uuid
409 stubs.api.pipeline_instances().create.assert_called_with(
410 body=JsonDiffMatcher(expect_pipeline))
413 def test_submit_container(self, stubs):
414 capture_stdout = cStringIO.StringIO()
416 exited = arvados_cwl.main(
417 ["--submit", "--no-wait", "--api=containers", "--debug",
418 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
419 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
420 self.assertEqual(exited, 0)
422 logging.exception("")
424 stubs.api.collections().create.assert_has_calls([
426 mock.call(body=JsonDiffMatcher({
428 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
430 'name': 'submit_tool.cwl dependencies',
431 }), ensure_unique_name=True),
432 mock.call().execute(),
433 mock.call(body=JsonDiffMatcher({
435 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
437 'name': 'submit_wf.cwl input',
438 }), ensure_unique_name=True),
439 mock.call().execute()])
441 expect_container = copy.deepcopy(stubs.expect_container_spec)
442 stubs.api.container_requests().create.assert_called_with(
443 body=JsonDiffMatcher(expect_container))
444 self.assertEqual(capture_stdout.getvalue(),
445 stubs.expect_container_request_uuid + '\n')
448 def test_submit_container_no_reuse(self, stubs):
449 capture_stdout = cStringIO.StringIO()
451 exited = arvados_cwl.main(
452 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
453 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
454 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
455 self.assertEqual(exited, 0)
457 logging.exception("")
459 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
460 '--disable-reuse', '--on-error=continue',
461 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
463 expect_container = copy.deepcopy(stubs.expect_container_spec)
464 stubs.api.container_requests().create.assert_called_with(
465 body=JsonDiffMatcher(expect_container))
466 self.assertEqual(capture_stdout.getvalue(),
467 stubs.expect_container_request_uuid + '\n')
471 def test_submit_container_on_error(self, stubs):
472 capture_stdout = cStringIO.StringIO()
474 exited = arvados_cwl.main(
475 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
476 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
477 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
478 self.assertEqual(exited, 0)
480 logging.exception("")
482 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
483 '--enable-reuse', '--on-error=stop',
484 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
486 expect_container = copy.deepcopy(stubs.expect_container_spec)
487 stubs.api.container_requests().create.assert_called_with(
488 body=JsonDiffMatcher(expect_container))
489 self.assertEqual(capture_stdout.getvalue(),
490 stubs.expect_container_request_uuid + '\n')
493 def test_submit_container_output_name(self, stubs):
494 output_name = "test_output_name"
496 capture_stdout = cStringIO.StringIO()
498 exited = arvados_cwl.main(
499 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
500 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
501 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
502 self.assertEqual(exited, 0)
504 logging.exception("")
506 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
507 "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
508 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
509 stubs.expect_container_spec["output_name"] = output_name
511 expect_container = copy.deepcopy(stubs.expect_container_spec)
512 stubs.api.container_requests().create.assert_called_with(
513 body=JsonDiffMatcher(expect_container))
514 self.assertEqual(capture_stdout.getvalue(),
515 stubs.expect_container_request_uuid + '\n')
518 def test_submit_container_output_tags(self, stubs):
519 output_tags = "tag0,tag1,tag2"
521 capture_stdout = cStringIO.StringIO()
523 exited = arvados_cwl.main(
524 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
525 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
526 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
527 self.assertEqual(exited, 0)
529 logging.exception("")
531 stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
532 "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
533 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
535 expect_container = copy.deepcopy(stubs.expect_container_spec)
536 stubs.api.container_requests().create.assert_called_with(
537 body=JsonDiffMatcher(expect_container))
538 self.assertEqual(capture_stdout.getvalue(),
539 stubs.expect_container_request_uuid + '\n')
542 def test_submit_container_runner_ram(self, stubs):
543 capture_stdout = cStringIO.StringIO()
545 exited = arvados_cwl.main(
546 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
547 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
548 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
549 self.assertEqual(exited, 0)
551 logging.exception("")
553 stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
555 expect_container = copy.deepcopy(stubs.expect_container_spec)
556 stubs.api.container_requests().create.assert_called_with(
557 body=JsonDiffMatcher(expect_container))
558 self.assertEqual(capture_stdout.getvalue(),
559 stubs.expect_container_request_uuid + '\n')
561 @mock.patch("arvados.collection.CollectionReader")
562 @mock.patch("time.sleep")
564 def test_submit_file_keepref(self, stubs, tm, collectionReader):
565 capture_stdout = cStringIO.StringIO()
566 exited = arvados_cwl.main(
567 ["--submit", "--no-wait", "--api=containers", "--debug",
568 "tests/wf/submit_keepref_wf.cwl"],
569 capture_stdout, sys.stderr, api_client=stubs.api)
570 self.assertEqual(exited, 0)
573 @mock.patch("arvados.collection.CollectionReader")
574 @mock.patch("time.sleep")
576 def test_submit_keepref(self, stubs, tm, reader):
577 capture_stdout = cStringIO.StringIO()
579 with open("tests/wf/expect_arvworkflow.cwl") as f:
580 reader().open().__enter__().read.return_value = f.read()
582 exited = arvados_cwl.main(
583 ["--submit", "--no-wait", "--api=containers", "--debug",
584 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
585 capture_stdout, sys.stderr, api_client=stubs.api)
586 self.assertEqual(exited, 0)
596 'path': '/var/spool/cwl/cwl.output.json',
599 '/var/lib/cwl/workflow': {
600 'portable_data_hash': '99999999999999999999999999999994+99',
603 '/var/lib/cwl/cwl.input.json': {
609 }, 'state': 'Committed',
611 'output_path': '/var/spool/cwl',
612 'name': 'expect_arvworkflow.cwl#main',
613 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
614 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
615 '--enable-reuse', '--on-error=continue',
616 '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
617 'cwd': '/var/spool/cwl',
618 'runtime_constraints': {
626 stubs.api.container_requests().create.assert_called_with(
627 body=JsonDiffMatcher(expect_container))
628 self.assertEqual(capture_stdout.getvalue(),
629 stubs.expect_container_request_uuid + '\n')
631 @mock.patch("time.sleep")
633 def test_submit_arvworkflow(self, stubs, tm):
634 capture_stdout = cStringIO.StringIO()
636 with open("tests/wf/expect_arvworkflow.cwl") as f:
637 stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
639 exited = arvados_cwl.main(
640 ["--submit", "--no-wait", "--api=containers", "--debug",
641 "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
642 capture_stdout, sys.stderr, api_client=stubs.api)
643 self.assertEqual(exited, 0)
653 'path': '/var/spool/cwl/cwl.output.json',
656 '/var/lib/cwl/workflow.json': {
659 'cwlVersion': 'v1.0',
664 {'type': 'string', 'id': '#main/x'}
667 {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
668 'run': '#submit_tool.cwl',
678 'inputBinding': {'position': 1},
680 'id': '#submit_tool.cwl/x'}
683 {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
685 'id': '#submit_tool.cwl',
687 'baseCommand': 'cat',
688 'class': 'CommandLineTool'
693 '/var/lib/cwl/cwl.input.json': {
699 }, 'state': 'Committed',
701 'output_path': '/var/spool/cwl',
702 'name': 'a test workflow',
703 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
704 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
705 '--enable-reuse', '--on-error=continue',
706 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
707 'cwd': '/var/spool/cwl',
708 'runtime_constraints': {
714 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
718 stubs.api.container_requests().create.assert_called_with(
719 body=JsonDiffMatcher(expect_container))
720 self.assertEqual(capture_stdout.getvalue(),
721 stubs.expect_container_request_uuid + '\n')
725 def test_submit_container_name(self, stubs):
726 capture_stdout = cStringIO.StringIO()
728 exited = arvados_cwl.main(
729 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
730 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
731 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
732 self.assertEqual(exited, 0)
734 logging.exception("")
736 stubs.expect_container_spec["name"] = "hello container 123"
738 expect_container = copy.deepcopy(stubs.expect_container_spec)
739 stubs.api.container_requests().create.assert_called_with(
740 body=JsonDiffMatcher(expect_container))
741 self.assertEqual(capture_stdout.getvalue(),
742 stubs.expect_container_request_uuid + '\n')
746 def test_submit_job_runner_image(self, stubs):
747 capture_stdout = cStringIO.StringIO()
749 exited = arvados_cwl.main(
750 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
751 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
752 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
753 self.assertEqual(exited, 0)
755 logging.exception("")
757 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
759 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
760 stubs.api.pipeline_instances().create.assert_called_with(
761 body=JsonDiffMatcher(expect_pipeline))
762 self.assertEqual(capture_stdout.getvalue(),
763 stubs.expect_pipeline_uuid + '\n')
766 def test_submit_container_runner_image(self, stubs):
767 capture_stdout = cStringIO.StringIO()
769 exited = arvados_cwl.main(
770 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
771 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
772 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
773 self.assertEqual(exited, 0)
775 logging.exception("")
777 stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
779 expect_container = copy.deepcopy(stubs.expect_container_spec)
780 stubs.api.container_requests().create.assert_called_with(
781 body=JsonDiffMatcher(expect_container))
782 self.assertEqual(capture_stdout.getvalue(),
783 stubs.expect_container_request_uuid + '\n')
786 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
787 @mock.patch("cwltool.docker.get_image")
788 @mock.patch("arvados.api")
789 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
790 arvrunner = mock.MagicMock()
791 arvrunner.project_uuid = ""
792 api.return_value = mock.MagicMock()
793 arvrunner.api = api.return_value
794 arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
795 {"items": [], "items_available": 0, "offset": 0},
796 {"items": [], "items_available": 0, "offset": 0},
797 {"items": [{"created_at": "",
799 "link_class": "docker_image_hash",
802 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
803 {"items": [], "items_available": 0, "offset": 0},
804 {"items": [{"created_at": "",
806 "link_class": "docker_image_repo+tag",
807 "name": "arvados/jobs:"+arvados_cwl.__version__,
809 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
810 {"items": [{"created_at": "",
812 "link_class": "docker_image_hash",
815 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
817 find_one_image_hash.return_value = "123456"
819 arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
820 {"items": [{"uuid": "",
824 }], "items_available": 1, "offset": 0},
825 {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
826 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
827 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
829 class TestCreateTemplate(unittest.TestCase):
830 existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
832 def _adjust_script_params(self, expect_component):
833 expect_component['script_parameters']['x'] = {
837 'value': '99999999999999999999999999999992+99/blorp.txt',
839 expect_component['script_parameters']['y'] = {
840 'dataclass': 'Collection',
843 'value': '99999999999999999999999999999998+99',
845 expect_component['script_parameters']['z'] = {
846 'dataclass': 'Collection',
852 def test_create(self, stubs):
853 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
855 capture_stdout = cStringIO.StringIO()
857 exited = arvados_cwl.main(
858 ["--create-workflow", "--debug",
860 "--project-uuid", project_uuid,
861 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
862 capture_stdout, sys.stderr, api_client=stubs.api)
863 self.assertEqual(exited, 0)
865 stubs.api.pipeline_instances().create.refute_called()
866 stubs.api.jobs().create.refute_called()
868 expect_component = copy.deepcopy(stubs.expect_job_spec)
869 self._adjust_script_params(expect_component)
872 "submit_wf.cwl": expect_component,
874 "name": "submit_wf.cwl",
875 "owner_uuid": project_uuid,
877 stubs.api.pipeline_templates().create.assert_called_with(
878 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
880 self.assertEqual(capture_stdout.getvalue(),
881 stubs.expect_pipeline_template_uuid + '\n')
885 def test_create_name(self, stubs):
886 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
888 capture_stdout = cStringIO.StringIO()
890 exited = arvados_cwl.main(
891 ["--create-workflow", "--debug",
892 "--project-uuid", project_uuid,
894 "--name", "testing 123",
895 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
896 capture_stdout, sys.stderr, api_client=stubs.api)
897 self.assertEqual(exited, 0)
899 stubs.api.pipeline_instances().create.refute_called()
900 stubs.api.jobs().create.refute_called()
902 expect_component = copy.deepcopy(stubs.expect_job_spec)
903 self._adjust_script_params(expect_component)
906 "testing 123": expect_component,
908 "name": "testing 123",
909 "owner_uuid": project_uuid,
911 stubs.api.pipeline_templates().create.assert_called_with(
912 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
914 self.assertEqual(capture_stdout.getvalue(),
915 stubs.expect_pipeline_template_uuid + '\n')
919 def test_update_name(self, stubs):
920 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
922 capture_stdout = cStringIO.StringIO()
924 exited = arvados_cwl.main(
925 ["--update-workflow", self.existing_template_uuid,
927 "--project-uuid", project_uuid,
929 "--name", "testing 123",
930 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
931 capture_stdout, sys.stderr, api_client=stubs.api)
932 self.assertEqual(exited, 0)
934 stubs.api.pipeline_instances().create.refute_called()
935 stubs.api.jobs().create.refute_called()
937 expect_component = copy.deepcopy(stubs.expect_job_spec)
938 self._adjust_script_params(expect_component)
941 "testing 123": expect_component,
943 "name": "testing 123",
944 "owner_uuid": project_uuid,
946 stubs.api.pipeline_templates().create.refute_called()
947 stubs.api.pipeline_templates().update.assert_called_with(
948 body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
950 self.assertEqual(capture_stdout.getvalue(),
951 self.existing_template_uuid + '\n')
954 class TestCreateWorkflow(unittest.TestCase):
955 existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
956 expect_workflow = open("tests/wf/expect_packed.cwl").read()
959 def test_create(self, stubs):
960 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
962 capture_stdout = cStringIO.StringIO()
964 exited = arvados_cwl.main(
965 ["--create-workflow", "--debug",
967 "--project-uuid", project_uuid,
968 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
969 capture_stdout, sys.stderr, api_client=stubs.api)
970 self.assertEqual(exited, 0)
972 stubs.api.pipeline_templates().create.refute_called()
973 stubs.api.container_requests().create.refute_called()
977 "owner_uuid": project_uuid,
978 "name": "submit_wf.cwl",
980 "definition": self.expect_workflow,
983 stubs.api.workflows().create.assert_called_with(
984 body=JsonDiffMatcher(body))
986 self.assertEqual(capture_stdout.getvalue(),
987 stubs.expect_workflow_uuid + '\n')
991 def test_create_name(self, stubs):
992 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
994 capture_stdout = cStringIO.StringIO()
996 exited = arvados_cwl.main(
997 ["--create-workflow", "--debug",
999 "--project-uuid", project_uuid,
1000 "--name", "testing 123",
1001 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1002 capture_stdout, sys.stderr, api_client=stubs.api)
1003 self.assertEqual(exited, 0)
1005 stubs.api.pipeline_templates().create.refute_called()
1006 stubs.api.container_requests().create.refute_called()
1010 "owner_uuid": project_uuid,
1011 "name": "testing 123",
1013 "definition": self.expect_workflow,
1016 stubs.api.workflows().create.assert_called_with(
1017 body=JsonDiffMatcher(body))
1019 self.assertEqual(capture_stdout.getvalue(),
1020 stubs.expect_workflow_uuid + '\n')
1023 def test_incompatible_api(self, stubs):
1024 capture_stderr = cStringIO.StringIO()
1025 logging.getLogger('arvados.cwl-runner').addHandler(
1026 logging.StreamHandler(capture_stderr))
1028 exited = arvados_cwl.main(
1029 ["--update-workflow", self.existing_workflow_uuid,
1032 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1033 sys.stderr, sys.stderr, api_client=stubs.api)
1034 self.assertEqual(exited, 1)
1035 self.assertRegexpMatches(
1036 capture_stderr.getvalue(),
1037 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1040 def test_update(self, stubs):
1041 capture_stdout = cStringIO.StringIO()
1043 exited = arvados_cwl.main(
1044 ["--update-workflow", self.existing_workflow_uuid,
1046 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1047 capture_stdout, sys.stderr, api_client=stubs.api)
1048 self.assertEqual(exited, 0)
1052 "name": "submit_wf.cwl",
1054 "definition": self.expect_workflow,
1057 stubs.api.workflows().update.assert_called_with(
1058 uuid=self.existing_workflow_uuid,
1059 body=JsonDiffMatcher(body))
1060 self.assertEqual(capture_stdout.getvalue(),
1061 self.existing_workflow_uuid + '\n')
1065 def test_update_name(self, stubs):
1066 capture_stdout = cStringIO.StringIO()
1068 exited = arvados_cwl.main(
1069 ["--update-workflow", self.existing_workflow_uuid,
1070 "--debug", "--name", "testing 123",
1071 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1072 capture_stdout, sys.stderr, api_client=stubs.api)
1073 self.assertEqual(exited, 0)
1077 "name": "testing 123",
1079 "definition": self.expect_workflow,
1082 stubs.api.workflows().update.assert_called_with(
1083 uuid=self.existing_workflow_uuid,
1084 body=JsonDiffMatcher(body))
1085 self.assertEqual(capture_stdout.getvalue(),
1086 self.existing_workflow_uuid + '\n')
1089 class TestTemplateInputs(unittest.TestCase):
1092 "inputs_test.cwl": {
1093 'runtime_constraints': {
1094 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1095 'min_ram_mb_per_node': 1024
1097 'script_parameters': {
1099 '99999999999999999999999999999991+99/'
1100 'wf/inputs_test.cwl',
1101 'optionalFloatInput': None,
1104 'dataclass': 'File',
1106 'title': "It's a file; we expect to find some characters in it.",
1107 'description': 'If there were anything further to say, it would be said here,\nor here.'
1111 'dataclass': 'number',
1113 'title': 'Floats like a duck',
1117 'optionalFloatInput': {
1118 'type': ['null', 'float'],
1119 'dataclass': 'number',
1124 'dataclass': 'boolean',
1126 'title': 'True or false?',
1129 'repository': 'arvados',
1130 'script_version': 'master',
1131 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1132 'script': 'cwl-runner',
1135 "name": "inputs_test.cwl",
1139 def test_inputs_empty(self, stubs):
1140 exited = arvados_cwl.main(
1141 ["--create-template",
1142 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1143 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1144 self.assertEqual(exited, 0)
1146 stubs.api.pipeline_templates().create.assert_called_with(
1147 body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1150 def test_inputs(self, stubs):
1151 exited = arvados_cwl.main(
1152 ["--create-template",
1153 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1154 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1155 self.assertEqual(exited, 0)
1157 expect_template = copy.deepcopy(self.expect_template)
1158 params = expect_template[
1159 "components"]["inputs_test.cwl"]["script_parameters"]
1160 params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
1161 params["floatInput"]["value"] = 1.234
1162 params["boolInput"]["value"] = True
1164 stubs.api.pipeline_templates().create.assert_called_with(
1165 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)