1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
16 import arvados.collection
18 import arvados_cwl.runner
21 from .matcher import JsonDiffMatcher, StripYAMLComments
22 from .mock_discovery import get_rootDesc
24 import ruamel.yaml as yaml
29 @functools.wraps(func)
30 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
31 @mock.patch("arvados.collection.KeepClient")
32 @mock.patch("arvados.keep.KeepClient")
33 @mock.patch("arvados.events.subscribe")
34 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
39 stubs.keepdocker = keepdocker
41 def putstub(p, **kwargs):
42 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
43 keep_client1().put.side_effect = putstub
44 keep_client1.put.side_effect = putstub
45 keep_client2().put.side_effect = putstub
46 keep_client2.put.side_effect = putstub
48 stubs.keep_client = keep_client2
49 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
50 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
51 stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
53 stubs.api = mock.MagicMock()
54 stubs.api._rootDesc = get_rootDesc()
56 stubs.api.users().current().execute.return_value = {
57 "uuid": stubs.fake_user_uuid,
59 stubs.api.collections().list().execute.return_value = {"items": []}
60 stubs.api.containers().current().execute.return_value = {
61 "uuid": stubs.fake_container_uuid,
64 class CollectionExecute(object):
65 def __init__(self, exe):
67 def execute(self, num_retries=None):
70 def collection_createstub(created_collections, body, ensure_unique_name=None):
71 mt = body["manifest_text"]
72 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
73 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
74 created_collections[uuid] = {
76 "portable_data_hash": pdh,
79 return CollectionExecute(created_collections[uuid])
81 def collection_getstub(created_collections, uuid):
82 for v in created_collections.itervalues():
83 if uuid in (v["uuid"], v["portable_data_hash"]):
84 return CollectionExecute(v)
86 created_collections = {
87 "99999999999999999999999999999998+99": {
89 "portable_data_hash": "99999999999999999999999999999998+99",
90 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
92 "99999999999999999999999999999994+99": {
94 "portable_data_hash": "99999999999999999999999999999994+99",
95 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
98 stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
99 stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
101 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
102 stubs.api.jobs().create().execute.return_value = {
103 "uuid": stubs.expect_job_uuid,
107 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
108 stubs.api.container_requests().create().execute.return_value = {
109 "uuid": stubs.expect_container_request_uuid,
110 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
114 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
115 stubs.api.pipeline_templates().create().execute.return_value = {
116 "uuid": stubs.expect_pipeline_template_uuid,
118 stubs.expect_job_spec = {
119 'runtime_constraints': {
120 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
121 'min_ram_mb_per_node': 1024
123 'script_parameters': {
125 'basename': 'blorp.txt',
126 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
130 'basename': '99999999999999999999999999999998+99',
131 'location': 'keep:99999999999999999999999999999998+99',
135 'basename': 'anonymous',
137 "basename": "renamed.txt",
139 "location": "keep:99999999999999999999999999999998+99/file1.txt",
144 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
146 'repository': 'arvados',
147 'script_version': 'master',
148 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
149 'script': 'cwl-runner'
151 stubs.pipeline_component = stubs.expect_job_spec.copy()
152 stubs.expect_pipeline_instance = {
153 'name': 'submit_wf.cwl',
154 'state': 'RunningOnServer',
158 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
159 'script_parameters': {
160 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
162 'basename': 'blorp.txt',
164 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
167 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
170 'basename': 'renamed.txt',
171 'class': 'File', 'location':
172 'keep:99999999999999999999999999999998+99/file1.txt',
176 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
178 'arv:enable_reuse': True,
179 'arv:on_error': 'continue'
181 'repository': 'arvados',
182 'script_version': 'master',
183 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
184 'script': 'cwl-runner',
185 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
189 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
190 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
191 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
192 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
193 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
194 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
197 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
198 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
200 with open("tests/wf/submit_wf_packed.cwl") as f:
201 expect_packed_workflow = yaml.round_trip_load(f)
203 stubs.expect_container_spec = {
210 '/var/lib/cwl/workflow.json': {
211 'content': expect_packed_workflow,
215 'path': '/var/spool/cwl/cwl.output.json',
218 '/var/lib/cwl/cwl.input.json': {
222 'basename': '99999999999999999999999999999998+99',
223 'location': 'keep:99999999999999999999999999999998+99',
224 'class': 'Directory'},
226 'basename': u'blorp.txt',
228 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
231 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
232 {'basename': 'renamed.txt',
234 'location': 'keep:99999999999999999999999999999998+99/file1.txt',
243 'state': 'Committed',
244 'command': ['arvados-cwl-runner', '--local', '--api=containers',
245 '--no-log-timestamps', '--disable-validate',
246 '--eval-timeout=20', '--thread-count=4',
247 '--enable-reuse', '--debug', '--on-error=continue',
248 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
249 'name': 'submit_wf.cwl',
250 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
251 'output_path': '/var/spool/cwl',
252 'cwd': '/var/spool/cwl',
253 'runtime_constraints': {
256 'ram': 1024*1024*1024
258 'use_existing': True,
263 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
264 stubs.api.workflows().create().execute.return_value = {
265 "uuid": stubs.expect_workflow_uuid,
267 def update_mock(**kwargs):
268 stubs.updated_uuid = kwargs.get('uuid')
270 stubs.api.workflows().update.side_effect = update_mock
271 stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
272 "uuid": stubs.updated_uuid,
275 return func(self, stubs, *args, **kwargs)
279 class TestSubmit(unittest.TestCase):
280 @mock.patch("arvados_cwl.runner.arv_docker_get_image")
281 @mock.patch("time.sleep")
283 def test_submit(self, stubs, tm, arvdock):
284 capture_stdout = cStringIO.StringIO()
285 exited = arvados_cwl.main(
286 ["--submit", "--no-wait", "--api=jobs", "--debug",
287 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
288 capture_stdout, sys.stderr, api_client=stubs.api)
289 self.assertEqual(exited, 0)
291 stubs.api.collections().create.assert_has_calls([
292 mock.call(body=JsonDiffMatcher({
294 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
295 'replication_desired': None,
296 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
297 }), ensure_unique_name=False),
298 mock.call(body=JsonDiffMatcher({
300 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
301 'replication_desired': None,
302 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
303 }), ensure_unique_name=False),
304 mock.call(body=JsonDiffMatcher({
306 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
307 'replication_desired': None,
308 'name': 'submit_wf.cwl',
309 }), ensure_unique_name=True) ])
311 arvdock.assert_has_calls([
312 mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
313 mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
316 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
317 stubs.api.pipeline_instances().create.assert_called_with(
318 body=JsonDiffMatcher(expect_pipeline))
319 self.assertEqual(capture_stdout.getvalue(),
320 stubs.expect_pipeline_uuid + '\n')
323 @mock.patch("time.sleep")
325 def test_submit_no_reuse(self, stubs, tm):
326 capture_stdout = cStringIO.StringIO()
327 exited = arvados_cwl.main(
328 ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
329 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
330 capture_stdout, sys.stderr, api_client=stubs.api)
331 self.assertEqual(exited, 0)
333 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
334 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
335 expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
337 stubs.api.pipeline_instances().create.assert_called_with(
338 body=JsonDiffMatcher(expect_pipeline))
339 self.assertEqual(capture_stdout.getvalue(),
340 stubs.expect_pipeline_uuid + '\n')
343 def test_error_when_multiple_storage_classes_specified(self, stubs):
344 storage_classes = "foo,bar"
345 exited = arvados_cwl.main(
346 ["--debug", "--storage-classes", storage_classes,
347 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
348 sys.stdin, sys.stderr, api_client=stubs.api)
349 self.assertEqual(exited, 1)
351 @mock.patch("time.sleep")
353 def test_submit_on_error(self, stubs, tm):
354 capture_stdout = cStringIO.StringIO()
355 exited = arvados_cwl.main(
356 ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
357 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
358 capture_stdout, sys.stderr, api_client=stubs.api)
359 self.assertEqual(exited, 0)
361 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
362 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
364 stubs.api.pipeline_instances().create.assert_called_with(
365 body=JsonDiffMatcher(expect_pipeline))
366 self.assertEqual(capture_stdout.getvalue(),
367 stubs.expect_pipeline_uuid + '\n')
370 @mock.patch("time.sleep")
372 def test_submit_runner_ram(self, stubs, tm):
373 capture_stdout = cStringIO.StringIO()
374 exited = arvados_cwl.main(
375 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
376 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
377 capture_stdout, sys.stderr, api_client=stubs.api)
378 self.assertEqual(exited, 0)
380 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
381 expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
383 stubs.api.pipeline_instances().create.assert_called_with(
384 body=JsonDiffMatcher(expect_pipeline))
385 self.assertEqual(capture_stdout.getvalue(),
386 stubs.expect_pipeline_uuid + '\n')
389 @mock.patch("time.sleep")
391 def test_submit_invalid_runner_ram(self, stubs, tm):
392 capture_stdout = cStringIO.StringIO()
393 exited = arvados_cwl.main(
394 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
395 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
396 capture_stdout, sys.stderr, api_client=stubs.api)
397 self.assertEqual(exited, 1)
399 @mock.patch("time.sleep")
401 def test_submit_output_name(self, stubs, tm):
402 output_name = "test_output_name"
404 capture_stdout = cStringIO.StringIO()
405 exited = arvados_cwl.main(
406 ["--submit", "--no-wait", "--debug", "--output-name", output_name,
407 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
408 capture_stdout, sys.stderr, api_client=stubs.api)
409 self.assertEqual(exited, 0)
411 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
412 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
414 stubs.api.pipeline_instances().create.assert_called_with(
415 body=JsonDiffMatcher(expect_pipeline))
416 self.assertEqual(capture_stdout.getvalue(),
417 stubs.expect_pipeline_uuid + '\n')
420 @mock.patch("time.sleep")
422 def test_submit_pipeline_name(self, stubs, tm):
423 capture_stdout = cStringIO.StringIO()
424 exited = arvados_cwl.main(
425 ["--submit", "--no-wait", "--debug", "--name=hello job 123",
426 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
427 capture_stdout, sys.stderr, api_client=stubs.api)
428 self.assertEqual(exited, 0)
430 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
431 expect_pipeline["name"] = "hello job 123"
433 stubs.api.pipeline_instances().create.assert_called_with(
434 body=JsonDiffMatcher(expect_pipeline))
435 self.assertEqual(capture_stdout.getvalue(),
436 stubs.expect_pipeline_uuid + '\n')
438 @mock.patch("time.sleep")
440 def test_submit_output_tags(self, stubs, tm):
441 output_tags = "tag0,tag1,tag2"
443 capture_stdout = cStringIO.StringIO()
444 exited = arvados_cwl.main(
445 ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
446 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
447 capture_stdout, sys.stderr, api_client=stubs.api)
448 self.assertEqual(exited, 0)
450 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
451 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
453 stubs.api.pipeline_instances().create.assert_called_with(
454 body=JsonDiffMatcher(expect_pipeline))
455 self.assertEqual(capture_stdout.getvalue(),
456 stubs.expect_pipeline_uuid + '\n')
458 @mock.patch("time.sleep")
460 def test_submit_with_project_uuid(self, stubs, tm):
461 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
463 exited = arvados_cwl.main(
464 ["--submit", "--no-wait", "--debug",
465 "--project-uuid", project_uuid,
466 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
467 sys.stdout, sys.stderr, api_client=stubs.api)
468 self.assertEqual(exited, 0)
470 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
471 expect_pipeline["owner_uuid"] = project_uuid
472 stubs.api.pipeline_instances().create.assert_called_with(
473 body=JsonDiffMatcher(expect_pipeline))
476 def test_submit_container(self, stubs):
477 capture_stdout = cStringIO.StringIO()
479 exited = arvados_cwl.main(
480 ["--submit", "--no-wait", "--api=containers", "--debug",
481 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
482 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
483 self.assertEqual(exited, 0)
485 logging.exception("")
487 stubs.api.collections().create.assert_has_calls([
488 mock.call(body=JsonDiffMatcher({
490 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
491 'replication_desired': None,
492 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
493 }), ensure_unique_name=False),
494 mock.call(body=JsonDiffMatcher({
496 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
497 'replication_desired': None,
498 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
499 }), ensure_unique_name=False)])
501 expect_container = copy.deepcopy(stubs.expect_container_spec)
502 stubs.api.container_requests().create.assert_called_with(
503 body=JsonDiffMatcher(expect_container))
504 self.assertEqual(capture_stdout.getvalue(),
505 stubs.expect_container_request_uuid + '\n')
508 def test_submit_container_no_reuse(self, stubs):
509 capture_stdout = cStringIO.StringIO()
511 exited = arvados_cwl.main(
512 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
513 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
514 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
515 self.assertEqual(exited, 0)
517 logging.exception("")
519 expect_container = copy.deepcopy(stubs.expect_container_spec)
520 expect_container["command"] = [
521 'arvados-cwl-runner', '--local', '--api=containers',
522 '--no-log-timestamps', '--disable-validate',
523 '--eval-timeout=20', '--thread-count=4',
524 '--disable-reuse', '--debug', '--on-error=continue',
525 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
526 expect_container["use_existing"] = False
528 stubs.api.container_requests().create.assert_called_with(
529 body=JsonDiffMatcher(expect_container))
530 self.assertEqual(capture_stdout.getvalue(),
531 stubs.expect_container_request_uuid + '\n')
535 def test_submit_container_reuse_disabled_by_workflow(self, stubs):
536 capture_stdout = cStringIO.StringIO()
538 exited = arvados_cwl.main(
539 ["--submit", "--no-wait", "--api=containers", "--debug",
540 "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
541 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
542 self.assertEqual(exited, 0)
544 expect_container = copy.deepcopy(stubs.expect_container_spec)
545 expect_container["command"] = [
546 'arvados-cwl-runner', '--local', '--api=containers',
547 '--no-log-timestamps', '--disable-validate',
548 '--eval-timeout=20', '--thread-count=4',
549 '--disable-reuse', '--debug', '--on-error=continue',
550 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
551 expect_container["use_existing"] = False
552 expect_container["name"] = "submit_wf_no_reuse.cwl"
553 expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
555 "class": "http://arvados.org/cwl#ReuseRequirement",
556 "enableReuse": False,
559 expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
560 "arv": "http://arvados.org/cwl#",
561 "cwltool": "http://commonwl.org/cwltool#"
564 stubs.api.container_requests().create.assert_called_with(
565 body=JsonDiffMatcher(expect_container))
566 self.assertEqual(capture_stdout.getvalue(),
567 stubs.expect_container_request_uuid + '\n')
571 def test_submit_container_on_error(self, stubs):
572 capture_stdout = cStringIO.StringIO()
574 exited = arvados_cwl.main(
575 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
576 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
577 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
578 self.assertEqual(exited, 0)
580 logging.exception("")
582 expect_container = copy.deepcopy(stubs.expect_container_spec)
583 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
584 '--no-log-timestamps', '--disable-validate',
585 '--eval-timeout=20', '--thread-count=4',
586 '--enable-reuse', '--debug', '--on-error=stop',
587 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
589 stubs.api.container_requests().create.assert_called_with(
590 body=JsonDiffMatcher(expect_container))
591 self.assertEqual(capture_stdout.getvalue(),
592 stubs.expect_container_request_uuid + '\n')
595 def test_submit_container_output_name(self, stubs):
596 output_name = "test_output_name"
598 capture_stdout = cStringIO.StringIO()
600 exited = arvados_cwl.main(
601 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
602 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
603 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
604 self.assertEqual(exited, 0)
606 logging.exception("")
608 expect_container = copy.deepcopy(stubs.expect_container_spec)
609 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
610 '--no-log-timestamps', '--disable-validate',
611 '--eval-timeout=20', '--thread-count=4',
613 "--output-name="+output_name, '--debug', '--on-error=continue',
614 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
615 expect_container["output_name"] = output_name
617 stubs.api.container_requests().create.assert_called_with(
618 body=JsonDiffMatcher(expect_container))
619 self.assertEqual(capture_stdout.getvalue(),
620 stubs.expect_container_request_uuid + '\n')
623 def test_submit_storage_classes(self, stubs):
624 capture_stdout = cStringIO.StringIO()
626 exited = arvados_cwl.main(
627 ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
628 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
629 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
630 self.assertEqual(exited, 0)
632 logging.exception("")
634 expect_container = copy.deepcopy(stubs.expect_container_spec)
635 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
636 '--no-log-timestamps', '--disable-validate',
637 '--eval-timeout=20', '--thread-count=4',
638 '--enable-reuse', "--debug",
639 "--storage-classes=foo", '--on-error=continue',
640 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
642 stubs.api.container_requests().create.assert_called_with(
643 body=JsonDiffMatcher(expect_container))
644 self.assertEqual(capture_stdout.getvalue(),
645 stubs.expect_container_request_uuid + '\n')
647 @mock.patch("arvados_cwl.task_queue.TaskQueue")
648 @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
649 @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
651 def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
652 def set_final_output(job_order, output_callback, runtimeContext):
653 output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
655 job.side_effect = set_final_output
658 exited = arvados_cwl.main(
659 ["--debug", "--local", "--storage-classes=foo",
660 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
661 sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
662 self.assertEqual(exited, 0)
664 logging.exception("")
666 make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
668 @mock.patch("arvados_cwl.task_queue.TaskQueue")
669 @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
670 @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
672 def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
673 def set_final_output(job_order, output_callback, runtimeContext):
674 output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
676 job.side_effect = set_final_output
679 exited = arvados_cwl.main(
680 ["--debug", "--local",
681 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
682 sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
683 self.assertEqual(exited, 0)
685 logging.exception("")
687 make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
690 def test_submit_container_output_ttl(self, stubs):
691 capture_stdout = cStringIO.StringIO()
693 exited = arvados_cwl.main(
694 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
695 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
696 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
697 self.assertEqual(exited, 0)
699 logging.exception("")
701 expect_container = copy.deepcopy(stubs.expect_container_spec)
702 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
703 '--no-log-timestamps', '--disable-validate',
704 '--eval-timeout=20', '--thread-count=4',
705 '--enable-reuse', '--debug', '--on-error=continue',
706 "--intermediate-output-ttl=3600",
707 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
709 stubs.api.container_requests().create.assert_called_with(
710 body=JsonDiffMatcher(expect_container))
711 self.assertEqual(capture_stdout.getvalue(),
712 stubs.expect_container_request_uuid + '\n')
715 def test_submit_container_trash_intermediate(self, stubs):
716 capture_stdout = cStringIO.StringIO()
718 exited = arvados_cwl.main(
719 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
720 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
721 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
722 self.assertEqual(exited, 0)
724 logging.exception("")
726 expect_container = copy.deepcopy(stubs.expect_container_spec)
727 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
728 '--no-log-timestamps', '--disable-validate',
729 '--eval-timeout=20', '--thread-count=4',
730 '--enable-reuse', '--debug', '--on-error=continue',
731 "--trash-intermediate",
732 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
734 stubs.api.container_requests().create.assert_called_with(
735 body=JsonDiffMatcher(expect_container))
736 self.assertEqual(capture_stdout.getvalue(),
737 stubs.expect_container_request_uuid + '\n')
740 def test_submit_container_output_tags(self, stubs):
741 output_tags = "tag0,tag1,tag2"
743 capture_stdout = cStringIO.StringIO()
745 exited = arvados_cwl.main(
746 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
747 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
748 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
749 self.assertEqual(exited, 0)
751 logging.exception("")
753 expect_container = copy.deepcopy(stubs.expect_container_spec)
754 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
755 '--no-log-timestamps', '--disable-validate',
756 '--eval-timeout=20', '--thread-count=4',
758 "--output-tags="+output_tags, '--debug', '--on-error=continue',
759 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
761 stubs.api.container_requests().create.assert_called_with(
762 body=JsonDiffMatcher(expect_container))
763 self.assertEqual(capture_stdout.getvalue(),
764 stubs.expect_container_request_uuid + '\n')
767 def test_submit_container_runner_ram(self, stubs):
768 capture_stdout = cStringIO.StringIO()
770 exited = arvados_cwl.main(
771 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
772 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
773 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
774 self.assertEqual(exited, 0)
776 logging.exception("")
778 expect_container = copy.deepcopy(stubs.expect_container_spec)
779 expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
781 stubs.api.container_requests().create.assert_called_with(
782 body=JsonDiffMatcher(expect_container))
783 self.assertEqual(capture_stdout.getvalue(),
784 stubs.expect_container_request_uuid + '\n')
786 @mock.patch("arvados.collection.CollectionReader")
787 @mock.patch("time.sleep")
789 def test_submit_file_keepref(self, stubs, tm, collectionReader):
790 capture_stdout = cStringIO.StringIO()
791 collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
792 exited = arvados_cwl.main(
793 ["--submit", "--no-wait", "--api=containers", "--debug",
794 "tests/wf/submit_keepref_wf.cwl"],
795 capture_stdout, sys.stderr, api_client=stubs.api)
796 self.assertEqual(exited, 0)
799 @mock.patch("arvados.collection.CollectionReader")
800 @mock.patch("time.sleep")
802 def test_submit_keepref(self, stubs, tm, reader):
803 capture_stdout = cStringIO.StringIO()
805 with open("tests/wf/expect_arvworkflow.cwl") as f:
806 reader().open().__enter__().read.return_value = f.read()
808 exited = arvados_cwl.main(
809 ["--submit", "--no-wait", "--api=containers", "--debug",
810 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
811 capture_stdout, sys.stderr, api_client=stubs.api)
812 self.assertEqual(exited, 0)
822 'path': '/var/spool/cwl/cwl.output.json',
825 '/var/lib/cwl/workflow': {
826 'portable_data_hash': '99999999999999999999999999999994+99',
829 '/var/lib/cwl/cwl.input.json': {
835 }, 'state': 'Committed',
836 'output_path': '/var/spool/cwl',
837 'name': 'expect_arvworkflow.cwl#main',
838 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
839 'command': ['arvados-cwl-runner', '--local', '--api=containers',
840 '--no-log-timestamps', '--disable-validate',
841 '--eval-timeout=20', '--thread-count=4',
842 '--enable-reuse', '--debug', '--on-error=continue',
843 '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
844 'cwd': '/var/spool/cwl',
845 'runtime_constraints': {
850 'use_existing': True,
855 stubs.api.container_requests().create.assert_called_with(
856 body=JsonDiffMatcher(expect_container))
857 self.assertEqual(capture_stdout.getvalue(),
858 stubs.expect_container_request_uuid + '\n')
861 @mock.patch("arvados.collection.CollectionReader")
862 @mock.patch("time.sleep")
864 def test_submit_jobs_keepref(self, stubs, tm, reader):
865 capture_stdout = cStringIO.StringIO()
867 with open("tests/wf/expect_arvworkflow.cwl") as f:
868 reader().open().__enter__().read.return_value = f.read()
870 exited = arvados_cwl.main(
871 ["--submit", "--no-wait", "--api=jobs", "--debug",
872 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
873 capture_stdout, sys.stderr, api_client=stubs.api)
874 self.assertEqual(exited, 0)
876 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
877 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
878 del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
879 del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
880 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
881 expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
882 stubs.api.pipeline_instances().create.assert_called_with(
883 body=JsonDiffMatcher(expect_pipeline))
885 @mock.patch("time.sleep")
887 def test_submit_arvworkflow(self, stubs, tm):
888 capture_stdout = cStringIO.StringIO()
890 with open("tests/wf/expect_arvworkflow.cwl") as f:
891 stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
893 exited = arvados_cwl.main(
894 ["--submit", "--no-wait", "--api=containers", "--debug",
895 "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
896 capture_stdout, sys.stderr, api_client=stubs.api)
897 self.assertEqual(exited, 0)
907 'path': '/var/spool/cwl/cwl.output.json',
910 '/var/lib/cwl/workflow.json': {
913 'cwlVersion': 'v1.0',
918 {'type': 'string', 'id': '#main/x'}
921 {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
922 'run': '#submit_tool.cwl',
932 'inputBinding': {'position': 1},
934 'id': '#submit_tool.cwl/x'}
937 {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
939 'id': '#submit_tool.cwl',
941 'baseCommand': 'cat',
942 'class': 'CommandLineTool'
947 '/var/lib/cwl/cwl.input.json': {
953 }, 'state': 'Committed',
954 'output_path': '/var/spool/cwl',
955 'name': 'a test workflow',
956 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
957 'command': ['arvados-cwl-runner', '--local', '--api=containers',
958 '--no-log-timestamps', '--disable-validate',
959 '--eval-timeout=20', '--thread-count=4',
960 '--enable-reuse', '--debug', '--on-error=continue',
961 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
962 'cwd': '/var/spool/cwl',
963 'runtime_constraints': {
968 'use_existing': True,
970 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
975 stubs.api.container_requests().create.assert_called_with(
976 body=JsonDiffMatcher(expect_container))
977 self.assertEqual(capture_stdout.getvalue(),
978 stubs.expect_container_request_uuid + '\n')
982 def test_submit_container_name(self, stubs):
983 capture_stdout = cStringIO.StringIO()
985 exited = arvados_cwl.main(
986 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
987 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
988 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
989 self.assertEqual(exited, 0)
991 logging.exception("")
993 expect_container = copy.deepcopy(stubs.expect_container_spec)
994 expect_container["name"] = "hello container 123"
996 stubs.api.container_requests().create.assert_called_with(
997 body=JsonDiffMatcher(expect_container))
998 self.assertEqual(capture_stdout.getvalue(),
999 stubs.expect_container_request_uuid + '\n')
1003 def test_submit_container_project(self, stubs):
1004 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1005 capture_stdout = cStringIO.StringIO()
1007 exited = arvados_cwl.main(
1008 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1009 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1010 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1011 self.assertEqual(exited, 0)
1013 logging.exception("")
1015 expect_container = copy.deepcopy(stubs.expect_container_spec)
1016 expect_container["owner_uuid"] = project_uuid
1017 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1018 '--no-log-timestamps', '--disable-validate',
1019 "--eval-timeout=20", "--thread-count=4",
1020 '--enable-reuse', '--debug', '--on-error=continue',
1021 '--project-uuid='+project_uuid,
1022 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1024 stubs.api.container_requests().create.assert_called_with(
1025 body=JsonDiffMatcher(expect_container))
1026 self.assertEqual(capture_stdout.getvalue(),
1027 stubs.expect_container_request_uuid + '\n')
1030 def test_submit_container_eval_timeout(self, stubs):
1031 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1032 capture_stdout = cStringIO.StringIO()
1034 exited = arvados_cwl.main(
1035 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1036 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1037 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1038 self.assertEqual(exited, 0)
1040 logging.exception("")
1042 expect_container = copy.deepcopy(stubs.expect_container_spec)
1043 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1044 '--no-log-timestamps', '--disable-validate',
1045 '--eval-timeout=60.0', '--thread-count=4',
1046 '--enable-reuse', '--debug', '--on-error=continue',
1047 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1049 stubs.api.container_requests().create.assert_called_with(
1050 body=JsonDiffMatcher(expect_container))
1051 self.assertEqual(capture_stdout.getvalue(),
1052 stubs.expect_container_request_uuid + '\n')
1056 def test_submit_container_thread_count(self, stubs):
1057 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1058 capture_stdout = cStringIO.StringIO()
1060 exited = arvados_cwl.main(
1061 ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1062 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1063 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1064 self.assertEqual(exited, 0)
1066 logging.exception("")
1068 expect_container = copy.deepcopy(stubs.expect_container_spec)
1069 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1070 '--no-log-timestamps', '--disable-validate',
1071 '--eval-timeout=20', '--thread-count=20',
1072 '--enable-reuse', '--debug', '--on-error=continue',
1073 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1075 stubs.api.container_requests().create.assert_called_with(
1076 body=JsonDiffMatcher(expect_container))
1077 self.assertEqual(capture_stdout.getvalue(),
1078 stubs.expect_container_request_uuid + '\n')
1082 def test_submit_job_runner_image(self, stubs):
1083 capture_stdout = cStringIO.StringIO()
1085 exited = arvados_cwl.main(
1086 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1087 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1088 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1089 self.assertEqual(exited, 0)
1091 logging.exception("")
1093 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
1095 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1096 stubs.api.pipeline_instances().create.assert_called_with(
1097 body=JsonDiffMatcher(expect_pipeline))
1098 self.assertEqual(capture_stdout.getvalue(),
1099 stubs.expect_pipeline_uuid + '\n')
1102 def test_submit_container_runner_image(self, stubs):
1103 capture_stdout = cStringIO.StringIO()
1105 exited = arvados_cwl.main(
1106 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1107 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1108 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1109 self.assertEqual(exited, 0)
1111 logging.exception("")
1113 stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
1115 expect_container = copy.deepcopy(stubs.expect_container_spec)
1116 stubs.api.container_requests().create.assert_called_with(
1117 body=JsonDiffMatcher(expect_container))
1118 self.assertEqual(capture_stdout.getvalue(),
1119 stubs.expect_container_request_uuid + '\n')
1122 def test_submit_priority(self, stubs):
1123 capture_stdout = cStringIO.StringIO()
1125 exited = arvados_cwl.main(
1126 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1127 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1128 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1129 self.assertEqual(exited, 0)
1131 logging.exception("")
1133 stubs.expect_container_spec["priority"] = 669
1135 expect_container = copy.deepcopy(stubs.expect_container_spec)
1136 stubs.api.container_requests().create.assert_called_with(
1137 body=JsonDiffMatcher(expect_container))
1138 self.assertEqual(capture_stdout.getvalue(),
1139 stubs.expect_container_request_uuid + '\n')
1143 def test_submit_wf_runner_resources(self, stubs):
1144 capture_stdout = cStringIO.StringIO()
1146 exited = arvados_cwl.main(
1147 ["--submit", "--no-wait", "--api=containers", "--debug",
1148 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1149 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1150 self.assertEqual(exited, 0)
1152 logging.exception("")
1154 expect_container = copy.deepcopy(stubs.expect_container_spec)
1155 expect_container["runtime_constraints"] = {
1160 expect_container["name"] = "submit_wf_runner_resources.cwl"
1161 expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1163 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1168 expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1169 "arv": "http://arvados.org/cwl#",
1172 stubs.api.container_requests().create.assert_called_with(
1173 body=JsonDiffMatcher(expect_container))
1174 self.assertEqual(capture_stdout.getvalue(),
1175 stubs.expect_container_request_uuid + '\n')
1178 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1179 @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1180 @mock.patch("arvados.api")
1181 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1182 arvrunner = mock.MagicMock()
1183 arvrunner.project_uuid = ""
1184 api.return_value = mock.MagicMock()
1185 arvrunner.api = api.return_value
1186 arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1187 "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1188 "link_class": "docker_image_repo+tag",
1189 "name": "arvados/jobs:"+arvados_cwl.__version__,
1191 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1192 {"items": [{"created_at": "",
1194 "link_class": "docker_image_hash",
1197 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1199 find_one_image_hash.return_value = "123456"
1201 arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1203 "manifest_text": "",
1205 }], "items_available": 1, "offset": 0},)
1206 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1207 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1208 arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1211 def test_submit_secrets(self, stubs):
1212 capture_stdout = cStringIO.StringIO()
1214 exited = arvados_cwl.main(
1215 ["--submit", "--no-wait", "--api=containers", "--debug",
1216 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1217 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1218 self.assertEqual(exited, 0)
1220 logging.exception("")
1223 expect_container = {
1225 "arvados-cwl-runner",
1228 "--no-log-timestamps",
1229 "--disable-validate",
1230 "--eval-timeout=20",
1234 "--on-error=continue",
1235 "/var/lib/cwl/workflow.json#main",
1236 "/var/lib/cwl/cwl.input.json"
1238 "container_image": "arvados/jobs:"+arvados_cwl.__version__,
1239 "cwd": "/var/spool/cwl",
1241 "/var/lib/cwl/cwl.input.json": {
1244 "$include": "/secrets/s0"
1249 "/var/lib/cwl/workflow.json": {
1254 "cwltool": "http://commonwl.org/cwltool#"
1260 "class": "CommandLineTool",
1263 "class": "http://commonwl.org/cwltool#Secrets",
1265 "#secret_job.cwl/pw"
1269 "id": "#secret_job.cwl",
1272 "id": "#secret_job.cwl/pw",
1278 "id": "#secret_job.cwl/out",
1282 "stdout": "hashed_example.txt",
1285 "class": "InitialWorkDirRequirement",
1288 "entry": "username: user\npassword: $(inputs.pw)\n",
1289 "entryname": "example.conf"
1296 "class": "Workflow",
1299 "class": "DockerRequirement",
1300 "dockerPull": "debian:8"
1303 "class": "http://commonwl.org/cwltool#Secrets",
1319 "outputSource": "#main/step1/out",
1325 "id": "#main/step1",
1328 "id": "#main/step1/pw",
1329 "source": "#main/pw"
1335 "run": "#secret_job.cwl"
1340 "cwlVersion": "v1.0"
1345 "kind": "collection",
1350 "path": "/var/spool/cwl/cwl.output.json"
1353 "name": "secret_wf.cwl",
1354 "output_path": "/var/spool/cwl",
1357 "runtime_constraints": {
1368 "state": "Committed",
1369 "use_existing": True
1372 stubs.api.container_requests().create.assert_called_with(
1373 body=JsonDiffMatcher(expect_container))
1374 self.assertEqual(capture_stdout.getvalue(),
1375 stubs.expect_container_request_uuid + '\n')
1378 def test_submit_request_uuid(self, stubs):
1379 stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1381 stubs.api.container_requests().update().execute.return_value = {
1382 "uuid": stubs.expect_container_request_uuid,
1383 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1387 capture_stdout = cStringIO.StringIO()
1389 exited = arvados_cwl.main(
1390 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1391 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1392 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1393 self.assertEqual(exited, 0)
1395 logging.exception("")
1397 stubs.api.container_requests().update.assert_called_with(
1398 uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1399 self.assertEqual(capture_stdout.getvalue(),
1400 stubs.expect_container_request_uuid + '\n')
1403 class TestCreateTemplate(unittest.TestCase):
1404 existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1406 def _adjust_script_params(self, expect_component):
1407 expect_component['script_parameters']['x'] = {
1408 'dataclass': 'File',
1411 'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1413 expect_component['script_parameters']['y'] = {
1414 'dataclass': 'Collection',
1416 'type': 'Directory',
1417 'value': '99999999999999999999999999999998+99',
1419 expect_component['script_parameters']['z'] = {
1420 'dataclass': 'Collection',
1422 'type': 'Directory',
1426 def test_create(self, stubs):
1427 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1429 capture_stdout = cStringIO.StringIO()
1431 exited = arvados_cwl.main(
1432 ["--create-workflow", "--debug",
1434 "--project-uuid", project_uuid,
1435 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1436 capture_stdout, sys.stderr, api_client=stubs.api)
1437 self.assertEqual(exited, 0)
1439 stubs.api.pipeline_instances().create.refute_called()
1440 stubs.api.jobs().create.refute_called()
1442 expect_component = copy.deepcopy(stubs.expect_job_spec)
1443 self._adjust_script_params(expect_component)
1446 "submit_wf.cwl": expect_component,
1448 "name": "submit_wf.cwl",
1449 "owner_uuid": project_uuid,
1451 stubs.api.pipeline_templates().create.assert_called_with(
1452 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1454 self.assertEqual(capture_stdout.getvalue(),
1455 stubs.expect_pipeline_template_uuid + '\n')
1459 def test_create_name(self, stubs):
1460 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1462 capture_stdout = cStringIO.StringIO()
1464 exited = arvados_cwl.main(
1465 ["--create-workflow", "--debug",
1466 "--project-uuid", project_uuid,
1468 "--name", "testing 123",
1469 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1470 capture_stdout, sys.stderr, api_client=stubs.api)
1471 self.assertEqual(exited, 0)
1473 stubs.api.pipeline_instances().create.refute_called()
1474 stubs.api.jobs().create.refute_called()
1476 expect_component = copy.deepcopy(stubs.expect_job_spec)
1477 self._adjust_script_params(expect_component)
1480 "testing 123": expect_component,
1482 "name": "testing 123",
1483 "owner_uuid": project_uuid,
1485 stubs.api.pipeline_templates().create.assert_called_with(
1486 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1488 self.assertEqual(capture_stdout.getvalue(),
1489 stubs.expect_pipeline_template_uuid + '\n')
1493 def test_update_name(self, stubs):
1494 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1496 capture_stdout = cStringIO.StringIO()
1498 exited = arvados_cwl.main(
1499 ["--update-workflow", self.existing_template_uuid,
1501 "--project-uuid", project_uuid,
1503 "--name", "testing 123",
1504 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1505 capture_stdout, sys.stderr, api_client=stubs.api)
1506 self.assertEqual(exited, 0)
1508 stubs.api.pipeline_instances().create.refute_called()
1509 stubs.api.jobs().create.refute_called()
1511 expect_component = copy.deepcopy(stubs.expect_job_spec)
1512 self._adjust_script_params(expect_component)
1515 "testing 123": expect_component,
1517 "name": "testing 123",
1518 "owner_uuid": project_uuid,
1520 stubs.api.pipeline_templates().create.refute_called()
1521 stubs.api.pipeline_templates().update.assert_called_with(
1522 body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1524 self.assertEqual(capture_stdout.getvalue(),
1525 self.existing_template_uuid + '\n')
1528 class TestCreateWorkflow(unittest.TestCase):
1529 existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1530 expect_workflow = StripYAMLComments(
1531 open("tests/wf/expect_packed.cwl").read())
1534 def test_create(self, stubs):
1535 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1537 capture_stdout = cStringIO.StringIO()
1539 exited = arvados_cwl.main(
1540 ["--create-workflow", "--debug",
1542 "--project-uuid", project_uuid,
1543 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1544 capture_stdout, sys.stderr, api_client=stubs.api)
1545 self.assertEqual(exited, 0)
1547 stubs.api.pipeline_templates().create.refute_called()
1548 stubs.api.container_requests().create.refute_called()
1552 "owner_uuid": project_uuid,
1553 "name": "submit_wf.cwl",
1555 "definition": self.expect_workflow,
1558 stubs.api.workflows().create.assert_called_with(
1559 body=JsonDiffMatcher(body))
1561 self.assertEqual(capture_stdout.getvalue(),
1562 stubs.expect_workflow_uuid + '\n')
1566 def test_create_name(self, stubs):
1567 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1569 capture_stdout = cStringIO.StringIO()
1571 exited = arvados_cwl.main(
1572 ["--create-workflow", "--debug",
1574 "--project-uuid", project_uuid,
1575 "--name", "testing 123",
1576 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1577 capture_stdout, sys.stderr, api_client=stubs.api)
1578 self.assertEqual(exited, 0)
1580 stubs.api.pipeline_templates().create.refute_called()
1581 stubs.api.container_requests().create.refute_called()
1585 "owner_uuid": project_uuid,
1586 "name": "testing 123",
1588 "definition": self.expect_workflow,
1591 stubs.api.workflows().create.assert_called_with(
1592 body=JsonDiffMatcher(body))
1594 self.assertEqual(capture_stdout.getvalue(),
1595 stubs.expect_workflow_uuid + '\n')
1598 def test_incompatible_api(self, stubs):
1599 capture_stderr = cStringIO.StringIO()
1600 logging.getLogger('arvados.cwl-runner').addHandler(
1601 logging.StreamHandler(capture_stderr))
1603 exited = arvados_cwl.main(
1604 ["--update-workflow", self.existing_workflow_uuid,
1607 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1608 sys.stderr, sys.stderr, api_client=stubs.api)
1609 self.assertEqual(exited, 1)
1610 self.assertRegexpMatches(
1611 capture_stderr.getvalue(),
1612 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1615 def test_update(self, stubs):
1616 capture_stdout = cStringIO.StringIO()
1618 exited = arvados_cwl.main(
1619 ["--update-workflow", self.existing_workflow_uuid,
1621 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1622 capture_stdout, sys.stderr, api_client=stubs.api)
1623 self.assertEqual(exited, 0)
1627 "name": "submit_wf.cwl",
1629 "definition": self.expect_workflow,
1632 stubs.api.workflows().update.assert_called_with(
1633 uuid=self.existing_workflow_uuid,
1634 body=JsonDiffMatcher(body))
1635 self.assertEqual(capture_stdout.getvalue(),
1636 self.existing_workflow_uuid + '\n')
1640 def test_update_name(self, stubs):
1641 capture_stdout = cStringIO.StringIO()
1643 exited = arvados_cwl.main(
1644 ["--update-workflow", self.existing_workflow_uuid,
1645 "--debug", "--name", "testing 123",
1646 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1647 capture_stdout, sys.stderr, api_client=stubs.api)
1648 self.assertEqual(exited, 0)
1652 "name": "testing 123",
1654 "definition": self.expect_workflow,
1657 stubs.api.workflows().update.assert_called_with(
1658 uuid=self.existing_workflow_uuid,
1659 body=JsonDiffMatcher(body))
1660 self.assertEqual(capture_stdout.getvalue(),
1661 self.existing_workflow_uuid + '\n')
1665 def test_create_collection_per_tool(self, stubs):
1666 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1668 capture_stdout = cStringIO.StringIO()
1670 exited = arvados_cwl.main(
1671 ["--create-workflow", "--debug",
1673 "--project-uuid", project_uuid,
1674 "tests/collection_per_tool/collection_per_tool.cwl"],
1675 capture_stdout, sys.stderr, api_client=stubs.api)
1676 self.assertEqual(exited, 0)
1678 toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1679 expect_workflow = StripYAMLComments(open(toolfile).read())
1683 "owner_uuid": project_uuid,
1684 "name": "collection_per_tool.cwl",
1686 "definition": expect_workflow,
1689 stubs.api.workflows().create.assert_called_with(
1690 body=JsonDiffMatcher(body))
1692 self.assertEqual(capture_stdout.getvalue(),
1693 stubs.expect_workflow_uuid + '\n')
1695 class TestTemplateInputs(unittest.TestCase):
1698 "inputs_test.cwl": {
1699 'runtime_constraints': {
1700 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1701 'min_ram_mb_per_node': 1024
1703 'script_parameters': {
1705 '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1706 'optionalFloatInput': None,
1709 'dataclass': 'File',
1711 'title': "It's a file; we expect to find some characters in it.",
1712 'description': 'If there were anything further to say, it would be said here,\nor here.'
1716 'dataclass': 'number',
1718 'title': 'Floats like a duck',
1722 'optionalFloatInput': {
1723 'type': ['null', 'float'],
1724 'dataclass': 'number',
1729 'dataclass': 'boolean',
1731 'title': 'True or false?',
1734 'repository': 'arvados',
1735 'script_version': 'master',
1736 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1737 'script': 'cwl-runner',
1740 "name": "inputs_test.cwl",
1744 def test_inputs_empty(self, stubs):
1745 exited = arvados_cwl.main(
1746 ["--create-template",
1747 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1748 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1749 self.assertEqual(exited, 0)
1751 stubs.api.pipeline_templates().create.assert_called_with(
1752 body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1755 def test_inputs(self, stubs):
1756 exited = arvados_cwl.main(
1757 ["--create-template",
1758 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1759 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1760 self.assertEqual(exited, 0)
1762 expect_template = copy.deepcopy(self.expect_template)
1763 params = expect_template[
1764 "components"]["inputs_test.cwl"]["script_parameters"]
1765 params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1766 params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1767 params["floatInput"]["value"] = 1.234
1768 params["boolInput"]["value"] = True
1770 stubs.api.pipeline_templates().create.assert_called_with(
1771 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)