1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
6 import arvados_cwl.context
7 from arvados_cwl.arvdocker import arv_docker_clear_cache
14 import cwltool.process
15 import cwltool.secrets
16 from schema_salad.ref_resolver import Loader
17 from schema_salad.sourceline import cmap
19 from .matcher import JsonDiffMatcher
21 if not os.getenv('ARVADOS_DEBUG'):
22 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
23 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
25 class CollectionMock(object):
26 def __init__(self, vwdmock, *args, **kwargs):
27 self.vwdmock = vwdmock
30 def open(self, *args, **kwargs):
32 return self.vwdmock.open(*args, **kwargs)
34 def copy(self, *args, **kwargs):
36 self.vwdmock.copy(*args, **kwargs)
38 def save_new(self, *args, **kwargs):
44 def portable_data_hash(self):
46 return arvados.config.EMPTY_BLOCK_LOCATOR
48 return "99999999999999999999999999999996+99"
51 class TestContainer(unittest.TestCase):
53 def helper(self, runner, enable_reuse=True):
54 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
56 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
57 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
58 loadingContext = arvados_cwl.context.ArvLoadingContext(
59 {"avsc_names": avsc_names,
61 "make_fs_access": make_fs_access,
63 "metadata": {"cwlVersion": "v1.0"}})
64 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
65 {"work_api": "containers",
67 "name": "test_run_"+str(enable_reuse),
68 "make_fs_access": make_fs_access,
70 "enable_reuse": enable_reuse,
73 return loadingContext, runtimeContext
75 # The test passes no builder.resources
76 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
77 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
78 def test_run(self, keepdocker):
79 for enable_reuse in (True, False):
80 arv_docker_clear_cache()
82 runner = mock.MagicMock()
83 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
84 runner.ignore_docker_for_reuse = False
85 runner.intermediate_output_ttl = 0
86 runner.secret_store = cwltool.secrets.SecretStore()
88 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
89 runner.api.collections().get().execute.return_value = {
90 "portable_data_hash": "99999999999999999999999999999993+99"}
96 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
98 "class": "CommandLineTool"
101 loadingContext, runtimeContext = self.helper(runner, enable_reuse)
103 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
104 arvtool.formatgraph = None
106 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
107 j.run(runtimeContext)
108 runner.api.container_requests().create.assert_called_with(
109 body=JsonDiffMatcher({
111 'HOME': '/var/spool/cwl',
114 'name': 'test_run_'+str(enable_reuse),
115 'runtime_constraints': {
119 'use_existing': enable_reuse,
122 '/tmp': {'kind': 'tmp',
123 "capacity": 1073741824
125 '/var/spool/cwl': {'kind': 'tmp',
126 "capacity": 1073741824 }
128 'state': 'Committed',
129 'output_name': 'Output for step test_run_'+str(enable_reuse),
130 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
131 'output_path': '/var/spool/cwl',
133 'container_image': 'arvados/jobs',
134 'command': ['ls', '/var/spool/cwl'],
135 'cwd': '/var/spool/cwl',
136 'scheduling_parameters': {},
141 # The test passes some fields in builder.resources
142 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
143 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
144 def test_resource_requirements(self, keepdocker):
145 arv_docker_clear_cache()
146 runner = mock.MagicMock()
147 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
148 runner.ignore_docker_for_reuse = False
149 runner.intermediate_output_ttl = 3600
150 runner.secret_store = cwltool.secrets.SecretStore()
152 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
153 runner.api.collections().get().execute.return_value = {
154 "portable_data_hash": "99999999999999999999999999999993+99"}
160 "class": "ResourceRequirement",
166 "class": "http://arvados.org/cwl#RuntimeConstraints",
169 "class": "http://arvados.org/cwl#APIRequirement",
171 "class": "http://arvados.org/cwl#PartitionRequirement",
174 "class": "http://arvados.org/cwl#IntermediateOutput",
177 "class": "http://arvados.org/cwl#ReuseRequirement",
182 "class": "CommandLineTool"
185 loadingContext, runtimeContext = self.helper(runner)
186 runtimeContext.name = "test_resource_requirements"
188 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
189 arvtool.formatgraph = None
190 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
191 j.run(runtimeContext)
193 call_args, call_kwargs = runner.api.container_requests().create.call_args
195 call_body_expected = {
197 'HOME': '/var/spool/cwl',
200 'name': 'test_resource_requirements',
201 'runtime_constraints': {
204 'keep_cache_ram': 536870912,
207 'use_existing': False,
210 '/tmp': {'kind': 'tmp',
211 "capacity": 4194304000 },
212 '/var/spool/cwl': {'kind': 'tmp',
213 "capacity": 5242880000 }
215 'state': 'Committed',
216 'output_name': 'Output for step test_resource_requirements',
217 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
218 'output_path': '/var/spool/cwl',
220 'container_image': 'arvados/jobs',
222 'cwd': '/var/spool/cwl',
223 'scheduling_parameters': {
224 'partitions': ['blurb']
230 call_body = call_kwargs.get('body', None)
231 self.assertNotEqual(None, call_body)
232 for key in call_body:
233 self.assertEqual(call_body_expected.get(key), call_body.get(key))
236 # The test passes some fields in builder.resources
237 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
238 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
239 @mock.patch("arvados.collection.Collection")
240 def test_initial_work_dir(self, collection_mock, keepdocker):
241 arv_docker_clear_cache()
242 runner = mock.MagicMock()
243 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
244 runner.ignore_docker_for_reuse = False
245 runner.intermediate_output_ttl = 0
246 runner.secret_store = cwltool.secrets.SecretStore()
248 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
249 runner.api.collections().get().execute.return_value = {
250 "portable_data_hash": "99999999999999999999999999999993+99"}
252 sourcemock = mock.MagicMock()
253 def get_collection_mock(p):
255 return (sourcemock, p.split("/", 1)[1])
257 return (sourcemock, "")
258 runner.fs_access.get_collection.side_effect = get_collection_mock
260 vwdmock = mock.MagicMock()
261 collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
267 "class": "InitialWorkDirRequirement",
271 "location": "keep:99999999999999999999999999999995+99/bar"
274 "class": "Directory",
276 "location": "keep:99999999999999999999999999999995+99"
280 "basename": "filename",
281 "location": "keep:99999999999999999999999999999995+99/baz/filename"
284 "class": "Directory",
285 "basename": "subdir",
286 "location": "keep:99999999999999999999999999999995+99/subdir"
291 "class": "CommandLineTool"
294 loadingContext, runtimeContext = self.helper(runner)
295 runtimeContext.name = "test_initial_work_dir"
297 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
298 arvtool.formatgraph = None
299 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
300 j.run(runtimeContext)
302 call_args, call_kwargs = runner.api.container_requests().create.call_args
304 vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
305 vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
306 vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
307 vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
309 call_body_expected = {
311 'HOME': '/var/spool/cwl',
314 'name': 'test_initial_work_dir',
315 'runtime_constraints': {
319 'use_existing': True,
322 '/tmp': {'kind': 'tmp',
323 "capacity": 1073741824 },
324 '/var/spool/cwl': {'kind': 'tmp',
325 "capacity": 1073741824 },
326 '/var/spool/cwl/foo': {
327 'kind': 'collection',
329 'portable_data_hash': '99999999999999999999999999999996+99'
331 '/var/spool/cwl/foo2': {
332 'kind': 'collection',
334 'portable_data_hash': '99999999999999999999999999999996+99'
336 '/var/spool/cwl/filename': {
337 'kind': 'collection',
339 'portable_data_hash': '99999999999999999999999999999996+99'
341 '/var/spool/cwl/subdir': {
342 'kind': 'collection',
344 'portable_data_hash': '99999999999999999999999999999996+99'
347 'state': 'Committed',
348 'output_name': 'Output for step test_initial_work_dir',
349 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
350 'output_path': '/var/spool/cwl',
352 'container_image': 'arvados/jobs',
354 'cwd': '/var/spool/cwl',
355 'scheduling_parameters': {
361 call_body = call_kwargs.get('body', None)
362 self.assertNotEqual(None, call_body)
363 for key in call_body:
364 self.assertEqual(call_body_expected.get(key), call_body.get(key))
367 # Test redirecting stdin/stdout/stderr
368 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
369 def test_redirects(self, keepdocker):
370 arv_docker_clear_cache()
372 runner = mock.MagicMock()
373 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
374 runner.ignore_docker_for_reuse = False
375 runner.intermediate_output_ttl = 0
376 runner.secret_store = cwltool.secrets.SecretStore()
378 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
379 runner.api.collections().get().execute.return_value = {
380 "portable_data_hash": "99999999999999999999999999999993+99"}
382 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
388 "stdout": "stdout.txt",
389 "stderr": "stderr.txt",
390 "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
391 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
393 "class": "CommandLineTool"
396 loadingContext, runtimeContext = self.helper(runner)
397 runtimeContext.name = "test_run_redirect"
399 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
400 arvtool.formatgraph = None
401 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
402 j.run(runtimeContext)
403 runner.api.container_requests().create.assert_called_with(
404 body=JsonDiffMatcher({
406 'HOME': '/var/spool/cwl',
409 'name': 'test_run_redirect',
410 'runtime_constraints': {
414 'use_existing': True,
417 '/tmp': {'kind': 'tmp',
418 "capacity": 1073741824 },
419 '/var/spool/cwl': {'kind': 'tmp',
420 "capacity": 1073741824 },
423 "path": "/var/spool/cwl/stderr.txt"
426 "kind": "collection",
428 "portable_data_hash": "99999999999999999999999999999996+99"
432 "path": "/var/spool/cwl/stdout.txt"
435 'state': 'Committed',
436 "output_name": "Output for step test_run_redirect",
437 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
438 'output_path': '/var/spool/cwl',
440 'container_image': 'arvados/jobs',
441 'command': ['ls', '/var/spool/cwl'],
442 'cwd': '/var/spool/cwl',
443 'scheduling_parameters': {},
448 @mock.patch("arvados.collection.Collection")
449 def test_done(self, col):
450 api = mock.MagicMock()
452 runner = mock.MagicMock()
454 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
455 runner.num_retries = 0
456 runner.ignore_docker_for_reuse = False
457 runner.intermediate_output_ttl = 0
458 runner.secret_store = cwltool.secrets.SecretStore()
460 runner.api.containers().get().execute.return_value = {"state":"Complete",
464 col().open.return_value = []
466 arvjob = arvados_cwl.ArvadosContainer(runner,
473 arvjob.output_callback = mock.MagicMock()
474 arvjob.collect_outputs = mock.MagicMock()
475 arvjob.successCodes = [0]
476 arvjob.outdir = "/var/spool/cwl"
477 arvjob.output_ttl = 3600
479 arvjob.collect_outputs.return_value = {"out": "stuff"}
483 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
484 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
485 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
486 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
487 "modified_at": "2017-05-26T12:01:22Z"
490 self.assertFalse(api.collections().create.called)
491 self.assertFalse(runner.runtime_status_error.called)
493 arvjob.collect_outputs.assert_called_with("keep:abc+123")
494 arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
495 runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
497 @mock.patch("arvados_cwl.done.logtail")
498 @mock.patch("arvados.collection.CollectionReader")
499 @mock.patch("arvados.collection.Collection")
500 def test_child_failure(self, col, reader, logtail):
501 api = mock.MagicMock()
503 runner = mock.MagicMock()
505 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
506 runner.num_retries = 0
507 runner.ignore_docker_for_reuse = False
508 runner.intermediate_output_ttl = 0
509 runner.secret_store = cwltool.secrets.SecretStore()
510 runner.label.return_value = '[container testjob]'
512 runner.api.containers().get().execute.return_value = {
519 col().open.return_value = []
520 logtail.return_value = 'some error detail'
522 arvjob = arvados_cwl.ArvadosContainer(runner,
529 arvjob.output_callback = mock.MagicMock()
530 arvjob.collect_outputs = mock.MagicMock()
531 arvjob.successCodes = [0]
532 arvjob.outdir = "/var/spool/cwl"
533 arvjob.output_ttl = 3600
534 arvjob.collect_outputs.return_value = {"out": "stuff"}
538 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
539 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
540 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
541 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
542 "modified_at": "2017-05-26T12:01:22Z"
545 runner.runtime_status_update.assert_called_with(
547 '[container testjob] zzzzz-xvhdp-zzzzzzzzzzzzzzz failed',
550 arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
552 # The test passes no builder.resources
553 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
554 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
555 def test_mounts(self, keepdocker):
556 arv_docker_clear_cache()
558 runner = mock.MagicMock()
559 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
560 runner.ignore_docker_for_reuse = False
561 runner.intermediate_output_ttl = 0
562 runner.secret_store = cwltool.secrets.SecretStore()
564 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
565 runner.api.collections().get().execute.return_value = {
566 "portable_data_hash": "99999999999999999999999999999994+99",
567 "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
569 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
578 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
580 "class": "CommandLineTool"
583 loadingContext, runtimeContext = self.helper(runner)
584 runtimeContext.name = "test_run_mounts"
586 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
587 arvtool.formatgraph = None
590 "class": "Directory",
591 "location": "keep:99999999999999999999999999999994+44",
595 "location": "keep:99999999999999999999999999999994+44/file1",
599 "location": "keep:99999999999999999999999999999994+44/file2",
604 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
605 j.run(runtimeContext)
606 runner.api.container_requests().create.assert_called_with(
607 body=JsonDiffMatcher({
609 'HOME': '/var/spool/cwl',
612 'name': 'test_run_mounts',
613 'runtime_constraints': {
617 'use_existing': True,
620 "/keep/99999999999999999999999999999994+44": {
621 "kind": "collection",
622 "portable_data_hash": "99999999999999999999999999999994+44"
624 '/tmp': {'kind': 'tmp',
625 "capacity": 1073741824 },
626 '/var/spool/cwl': {'kind': 'tmp',
627 "capacity": 1073741824 }
629 'state': 'Committed',
630 'output_name': 'Output for step test_run_mounts',
631 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
632 'output_path': '/var/spool/cwl',
634 'container_image': 'arvados/jobs',
635 'command': ['ls', '/var/spool/cwl'],
636 'cwd': '/var/spool/cwl',
637 'scheduling_parameters': {},
642 # The test passes no builder.resources
643 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
644 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
645 def test_secrets(self, keepdocker):
646 arv_docker_clear_cache()
648 runner = mock.MagicMock()
649 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
650 runner.ignore_docker_for_reuse = False
651 runner.intermediate_output_ttl = 0
652 runner.secret_store = cwltool.secrets.SecretStore()
654 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
655 runner.api.collections().get().execute.return_value = {
656 "portable_data_hash": "99999999999999999999999999999993+99"}
658 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
660 tool = cmap({"arguments": ["md5sum", "example.conf"],
661 "class": "CommandLineTool",
664 "class": "http://commonwl.org/cwltool#Secrets",
670 "id": "#secret_job.cwl",
673 "id": "#secret_job.cwl/pw",
681 "class": "InitialWorkDirRequirement",
684 "entry": "username: user\npassword: $(inputs.pw)\n",
685 "entryname": "example.conf"
691 loadingContext, runtimeContext = self.helper(runner)
692 runtimeContext.name = "test_secrets"
694 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
695 arvtool.formatgraph = None
697 job_order = {"pw": "blorp"}
698 runner.secret_store.store(["pw"], job_order)
700 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
701 j.run(runtimeContext)
702 runner.api.container_requests().create.assert_called_with(
703 body=JsonDiffMatcher({
705 'HOME': '/var/spool/cwl',
708 'name': 'test_secrets',
709 'runtime_constraints': {
713 'use_existing': True,
716 '/tmp': {'kind': 'tmp',
717 "capacity": 1073741824
719 '/var/spool/cwl': {'kind': 'tmp',
720 "capacity": 1073741824 }
722 'state': 'Committed',
723 'output_name': 'Output for step test_secrets',
724 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
725 'output_path': '/var/spool/cwl',
727 'container_image': 'arvados/jobs',
728 'command': ['md5sum', 'example.conf'],
729 'cwd': '/var/spool/cwl',
730 'scheduling_parameters': {},
733 "/var/spool/cwl/example.conf": {
734 "content": "username: user\npassword: blorp\n",
740 # The test passes no builder.resources
741 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
742 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
743 def test_timelimit(self, keepdocker):
744 arv_docker_clear_cache()
746 runner = mock.MagicMock()
747 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
748 runner.ignore_docker_for_reuse = False
749 runner.intermediate_output_ttl = 0
750 runner.secret_store = cwltool.secrets.SecretStore()
752 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
753 runner.api.collections().get().execute.return_value = {
754 "portable_data_hash": "99999999999999999999999999999993+99"}
760 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
762 "class": "CommandLineTool",
765 "class": "http://commonwl.org/cwltool#TimeLimit",
771 loadingContext, runtimeContext = self.helper(runner)
772 runtimeContext.name = "test_timelimit"
774 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
775 arvtool.formatgraph = None
777 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
778 j.run(runtimeContext)
780 _, kwargs = runner.api.container_requests().create.call_args
781 self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))