1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
6 import arvados_cwl.context
7 from arvados_cwl.arvdocker import arv_docker_clear_cache
13 import cwltool.process
14 import cwltool.secrets
15 from schema_salad.ref_resolver import Loader
16 from schema_salad.sourceline import cmap
18 from .matcher import JsonDiffMatcher
20 if not os.getenv('ARVADOS_DEBUG'):
21 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
22 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
24 class TestContainer(unittest.TestCase):
26 def helper(self, runner, enable_reuse=True):
27 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
29 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
30 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
31 loadingContext = arvados_cwl.context.ArvLoadingContext(
32 {"avsc_names": avsc_names,
34 "make_fs_access": make_fs_access,
36 "metadata": {"cwlVersion": "v1.0"}})
37 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
38 {"work_api": "containers",
40 "name": "test_run_"+str(enable_reuse),
41 "make_fs_access": make_fs_access,
43 "enable_reuse": enable_reuse,
46 return loadingContext, runtimeContext
48 # The test passes no builder.resources
49 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
50 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
51 def test_run(self, keepdocker):
52 for enable_reuse in (True, False):
53 arv_docker_clear_cache()
55 runner = mock.MagicMock()
56 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
57 runner.ignore_docker_for_reuse = False
58 runner.intermediate_output_ttl = 0
59 runner.secret_store = cwltool.secrets.SecretStore()
61 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
62 runner.api.collections().get().execute.return_value = {
63 "portable_data_hash": "99999999999999999999999999999993+99"}
69 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
71 "class": "CommandLineTool"
74 loadingContext, runtimeContext = self.helper(runner, enable_reuse)
76 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
77 arvtool.formatgraph = None
79 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
81 runner.api.container_requests().create.assert_called_with(
82 body=JsonDiffMatcher({
84 'HOME': '/var/spool/cwl',
87 'name': 'test_run_'+str(enable_reuse),
88 'runtime_constraints': {
92 'use_existing': enable_reuse,
95 '/tmp': {'kind': 'tmp',
96 "capacity": 1073741824
98 '/var/spool/cwl': {'kind': 'tmp',
99 "capacity": 1073741824 }
101 'state': 'Committed',
102 'output_name': 'Output for step test_run_'+str(enable_reuse),
103 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
104 'output_path': '/var/spool/cwl',
106 'container_image': 'arvados/jobs',
107 'command': ['ls', '/var/spool/cwl'],
108 'cwd': '/var/spool/cwl',
109 'scheduling_parameters': {},
114 # The test passes some fields in builder.resources
115 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
116 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
117 def test_resource_requirements(self, keepdocker):
118 arv_docker_clear_cache()
119 runner = mock.MagicMock()
120 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
121 runner.ignore_docker_for_reuse = False
122 runner.intermediate_output_ttl = 3600
123 runner.secret_store = cwltool.secrets.SecretStore()
125 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
126 runner.api.collections().get().execute.return_value = {
127 "portable_data_hash": "99999999999999999999999999999993+99"}
133 "class": "ResourceRequirement",
139 "class": "http://arvados.org/cwl#RuntimeConstraints",
142 "class": "http://arvados.org/cwl#APIRequirement",
144 "class": "http://arvados.org/cwl#PartitionRequirement",
147 "class": "http://arvados.org/cwl#IntermediateOutput",
150 "class": "http://arvados.org/cwl#ReuseRequirement",
155 "class": "CommandLineTool"
158 loadingContext, runtimeContext = self.helper(runner)
159 runtimeContext.name = "test_resource_requirements"
161 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
162 arvtool.formatgraph = None
163 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
164 j.run(runtimeContext)
166 call_args, call_kwargs = runner.api.container_requests().create.call_args
168 call_body_expected = {
170 'HOME': '/var/spool/cwl',
173 'name': 'test_resource_requirements',
174 'runtime_constraints': {
177 'keep_cache_ram': 536870912,
180 'use_existing': False,
183 '/tmp': {'kind': 'tmp',
184 "capacity": 4194304000 },
185 '/var/spool/cwl': {'kind': 'tmp',
186 "capacity": 5242880000 }
188 'state': 'Committed',
189 'output_name': 'Output for step test_resource_requirements',
190 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
191 'output_path': '/var/spool/cwl',
193 'container_image': 'arvados/jobs',
195 'cwd': '/var/spool/cwl',
196 'scheduling_parameters': {
197 'partitions': ['blurb']
203 call_body = call_kwargs.get('body', None)
204 self.assertNotEqual(None, call_body)
205 for key in call_body:
206 self.assertEqual(call_body_expected.get(key), call_body.get(key))
209 # The test passes some fields in builder.resources
210 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
211 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
212 @mock.patch("arvados.collection.Collection")
213 def test_initial_work_dir(self, collection_mock, keepdocker):
214 arv_docker_clear_cache()
215 runner = mock.MagicMock()
216 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
217 runner.ignore_docker_for_reuse = False
218 runner.intermediate_output_ttl = 0
219 runner.secret_store = cwltool.secrets.SecretStore()
221 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
222 runner.api.collections().get().execute.return_value = {
223 "portable_data_hash": "99999999999999999999999999999993+99"}
225 sourcemock = mock.MagicMock()
226 def get_collection_mock(p):
228 return (sourcemock, p.split("/", 1)[1])
230 return (sourcemock, "")
231 runner.fs_access.get_collection.side_effect = get_collection_mock
233 vwdmock = mock.MagicMock()
234 collection_mock.return_value = vwdmock
235 vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
241 "class": "InitialWorkDirRequirement",
245 "location": "keep:99999999999999999999999999999995+99/bar"
248 "class": "Directory",
250 "location": "keep:99999999999999999999999999999995+99"
254 "basename": "filename",
255 "location": "keep:99999999999999999999999999999995+99/baz/filename"
258 "class": "Directory",
259 "basename": "subdir",
260 "location": "keep:99999999999999999999999999999995+99/subdir"
265 "class": "CommandLineTool"
268 loadingContext, runtimeContext = self.helper(runner)
269 runtimeContext.name = "test_initial_work_dir"
271 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
272 arvtool.formatgraph = None
273 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
274 j.run(runtimeContext)
276 call_args, call_kwargs = runner.api.container_requests().create.call_args
278 vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
279 vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
280 vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
281 vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
283 call_body_expected = {
285 'HOME': '/var/spool/cwl',
288 'name': 'test_initial_work_dir',
289 'runtime_constraints': {
293 'use_existing': True,
296 '/tmp': {'kind': 'tmp',
297 "capacity": 1073741824 },
298 '/var/spool/cwl': {'kind': 'tmp',
299 "capacity": 1073741824 },
300 '/var/spool/cwl/foo': {
301 'kind': 'collection',
303 'portable_data_hash': '99999999999999999999999999999996+99'
305 '/var/spool/cwl/foo2': {
306 'kind': 'collection',
308 'portable_data_hash': '99999999999999999999999999999996+99'
310 '/var/spool/cwl/filename': {
311 'kind': 'collection',
313 'portable_data_hash': '99999999999999999999999999999996+99'
315 '/var/spool/cwl/subdir': {
316 'kind': 'collection',
318 'portable_data_hash': '99999999999999999999999999999996+99'
321 'state': 'Committed',
322 'output_name': 'Output for step test_initial_work_dir',
323 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
324 'output_path': '/var/spool/cwl',
326 'container_image': 'arvados/jobs',
328 'cwd': '/var/spool/cwl',
329 'scheduling_parameters': {
335 call_body = call_kwargs.get('body', None)
336 self.assertNotEqual(None, call_body)
337 for key in call_body:
338 self.assertEqual(call_body_expected.get(key), call_body.get(key))
341 # Test redirecting stdin/stdout/stderr
342 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
343 def test_redirects(self, keepdocker):
344 arv_docker_clear_cache()
346 runner = mock.MagicMock()
347 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
348 runner.ignore_docker_for_reuse = False
349 runner.intermediate_output_ttl = 0
350 runner.secret_store = cwltool.secrets.SecretStore()
352 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
353 runner.api.collections().get().execute.return_value = {
354 "portable_data_hash": "99999999999999999999999999999993+99"}
356 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
362 "stdout": "stdout.txt",
363 "stderr": "stderr.txt",
364 "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
365 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
367 "class": "CommandLineTool"
370 loadingContext, runtimeContext = self.helper(runner)
371 runtimeContext.name = "test_run_redirect"
373 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
374 arvtool.formatgraph = None
375 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
376 j.run(runtimeContext)
377 runner.api.container_requests().create.assert_called_with(
378 body=JsonDiffMatcher({
380 'HOME': '/var/spool/cwl',
383 'name': 'test_run_redirect',
384 'runtime_constraints': {
388 'use_existing': True,
391 '/tmp': {'kind': 'tmp',
392 "capacity": 1073741824 },
393 '/var/spool/cwl': {'kind': 'tmp',
394 "capacity": 1073741824 },
397 "path": "/var/spool/cwl/stderr.txt"
400 "kind": "collection",
402 "portable_data_hash": "99999999999999999999999999999996+99"
406 "path": "/var/spool/cwl/stdout.txt"
409 'state': 'Committed',
410 "output_name": "Output for step test_run_redirect",
411 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
412 'output_path': '/var/spool/cwl',
414 'container_image': 'arvados/jobs',
415 'command': ['ls', '/var/spool/cwl'],
416 'cwd': '/var/spool/cwl',
417 'scheduling_parameters': {},
422 @mock.patch("arvados.collection.Collection")
423 def test_done(self, col):
424 api = mock.MagicMock()
426 runner = mock.MagicMock()
428 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
429 runner.num_retries = 0
430 runner.ignore_docker_for_reuse = False
431 runner.intermediate_output_ttl = 0
432 runner.secret_store = cwltool.secrets.SecretStore()
434 runner.api.containers().get().execute.return_value = {"state":"Complete",
438 col().open.return_value = []
440 arvjob = arvados_cwl.ArvadosContainer(runner,
447 arvjob.output_callback = mock.MagicMock()
448 arvjob.collect_outputs = mock.MagicMock()
449 arvjob.successCodes = [0]
450 arvjob.outdir = "/var/spool/cwl"
451 arvjob.output_ttl = 3600
453 arvjob.collect_outputs.return_value = {"out": "stuff"}
457 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
458 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
459 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
460 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
461 "modified_at": "2017-05-26T12:01:22Z"
464 self.assertFalse(api.collections().create.called)
466 arvjob.collect_outputs.assert_called_with("keep:abc+123")
467 arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
468 runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
470 # The test passes no builder.resources
471 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
472 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
473 def test_mounts(self, keepdocker):
474 arv_docker_clear_cache()
476 runner = mock.MagicMock()
477 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
478 runner.ignore_docker_for_reuse = False
479 runner.intermediate_output_ttl = 0
480 runner.secret_store = cwltool.secrets.SecretStore()
482 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
483 runner.api.collections().get().execute.return_value = {
484 "portable_data_hash": "99999999999999999999999999999993+99"}
486 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
495 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
497 "class": "CommandLineTool"
500 loadingContext, runtimeContext = self.helper(runner)
501 runtimeContext.name = "test_run_mounts"
503 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
504 arvtool.formatgraph = None
507 "class": "Directory",
508 "location": "keep:99999999999999999999999999999994+44",
512 "location": "keep:99999999999999999999999999999994+44/file1",
516 "location": "keep:99999999999999999999999999999994+44/file2",
521 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
522 j.run(runtimeContext)
523 runner.api.container_requests().create.assert_called_with(
524 body=JsonDiffMatcher({
526 'HOME': '/var/spool/cwl',
529 'name': 'test_run_mounts',
530 'runtime_constraints': {
534 'use_existing': True,
537 "/keep/99999999999999999999999999999994+44": {
538 "kind": "collection",
539 "portable_data_hash": "99999999999999999999999999999994+44"
541 '/tmp': {'kind': 'tmp',
542 "capacity": 1073741824 },
543 '/var/spool/cwl': {'kind': 'tmp',
544 "capacity": 1073741824 }
546 'state': 'Committed',
547 'output_name': 'Output for step test_run_mounts',
548 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
549 'output_path': '/var/spool/cwl',
551 'container_image': 'arvados/jobs',
552 'command': ['ls', '/var/spool/cwl'],
553 'cwd': '/var/spool/cwl',
554 'scheduling_parameters': {},
559 # The test passes no builder.resources
560 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
561 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
562 def test_secrets(self, keepdocker):
563 arv_docker_clear_cache()
565 runner = mock.MagicMock()
566 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
567 runner.ignore_docker_for_reuse = False
568 runner.intermediate_output_ttl = 0
569 runner.secret_store = cwltool.secrets.SecretStore()
571 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
572 runner.api.collections().get().execute.return_value = {
573 "portable_data_hash": "99999999999999999999999999999993+99"}
575 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
577 tool = cmap({"arguments": ["md5sum", "example.conf"],
578 "class": "CommandLineTool",
581 "class": "http://commonwl.org/cwltool#Secrets",
587 "id": "#secret_job.cwl",
590 "id": "#secret_job.cwl/pw",
598 "class": "InitialWorkDirRequirement",
601 "entry": "username: user\npassword: $(inputs.pw)\n",
602 "entryname": "example.conf"
608 loadingContext, runtimeContext = self.helper(runner)
609 runtimeContext.name = "test_secrets"
611 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
612 arvtool.formatgraph = None
614 job_order = {"pw": "blorp"}
615 runner.secret_store.store(["pw"], job_order)
617 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
618 j.run(runtimeContext)
619 runner.api.container_requests().create.assert_called_with(
620 body=JsonDiffMatcher({
622 'HOME': '/var/spool/cwl',
625 'name': 'test_secrets',
626 'runtime_constraints': {
630 'use_existing': True,
633 '/tmp': {'kind': 'tmp',
634 "capacity": 1073741824
636 '/var/spool/cwl': {'kind': 'tmp',
637 "capacity": 1073741824 }
639 'state': 'Committed',
640 'output_name': 'Output for step test_secrets',
641 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
642 'output_path': '/var/spool/cwl',
644 'container_image': 'arvados/jobs',
645 'command': ['md5sum', 'example.conf'],
646 'cwd': '/var/spool/cwl',
647 'scheduling_parameters': {},
650 "/var/spool/cwl/example.conf": {
651 "content": "username: user\npassword: blorp\n",
657 # The test passes no builder.resources
658 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
659 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
660 def test_timelimit(self, keepdocker):
661 arv_docker_clear_cache()
663 runner = mock.MagicMock()
664 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
665 runner.ignore_docker_for_reuse = False
666 runner.intermediate_output_ttl = 0
667 runner.secret_store = cwltool.secrets.SecretStore()
669 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
670 runner.api.collections().get().execute.return_value = {
671 "portable_data_hash": "99999999999999999999999999999993+99"}
677 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
679 "class": "CommandLineTool",
682 "class": "http://commonwl.org/cwltool#TimeLimit",
688 loadingContext, runtimeContext = self.helper(runner)
689 runtimeContext.name = "test_timelimit"
691 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
692 arvtool.formatgraph = None
694 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
695 j.run(runtimeContext)
697 _, kwargs = runner.api.container_requests().create.call_args
698 self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))