1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from builtins import str
6 from builtins import object
9 import arvados_cwl.context
10 import arvados_cwl.util
11 from arvados_cwl.arvdocker import arv_docker_clear_cache
19 import cwltool.process
20 import cwltool.secrets
21 from cwltool.update import INTERNAL_VERSION
22 from schema_salad.ref_resolver import Loader
23 from schema_salad.sourceline import cmap
25 from .matcher import JsonDiffMatcher, StripYAMLComments
26 from .mock_discovery import get_rootDesc
28 if not os.getenv('ARVADOS_DEBUG'):
29 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
30 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
32 class CollectionMock(object):
33 def __init__(self, vwdmock, *args, **kwargs):
34 self.vwdmock = vwdmock
37 def open(self, *args, **kwargs):
39 return self.vwdmock.open(*args, **kwargs)
41 def copy(self, *args, **kwargs):
43 self.vwdmock.copy(*args, **kwargs)
45 def save_new(self, *args, **kwargs):
51 def portable_data_hash(self):
53 return arvados.config.EMPTY_BLOCK_LOCATOR
55 return "99999999999999999999999999999996+99"
58 class TestContainer(unittest.TestCase):
61 cwltool.process._names = set()
62 arv_docker_clear_cache()
64 def helper(self, runner, enable_reuse=True):
65 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
67 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
68 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
69 loadingContext = arvados_cwl.context.ArvLoadingContext(
70 {"avsc_names": avsc_names,
72 "make_fs_access": make_fs_access,
74 "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
75 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
76 {"work_api": "containers",
78 "name": "test_run_"+str(enable_reuse),
79 "make_fs_access": make_fs_access,
81 "enable_reuse": enable_reuse,
83 "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
86 return loadingContext, runtimeContext
88 # Helper function to set up the ArvCwlExecutor to use the containers api
89 # and test that the RuntimeStatusLoggingHandler is set up correctly
90 def setup_and_test_container_executor_and_logging(self, gcc_mock) :
91 api = mock.MagicMock()
92 api._rootDesc = copy.deepcopy(get_rootDesc())
94 # Make sure ArvCwlExecutor thinks it's running inside a container so it
95 # adds the logging handler that will call runtime_status_update() mock
96 self.assertFalse(gcc_mock.called)
97 runner = arvados_cwl.ArvCwlExecutor(api)
98 self.assertEqual(runner.work_api, 'containers')
99 root_logger = logging.getLogger('')
100 handlerClasses = [h.__class__ for h in root_logger.handlers]
101 self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
104 # The test passes no builder.resources
105 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
106 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
107 def test_run(self, keepdocker):
108 for enable_reuse in (True, False):
109 arv_docker_clear_cache()
111 runner = mock.MagicMock()
112 runner.ignore_docker_for_reuse = False
113 runner.intermediate_output_ttl = 0
114 runner.secret_store = cwltool.secrets.SecretStore()
116 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
117 runner.api.collections().get().execute.return_value = {
118 "portable_data_hash": "99999999999999999999999999999993+99"}
124 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
126 "class": "CommandLineTool"
129 loadingContext, runtimeContext = self.helper(runner, enable_reuse)
131 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
132 arvtool.formatgraph = None
134 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
135 j.run(runtimeContext)
136 runner.api.container_requests().create.assert_called_with(
137 body=JsonDiffMatcher({
139 'HOME': '/var/spool/cwl',
142 'name': 'test_run_'+str(enable_reuse),
143 'runtime_constraints': {
147 'use_existing': enable_reuse,
150 '/tmp': {'kind': 'tmp',
151 "capacity": 1073741824
153 '/var/spool/cwl': {'kind': 'tmp',
154 "capacity": 1073741824 }
156 'state': 'Committed',
157 'output_name': 'Output for step test_run_'+str(enable_reuse),
158 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
159 'output_path': '/var/spool/cwl',
161 'container_image': '99999999999999999999999999999993+99',
162 'command': ['ls', '/var/spool/cwl'],
163 'cwd': '/var/spool/cwl',
164 'scheduling_parameters': {},
167 'output_storage_classes': ["default"]
170 # The test passes some fields in builder.resources
171 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
172 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
173 def test_resource_requirements(self, keepdocker):
174 runner = mock.MagicMock()
175 runner.ignore_docker_for_reuse = False
176 runner.intermediate_output_ttl = 3600
177 runner.secret_store = cwltool.secrets.SecretStore()
179 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
180 runner.api.collections().get().execute.return_value = {
181 "portable_data_hash": "99999999999999999999999999999993+99"}
187 "class": "ResourceRequirement",
193 "class": "http://arvados.org/cwl#RuntimeConstraints",
196 "class": "http://arvados.org/cwl#APIRequirement",
198 "class": "http://arvados.org/cwl#PartitionRequirement",
201 "class": "http://arvados.org/cwl#IntermediateOutput",
204 "class": "http://arvados.org/cwl#ReuseRequirement",
209 "class": "CommandLineTool"
212 loadingContext, runtimeContext = self.helper(runner)
213 runtimeContext.name = "test_resource_requirements"
215 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
216 arvtool.formatgraph = None
217 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
218 j.run(runtimeContext)
220 call_args, call_kwargs = runner.api.container_requests().create.call_args
222 call_body_expected = {
224 'HOME': '/var/spool/cwl',
227 'name': 'test_resource_requirements',
228 'runtime_constraints': {
231 'keep_cache_ram': 536870912,
234 'use_existing': False,
237 '/tmp': {'kind': 'tmp',
238 "capacity": 4194304000 },
239 '/var/spool/cwl': {'kind': 'tmp',
240 "capacity": 5242880000 }
242 'state': 'Committed',
243 'output_name': 'Output for step test_resource_requirements',
244 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
245 'output_path': '/var/spool/cwl',
247 'container_image': '99999999999999999999999999999993+99',
249 'cwd': '/var/spool/cwl',
250 'scheduling_parameters': {
251 'partitions': ['blurb']
255 'output_storage_classes': ["default"]
258 call_body = call_kwargs.get('body', None)
259 self.assertNotEqual(None, call_body)
260 for key in call_body:
261 self.assertEqual(call_body_expected.get(key), call_body.get(key))
264 # The test passes some fields in builder.resources
265 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
266 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
267 @mock.patch("arvados.collection.Collection")
268 def test_initial_work_dir(self, collection_mock, keepdocker):
269 runner = mock.MagicMock()
270 runner.ignore_docker_for_reuse = False
271 runner.intermediate_output_ttl = 0
272 runner.secret_store = cwltool.secrets.SecretStore()
274 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
275 runner.api.collections().get().execute.return_value = {
276 "portable_data_hash": "99999999999999999999999999999993+99"}
278 sourcemock = mock.MagicMock()
279 def get_collection_mock(p):
281 return (sourcemock, p.split("/", 1)[1])
283 return (sourcemock, "")
284 runner.fs_access.get_collection.side_effect = get_collection_mock
286 vwdmock = mock.MagicMock()
287 collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
293 "class": "InitialWorkDirRequirement",
297 "location": "keep:99999999999999999999999999999995+99/bar"
300 "class": "Directory",
302 "location": "keep:99999999999999999999999999999995+99"
306 "basename": "filename",
307 "location": "keep:99999999999999999999999999999995+99/baz/filename"
310 "class": "Directory",
311 "basename": "subdir",
312 "location": "keep:99999999999999999999999999999995+99/subdir"
317 "class": "CommandLineTool"
320 loadingContext, runtimeContext = self.helper(runner)
321 runtimeContext.name = "test_initial_work_dir"
323 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
324 arvtool.formatgraph = None
325 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
326 j.run(runtimeContext)
328 call_args, call_kwargs = runner.api.container_requests().create.call_args
330 vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
331 vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
332 vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
333 vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
335 call_body_expected = {
337 'HOME': '/var/spool/cwl',
340 'name': 'test_initial_work_dir',
341 'runtime_constraints': {
345 'use_existing': True,
348 '/tmp': {'kind': 'tmp',
349 "capacity": 1073741824 },
350 '/var/spool/cwl': {'kind': 'tmp',
351 "capacity": 1073741824 },
352 '/var/spool/cwl/foo': {
353 'kind': 'collection',
355 'portable_data_hash': '99999999999999999999999999999996+99'
357 '/var/spool/cwl/foo2': {
358 'kind': 'collection',
360 'portable_data_hash': '99999999999999999999999999999996+99'
362 '/var/spool/cwl/filename': {
363 'kind': 'collection',
365 'portable_data_hash': '99999999999999999999999999999996+99'
367 '/var/spool/cwl/subdir': {
368 'kind': 'collection',
370 'portable_data_hash': '99999999999999999999999999999996+99'
373 'state': 'Committed',
374 'output_name': 'Output for step test_initial_work_dir',
375 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
376 'output_path': '/var/spool/cwl',
378 'container_image': '99999999999999999999999999999993+99',
380 'cwd': '/var/spool/cwl',
381 'scheduling_parameters': {
385 'output_storage_classes': ["default"]
388 call_body = call_kwargs.get('body', None)
389 self.assertNotEqual(None, call_body)
390 for key in call_body:
391 self.assertEqual(call_body_expected.get(key), call_body.get(key))
394 # Test redirecting stdin/stdout/stderr
395 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
396 def test_redirects(self, keepdocker):
397 runner = mock.MagicMock()
398 runner.ignore_docker_for_reuse = False
399 runner.intermediate_output_ttl = 0
400 runner.secret_store = cwltool.secrets.SecretStore()
402 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
403 runner.api.collections().get().execute.return_value = {
404 "portable_data_hash": "99999999999999999999999999999993+99"}
406 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
412 "stdout": "stdout.txt",
413 "stderr": "stderr.txt",
414 "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
415 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
417 "class": "CommandLineTool"
420 loadingContext, runtimeContext = self.helper(runner)
421 runtimeContext.name = "test_run_redirect"
423 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
424 arvtool.formatgraph = None
425 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
426 j.run(runtimeContext)
427 runner.api.container_requests().create.assert_called_with(
428 body=JsonDiffMatcher({
430 'HOME': '/var/spool/cwl',
433 'name': 'test_run_redirect',
434 'runtime_constraints': {
438 'use_existing': True,
441 '/tmp': {'kind': 'tmp',
442 "capacity": 1073741824 },
443 '/var/spool/cwl': {'kind': 'tmp',
444 "capacity": 1073741824 },
447 "path": "/var/spool/cwl/stderr.txt"
450 "kind": "collection",
452 "portable_data_hash": "99999999999999999999999999999996+99"
456 "path": "/var/spool/cwl/stdout.txt"
459 'state': 'Committed',
460 "output_name": "Output for step test_run_redirect",
461 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
462 'output_path': '/var/spool/cwl',
464 'container_image': '99999999999999999999999999999993+99',
465 'command': ['ls', '/var/spool/cwl'],
466 'cwd': '/var/spool/cwl',
467 'scheduling_parameters': {},
470 'output_storage_classes': ["default"]
473 @mock.patch("arvados.collection.Collection")
474 def test_done(self, col):
475 api = mock.MagicMock()
477 runner = mock.MagicMock()
479 runner.num_retries = 0
480 runner.ignore_docker_for_reuse = False
481 runner.intermediate_output_ttl = 0
482 runner.secret_store = cwltool.secrets.SecretStore()
484 runner.api.containers().get().execute.return_value = {"state":"Complete",
488 col().open.return_value = []
490 loadingContext, runtimeContext = self.helper(runner)
492 arvjob = arvados_cwl.ArvadosContainer(runner,
500 arvjob.output_callback = mock.MagicMock()
501 arvjob.collect_outputs = mock.MagicMock()
502 arvjob.successCodes = [0]
503 arvjob.outdir = "/var/spool/cwl"
504 arvjob.output_ttl = 3600
506 arvjob.collect_outputs.return_value = {"out": "stuff"}
510 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
511 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
512 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
513 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
514 "modified_at": "2017-05-26T12:01:22Z"
517 self.assertFalse(api.collections().create.called)
518 self.assertFalse(runner.runtime_status_error.called)
520 arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
521 arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
522 runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
524 # Test to make sure we dont call runtime_status_update if we already did
525 # some where higher up in the call stack
526 @mock.patch("arvados_cwl.util.get_current_container")
527 def test_recursive_runtime_status_update(self, gcc_mock):
528 self.setup_and_test_container_executor_and_logging(gcc_mock)
529 root_logger = logging.getLogger('')
531 # get_current_container is invoked when we call runtime_status_update
532 # so try and log again!
533 gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
535 root_logger.error("First Error")
537 self.fail("RuntimeStatusLoggingHandler should not be called recursively")
540 # Test to make sure that an exception raised from
541 # get_current_container doesn't cause the logger to raise an
543 @mock.patch("arvados_cwl.util.get_current_container")
544 def test_runtime_status_get_current_container_exception(self, gcc_mock):
545 self.setup_and_test_container_executor_and_logging(gcc_mock)
546 root_logger = logging.getLogger('')
548 # get_current_container is invoked when we call
549 # runtime_status_update, it is going to also raise an
551 gcc_mock.side_effect = Exception("Second Error")
553 root_logger.error("First Error")
555 self.fail("Exception in logger should not propagate")
556 self.assertTrue(gcc_mock.called)
558 @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
559 @mock.patch("arvados_cwl.util.get_current_container")
560 @mock.patch("arvados.collection.CollectionReader")
561 @mock.patch("arvados.collection.Collection")
562 def test_child_failure(self, col, reader, gcc_mock, rts_mock):
563 runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
565 gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
566 self.assertTrue(gcc_mock.called)
568 runner.num_retries = 0
569 runner.ignore_docker_for_reuse = False
570 runner.intermediate_output_ttl = 0
571 runner.secret_store = cwltool.secrets.SecretStore()
572 runner.label = mock.MagicMock()
573 runner.label.return_value = '[container testjob]'
575 runner.api.containers().get().execute.return_value = {
582 col().open.return_value = []
584 loadingContext, runtimeContext = self.helper(runner)
586 arvjob = arvados_cwl.ArvadosContainer(runner,
594 arvjob.output_callback = mock.MagicMock()
595 arvjob.collect_outputs = mock.MagicMock()
596 arvjob.successCodes = [0]
597 arvjob.outdir = "/var/spool/cwl"
598 arvjob.output_ttl = 3600
599 arvjob.collect_outputs.return_value = {"out": "stuff"}
603 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
604 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
605 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
606 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
607 "modified_at": "2017-05-26T12:01:22Z"
610 rts_mock.assert_called_with(
612 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
613 ' ** log is empty **'
615 arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
617 # The test passes no builder.resources
618 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
619 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
620 def test_mounts(self, keepdocker):
621 runner = mock.MagicMock()
622 runner.ignore_docker_for_reuse = False
623 runner.intermediate_output_ttl = 0
624 runner.secret_store = cwltool.secrets.SecretStore()
626 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
627 runner.api.collections().get().execute.return_value = {
628 "portable_data_hash": "99999999999999999999999999999994+99",
629 "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
631 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
640 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
642 "class": "CommandLineTool"
645 loadingContext, runtimeContext = self.helper(runner)
646 runtimeContext.name = "test_run_mounts"
648 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
649 arvtool.formatgraph = None
652 "class": "Directory",
653 "location": "keep:99999999999999999999999999999994+44",
654 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
658 "location": "keep:99999999999999999999999999999994+44/file1",
662 "location": "keep:99999999999999999999999999999994+44/file2",
667 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
668 j.run(runtimeContext)
669 runner.api.container_requests().create.assert_called_with(
670 body=JsonDiffMatcher({
672 'HOME': '/var/spool/cwl',
675 'name': 'test_run_mounts',
676 'runtime_constraints': {
680 'use_existing': True,
683 "/keep/99999999999999999999999999999994+44": {
684 "kind": "collection",
685 "portable_data_hash": "99999999999999999999999999999994+44",
686 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
688 '/tmp': {'kind': 'tmp',
689 "capacity": 1073741824 },
690 '/var/spool/cwl': {'kind': 'tmp',
691 "capacity": 1073741824 }
693 'state': 'Committed',
694 'output_name': 'Output for step test_run_mounts',
695 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
696 'output_path': '/var/spool/cwl',
698 'container_image': '99999999999999999999999999999994+99',
699 'command': ['ls', '/var/spool/cwl'],
700 'cwd': '/var/spool/cwl',
701 'scheduling_parameters': {},
704 'output_storage_classes': ["default"]
707 # The test passes no builder.resources
708 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
709 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
710 def test_secrets(self, keepdocker):
711 runner = mock.MagicMock()
712 runner.ignore_docker_for_reuse = False
713 runner.intermediate_output_ttl = 0
714 runner.secret_store = cwltool.secrets.SecretStore()
716 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
717 runner.api.collections().get().execute.return_value = {
718 "portable_data_hash": "99999999999999999999999999999993+99"}
720 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
722 tool = cmap({"arguments": ["md5sum", "example.conf"],
723 "class": "CommandLineTool",
726 "class": "http://commonwl.org/cwltool#Secrets",
732 "id": "#secret_job.cwl",
735 "id": "#secret_job.cwl/pw",
743 "class": "InitialWorkDirRequirement",
746 "entry": "username: user\npassword: $(inputs.pw)\n",
747 "entryname": "example.conf"
753 loadingContext, runtimeContext = self.helper(runner)
754 runtimeContext.name = "test_secrets"
756 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
757 arvtool.formatgraph = None
759 job_order = {"pw": "blorp"}
760 runner.secret_store.store(["pw"], job_order)
762 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
763 j.run(runtimeContext)
764 runner.api.container_requests().create.assert_called_with(
765 body=JsonDiffMatcher({
767 'HOME': '/var/spool/cwl',
770 'name': 'test_secrets',
771 'runtime_constraints': {
775 'use_existing': True,
778 '/tmp': {'kind': 'tmp',
779 "capacity": 1073741824
781 '/var/spool/cwl': {'kind': 'tmp',
782 "capacity": 1073741824 }
784 'state': 'Committed',
785 'output_name': 'Output for step test_secrets',
786 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
787 'output_path': '/var/spool/cwl',
789 'container_image': '99999999999999999999999999999993+99',
790 'command': ['md5sum', 'example.conf'],
791 'cwd': '/var/spool/cwl',
792 'scheduling_parameters': {},
795 "/var/spool/cwl/example.conf": {
796 "content": "username: user\npassword: blorp\n",
800 'output_storage_classes': ["default"]
803 # The test passes no builder.resources
804 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
805 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
806 def test_timelimit(self, keepdocker):
807 runner = mock.MagicMock()
808 runner.ignore_docker_for_reuse = False
809 runner.intermediate_output_ttl = 0
810 runner.secret_store = cwltool.secrets.SecretStore()
812 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
813 runner.api.collections().get().execute.return_value = {
814 "portable_data_hash": "99999999999999999999999999999993+99"}
820 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
822 "class": "CommandLineTool",
825 "class": "ToolTimeLimit",
831 loadingContext, runtimeContext = self.helper(runner)
832 runtimeContext.name = "test_timelimit"
834 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
835 arvtool.formatgraph = None
837 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
838 j.run(runtimeContext)
840 _, kwargs = runner.api.container_requests().create.call_args
841 self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
844 # The test passes no builder.resources
845 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
846 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
847 def test_setting_storage_class(self, keepdocker):
848 arv_docker_clear_cache()
850 runner = mock.MagicMock()
851 runner.ignore_docker_for_reuse = False
852 runner.intermediate_output_ttl = 0
853 runner.secret_store = cwltool.secrets.SecretStore()
855 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
856 runner.api.collections().get().execute.return_value = {
857 "portable_data_hash": "99999999999999999999999999999993+99"}
863 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
865 "class": "CommandLineTool",
868 "class": "http://arvados.org/cwl#OutputStorageClass",
869 "finalStorageClass": ["baz_sc", "qux_sc"],
870 "intermediateStorageClass": ["foo_sc", "bar_sc"]
875 loadingContext, runtimeContext = self.helper(runner, True)
877 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
878 arvtool.formatgraph = None
880 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
881 j.run(runtimeContext)
882 runner.api.container_requests().create.assert_called_with(
883 body=JsonDiffMatcher({
885 'HOME': '/var/spool/cwl',
888 'name': 'test_run_True',
889 'runtime_constraints': {
893 'use_existing': True,
896 '/tmp': {'kind': 'tmp',
897 "capacity": 1073741824
899 '/var/spool/cwl': {'kind': 'tmp',
900 "capacity": 1073741824 }
902 'state': 'Committed',
903 'output_name': 'Output for step test_run_True',
904 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
905 'output_path': '/var/spool/cwl',
907 'container_image': '99999999999999999999999999999993+99',
908 'command': ['ls', '/var/spool/cwl'],
909 'cwd': '/var/spool/cwl',
910 'scheduling_parameters': {},
913 'output_storage_classes': ["foo_sc", "bar_sc"]
917 class TestWorkflow(unittest.TestCase):
919 cwltool.process._names = set()
920 arv_docker_clear_cache()
922 def helper(self, runner, enable_reuse=True):
923 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
925 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
926 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
928 document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
929 document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
930 document_loader.fetch_text = document_loader.fetcher.fetch_text
931 document_loader.check_exists = document_loader.fetcher.check_exists
933 loadingContext = arvados_cwl.context.ArvLoadingContext(
934 {"avsc_names": avsc_names,
936 "make_fs_access": make_fs_access,
937 "loader": document_loader,
938 "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
939 "construct_tool_object": runner.arv_make_tool})
940 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
941 {"work_api": "containers",
943 "name": "test_run_wf_"+str(enable_reuse),
944 "make_fs_access": make_fs_access,
946 "enable_reuse": enable_reuse,
949 return loadingContext, runtimeContext
951 # The test passes no builder.resources
952 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
953 @mock.patch("arvados.collection.CollectionReader")
954 @mock.patch("arvados.collection.Collection")
955 @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
956 def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
957 arvados_cwl.add_arv_hints()
959 api = mock.MagicMock()
960 api._rootDesc = get_rootDesc()
962 runner = arvados_cwl.executor.ArvCwlExecutor(api)
963 self.assertEqual(runner.work_api, 'containers')
965 list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
966 runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
967 runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
968 "portable_data_hash": "99999999999999999999999999999993+99"}]}
970 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
971 runner.ignore_docker_for_reuse = False
972 runner.num_retries = 0
973 runner.secret_store = cwltool.secrets.SecretStore()
975 loadingContext, runtimeContext = self.helper(runner)
976 runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
978 mockcollectionreader().exists.return_value = True
980 tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
981 metadata["cwlVersion"] = tool["cwlVersion"]
983 mockc = mock.MagicMock()
984 mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
985 mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
987 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
988 arvtool.formatgraph = None
989 it = arvtool.job({}, mock.MagicMock(), runtimeContext)
991 next(it).run(runtimeContext)
992 next(it).run(runtimeContext)
994 with open("tests/wf/scatter2_subwf.cwl") as f:
995 subwf = StripYAMLComments(f.read()).rstrip()
997 runner.api.container_requests().create.assert_called_with(
998 body=JsonDiffMatcher({
1003 "--preserve-entire-environment",
1007 "container_image": "99999999999999999999999999999993+99",
1008 "cwd": "/var/spool/cwl",
1010 "HOME": "/var/spool/cwl",
1014 "/keep/99999999999999999999999999999999+118": {
1015 "kind": "collection",
1016 "portable_data_hash": "99999999999999999999999999999999+118"
1019 "capacity": 1073741824,
1023 "capacity": 1073741824,
1026 "/var/spool/cwl/cwl.input.yml": {
1027 "kind": "collection",
1028 "path": "cwl.input.yml",
1029 "portable_data_hash": "99999999999999999999999999999996+99"
1031 "/var/spool/cwl/workflow.cwl": {
1032 "kind": "collection",
1033 "path": "workflow.cwl",
1034 "portable_data_hash": "99999999999999999999999999999996+99"
1038 "path": "/var/spool/cwl/cwl.output.json"
1041 "name": "scatterstep",
1042 "output_name": "Output for step scatterstep",
1043 "output_path": "/var/spool/cwl",
1047 "runtime_constraints": {
1051 "scheduling_parameters": {},
1052 "secret_mounts": {},
1053 "state": "Committed",
1054 "use_existing": True,
1055 'output_storage_classes': ["default"]
1057 mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1058 mockc.open().__enter__().write.assert_has_calls([mock.call(
1061 "basename": "token.txt",
1063 "location": "/keep/99999999999999999999999999999999+118/token.txt",
1069 # The test passes no builder.resources
1070 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1071 @mock.patch("arvados.collection.CollectionReader")
1072 @mock.patch("arvados.collection.Collection")
1073 @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1074 def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1075 arvados_cwl.add_arv_hints()
1077 api = mock.MagicMock()
1078 api._rootDesc = get_rootDesc()
1080 runner = arvados_cwl.executor.ArvCwlExecutor(api)
1081 self.assertEqual(runner.work_api, 'containers')
1083 list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1084 runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1085 "portable_data_hash": "99999999999999999999999999999993+99"}
1086 runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1087 "portable_data_hash": "99999999999999999999999999999993+99"}]}
1089 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1090 runner.ignore_docker_for_reuse = False
1091 runner.num_retries = 0
1092 runner.secret_store = cwltool.secrets.SecretStore()
1094 loadingContext, runtimeContext = self.helper(runner)
1095 runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1096 loadingContext.do_update = True
1097 tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1099 mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1101 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1102 arvtool.formatgraph = None
1103 it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1105 next(it).run(runtimeContext)
1106 next(it).run(runtimeContext)
1108 with open("tests/wf/echo-subwf.cwl") as f:
1109 subwf = StripYAMLComments(f.read())
1111 runner.api.container_requests().create.assert_called_with(
1112 body=JsonDiffMatcher({
1114 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1115 'scheduling_parameters': {},
1116 'name': u'echo-subwf',
1117 'secret_mounts': {},
1118 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1122 '/var/spool/cwl/cwl.input.yml': {
1123 'portable_data_hash': '99999999999999999999999999999996+99',
1124 'kind': 'collection',
1125 'path': 'cwl.input.yml'
1127 '/var/spool/cwl/workflow.cwl': {
1128 'portable_data_hash': '99999999999999999999999999999996+99',
1129 'kind': 'collection',
1130 'path': 'workflow.cwl'
1133 'path': '/var/spool/cwl/cwl.output.json',
1138 'capacity': 1073741824
1139 }, '/var/spool/cwl': {
1141 'capacity': 3221225472
1144 'state': 'Committed',
1145 'output_path': '/var/spool/cwl',
1146 'container_image': '99999999999999999999999999999993+99',
1151 u'--preserve-entire-environment',
1155 'use_existing': True,
1156 'output_name': u'Output for step echo-subwf',
1157 'cwd': '/var/spool/cwl',
1158 'output_storage_classes': ["default"]
1161 def test_default_work_api(self):
1162 arvados_cwl.add_arv_hints()
1164 api = mock.MagicMock()
1165 api._rootDesc = copy.deepcopy(get_rootDesc())
1166 runner = arvados_cwl.executor.ArvCwlExecutor(api)
1167 self.assertEqual(runner.work_api, 'containers')