1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from builtins import str
6 from builtins import object
9 import arvados_cwl.context
10 import arvados_cwl.util
11 from arvados_cwl.arvdocker import arv_docker_clear_cache
20 import cwltool.process
21 import cwltool.secrets
22 import cwltool.load_tool
23 from cwltool.update import INTERNAL_VERSION
24 from schema_salad.ref_resolver import Loader
25 from schema_salad.sourceline import cmap
27 from .matcher import JsonDiffMatcher, StripYAMLComments
28 from .mock_discovery import get_rootDesc
30 if not os.getenv('ARVADOS_DEBUG'):
31 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
32 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
34 class CollectionMock(object):
35 def __init__(self, vwdmock, *args, **kwargs):
36 self.vwdmock = vwdmock
39 def open(self, *args, **kwargs):
41 return self.vwdmock.open(*args, **kwargs)
43 def copy(self, *args, **kwargs):
45 self.vwdmock.copy(*args, **kwargs)
47 def save_new(self, *args, **kwargs):
53 def portable_data_hash(self):
55 return arvados.config.EMPTY_BLOCK_LOCATOR
57 return "99999999999999999999999999999996+99"
60 class TestContainer(unittest.TestCase):
63 cwltool.process._names = set()
64 arv_docker_clear_cache()
66 def helper(self, runner, enable_reuse=True):
67 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
69 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
70 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
71 fs_access = mock.MagicMock()
72 fs_access.exists.return_value = True
74 loadingContext = arvados_cwl.context.ArvLoadingContext(
75 {"avsc_names": avsc_names,
77 "make_fs_access": make_fs_access,
78 "construct_tool_object": runner.arv_make_tool,
79 "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
81 "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
83 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
84 {"work_api": "containers",
86 "name": "test_run_"+str(enable_reuse),
87 "make_fs_access": make_fs_access,
90 "enable_reuse": enable_reuse,
92 "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
93 "workflow_eval_lock": threading.Condition(threading.RLock())
96 if isinstance(runner, mock.MagicMock):
97 def make_tool(toolpath_object, loadingContext):
98 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
99 runner.arv_make_tool.side_effect = make_tool
101 return loadingContext, runtimeContext
103 # Helper function to set up the ArvCwlExecutor to use the containers api
104 # and test that the RuntimeStatusLoggingHandler is set up correctly
105 def setup_and_test_container_executor_and_logging(self, gcc_mock) :
106 api = mock.MagicMock()
107 api._rootDesc = copy.deepcopy(get_rootDesc())
109 # Make sure ArvCwlExecutor thinks it's running inside a container so it
110 # adds the logging handler that will call runtime_status_update() mock
111 self.assertFalse(gcc_mock.called)
112 runner = arvados_cwl.ArvCwlExecutor(api)
113 self.assertEqual(runner.work_api, 'containers')
114 root_logger = logging.getLogger('')
115 handlerClasses = [h.__class__ for h in root_logger.handlers]
116 self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
119 # The test passes no builder.resources
120 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
121 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
122 def test_run(self, keepdocker):
123 for enable_reuse in (True, False):
124 arv_docker_clear_cache()
126 runner = mock.MagicMock()
127 runner.ignore_docker_for_reuse = False
128 runner.intermediate_output_ttl = 0
129 runner.secret_store = cwltool.secrets.SecretStore()
130 runner.api._rootDesc = {"revision": "20210628"}
132 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
133 runner.api.collections().get().execute.return_value = {
134 "portable_data_hash": "99999999999999999999999999999993+99"}
140 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
142 "class": "CommandLineTool",
146 loadingContext, runtimeContext = self.helper(runner, enable_reuse)
148 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
149 arvtool.formatgraph = None
151 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
152 j.run(runtimeContext)
153 runner.api.container_requests().create.assert_called_with(
154 body=JsonDiffMatcher({
156 'HOME': '/var/spool/cwl',
159 'name': 'test_run_'+str(enable_reuse),
160 'runtime_constraints': {
164 'use_existing': enable_reuse,
167 '/tmp': {'kind': 'tmp',
168 "capacity": 1073741824
170 '/var/spool/cwl': {'kind': 'tmp',
171 "capacity": 1073741824 }
173 'state': 'Committed',
174 'output_name': 'Output for step test_run_'+str(enable_reuse),
175 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
176 'output_path': '/var/spool/cwl',
178 'container_image': '99999999999999999999999999999993+99',
179 'command': ['ls', '/var/spool/cwl'],
180 'cwd': '/var/spool/cwl',
181 'scheduling_parameters': {},
184 'output_storage_classes': ["default"]
187 # The test passes some fields in builder.resources
188 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
189 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
190 def test_resource_requirements(self, keepdocker):
191 arvados_cwl.add_arv_hints()
192 runner = mock.MagicMock()
193 runner.ignore_docker_for_reuse = False
194 runner.intermediate_output_ttl = 3600
195 runner.secret_store = cwltool.secrets.SecretStore()
196 runner.api._rootDesc = {"revision": "20210628"}
198 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
199 runner.api.collections().get().execute.return_value = {
200 "portable_data_hash": "99999999999999999999999999999993+99"}
206 "class": "ResourceRequirement",
212 "class": "http://arvados.org/cwl#RuntimeConstraints",
215 "class": "http://arvados.org/cwl#APIRequirement",
217 "class": "http://arvados.org/cwl#PartitionRequirement",
220 "class": "http://arvados.org/cwl#IntermediateOutput",
223 "class": "WorkReuse",
228 "class": "CommandLineTool",
232 loadingContext, runtimeContext = self.helper(runner)
233 runtimeContext.name = "test_resource_requirements"
235 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
236 arvtool.formatgraph = None
237 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
238 j.run(runtimeContext)
240 call_args, call_kwargs = runner.api.container_requests().create.call_args
242 call_body_expected = {
244 'HOME': '/var/spool/cwl',
247 'name': 'test_resource_requirements',
248 'runtime_constraints': {
251 'keep_cache_ram': 536870912,
254 'use_existing': False,
257 '/tmp': {'kind': 'tmp',
258 "capacity": 4194304000 },
259 '/var/spool/cwl': {'kind': 'tmp',
260 "capacity": 5242880000 }
262 'state': 'Committed',
263 'output_name': 'Output for step test_resource_requirements',
264 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
265 'output_path': '/var/spool/cwl',
267 'container_image': '99999999999999999999999999999993+99',
269 'cwd': '/var/spool/cwl',
270 'scheduling_parameters': {
271 'partitions': ['blurb']
275 'output_storage_classes': ["default"]
278 call_body = call_kwargs.get('body', None)
279 self.assertNotEqual(None, call_body)
280 for key in call_body:
281 self.assertEqual(call_body_expected.get(key), call_body.get(key))
284 # The test passes some fields in builder.resources
285 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
286 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
287 @mock.patch("arvados.collection.Collection")
288 def test_initial_work_dir(self, collection_mock, keepdocker):
289 runner = mock.MagicMock()
290 runner.ignore_docker_for_reuse = False
291 runner.intermediate_output_ttl = 0
292 runner.secret_store = cwltool.secrets.SecretStore()
293 runner.api._rootDesc = {"revision": "20210628"}
295 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
296 runner.api.collections().get().execute.return_value = {
297 "portable_data_hash": "99999999999999999999999999999993+99"}
299 sourcemock = mock.MagicMock()
300 def get_collection_mock(p):
302 return (sourcemock, p.split("/", 1)[1])
304 return (sourcemock, "")
305 runner.fs_access.get_collection.side_effect = get_collection_mock
307 vwdmock = mock.MagicMock()
308 collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
314 "class": "InitialWorkDirRequirement",
318 "location": "keep:99999999999999999999999999999995+99/bar"
321 "class": "Directory",
323 "location": "keep:99999999999999999999999999999995+99"
327 "basename": "filename",
328 "location": "keep:99999999999999999999999999999995+99/baz/filename"
331 "class": "Directory",
332 "basename": "subdir",
333 "location": "keep:99999999999999999999999999999995+99/subdir"
337 "class": "CommandLineTool",
338 "cwlVersion": "v1.2",
342 loadingContext, runtimeContext = self.helper(runner)
343 runtimeContext.name = "test_initial_work_dir"
345 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
347 arvtool.formatgraph = None
348 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
349 j.run(runtimeContext)
351 call_args, call_kwargs = runner.api.container_requests().create.call_args
353 vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
354 vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
355 vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
356 vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
358 call_body_expected = {
360 'HOME': '/var/spool/cwl',
363 'name': 'test_initial_work_dir',
364 'runtime_constraints': {
368 'use_existing': True,
371 '/tmp': {'kind': 'tmp',
372 "capacity": 1073741824 },
373 '/var/spool/cwl': {'kind': 'tmp',
374 "capacity": 1073741824 },
375 '/var/spool/cwl/foo': {
376 'kind': 'collection',
378 'portable_data_hash': '99999999999999999999999999999996+99'
380 '/var/spool/cwl/foo2': {
381 'kind': 'collection',
383 'portable_data_hash': '99999999999999999999999999999996+99'
385 '/var/spool/cwl/filename': {
386 'kind': 'collection',
388 'portable_data_hash': '99999999999999999999999999999996+99'
390 '/var/spool/cwl/subdir': {
391 'kind': 'collection',
393 'portable_data_hash': '99999999999999999999999999999996+99'
396 'state': 'Committed',
397 'output_name': 'Output for step test_initial_work_dir',
398 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
399 'output_path': '/var/spool/cwl',
401 'container_image': '99999999999999999999999999999993+99',
403 'cwd': '/var/spool/cwl',
404 'scheduling_parameters': {
408 'output_storage_classes': ["default"]
411 call_body = call_kwargs.get('body', None)
412 self.assertNotEqual(None, call_body)
413 for key in call_body:
414 self.assertEqual(call_body_expected.get(key), call_body.get(key))
417 # Test redirecting stdin/stdout/stderr
418 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
419 def test_redirects(self, keepdocker):
420 runner = mock.MagicMock()
421 runner.ignore_docker_for_reuse = False
422 runner.intermediate_output_ttl = 0
423 runner.secret_store = cwltool.secrets.SecretStore()
424 runner.api._rootDesc = {"revision": "20210628"}
426 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
427 runner.api.collections().get().execute.return_value = {
428 "portable_data_hash": "99999999999999999999999999999993+99"}
430 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
436 "stdout": "stdout.txt",
437 "stderr": "stderr.txt",
438 "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
439 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
441 "class": "CommandLineTool",
445 loadingContext, runtimeContext = self.helper(runner)
446 runtimeContext.name = "test_run_redirect"
448 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
449 arvtool.formatgraph = None
450 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
451 j.run(runtimeContext)
452 runner.api.container_requests().create.assert_called_with(
453 body=JsonDiffMatcher({
455 'HOME': '/var/spool/cwl',
458 'name': 'test_run_redirect',
459 'runtime_constraints': {
463 'use_existing': True,
466 '/tmp': {'kind': 'tmp',
467 "capacity": 1073741824 },
468 '/var/spool/cwl': {'kind': 'tmp',
469 "capacity": 1073741824 },
472 "path": "/var/spool/cwl/stderr.txt"
475 "kind": "collection",
477 "portable_data_hash": "99999999999999999999999999999996+99"
481 "path": "/var/spool/cwl/stdout.txt"
484 'state': 'Committed',
485 "output_name": "Output for step test_run_redirect",
486 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
487 'output_path': '/var/spool/cwl',
489 'container_image': '99999999999999999999999999999993+99',
490 'command': ['ls', '/var/spool/cwl'],
491 'cwd': '/var/spool/cwl',
492 'scheduling_parameters': {},
495 'output_storage_classes': ["default"]
498 @mock.patch("arvados.collection.Collection")
499 def test_done(self, col):
500 api = mock.MagicMock()
502 runner = mock.MagicMock()
504 runner.num_retries = 0
505 runner.ignore_docker_for_reuse = False
506 runner.intermediate_output_ttl = 0
507 runner.secret_store = cwltool.secrets.SecretStore()
509 runner.api.containers().get().execute.return_value = {"state":"Complete",
513 col().open.return_value = []
515 loadingContext, runtimeContext = self.helper(runner)
517 arvjob = arvados_cwl.ArvadosContainer(runner,
525 arvjob.output_callback = mock.MagicMock()
526 arvjob.collect_outputs = mock.MagicMock()
527 arvjob.successCodes = [0]
528 arvjob.outdir = "/var/spool/cwl"
529 arvjob.output_ttl = 3600
531 arvjob.collect_outputs.return_value = {"out": "stuff"}
535 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
536 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
537 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
538 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
539 "modified_at": "2017-05-26T12:01:22Z"
542 self.assertFalse(api.collections().create.called)
543 self.assertFalse(runner.runtime_status_error.called)
545 arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
546 arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
547 runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
549 # Test to make sure we dont call runtime_status_update if we already did
550 # some where higher up in the call stack
551 @mock.patch("arvados_cwl.util.get_current_container")
552 def test_recursive_runtime_status_update(self, gcc_mock):
553 self.setup_and_test_container_executor_and_logging(gcc_mock)
554 root_logger = logging.getLogger('')
556 # get_current_container is invoked when we call runtime_status_update
557 # so try and log again!
558 gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
560 root_logger.error("First Error")
562 self.fail("RuntimeStatusLoggingHandler should not be called recursively")
565 # Test to make sure that an exception raised from
566 # get_current_container doesn't cause the logger to raise an
568 @mock.patch("arvados_cwl.util.get_current_container")
569 def test_runtime_status_get_current_container_exception(self, gcc_mock):
570 self.setup_and_test_container_executor_and_logging(gcc_mock)
571 root_logger = logging.getLogger('')
573 # get_current_container is invoked when we call
574 # runtime_status_update, it is going to also raise an
576 gcc_mock.side_effect = Exception("Second Error")
578 root_logger.error("First Error")
580 self.fail("Exception in logger should not propagate")
581 self.assertTrue(gcc_mock.called)
583 @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
584 @mock.patch("arvados_cwl.util.get_current_container")
585 @mock.patch("arvados.collection.CollectionReader")
586 @mock.patch("arvados.collection.Collection")
587 def test_child_failure(self, col, reader, gcc_mock, rts_mock):
588 runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
590 gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
591 self.assertTrue(gcc_mock.called)
593 runner.num_retries = 0
594 runner.ignore_docker_for_reuse = False
595 runner.intermediate_output_ttl = 0
596 runner.secret_store = cwltool.secrets.SecretStore()
597 runner.label = mock.MagicMock()
598 runner.label.return_value = '[container testjob]'
600 runner.api.containers().get().execute.return_value = {
607 col().open.return_value = []
609 loadingContext, runtimeContext = self.helper(runner)
611 arvjob = arvados_cwl.ArvadosContainer(runner,
619 arvjob.output_callback = mock.MagicMock()
620 arvjob.collect_outputs = mock.MagicMock()
621 arvjob.successCodes = [0]
622 arvjob.outdir = "/var/spool/cwl"
623 arvjob.output_ttl = 3600
624 arvjob.collect_outputs.return_value = {"out": "stuff"}
628 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
629 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
630 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
631 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
632 "modified_at": "2017-05-26T12:01:22Z"
635 rts_mock.assert_called_with(
637 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
638 ' ** log is empty **'
640 arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
642 # The test passes no builder.resources
643 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
644 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
645 def test_mounts(self, keepdocker):
646 runner = mock.MagicMock()
647 runner.ignore_docker_for_reuse = False
648 runner.intermediate_output_ttl = 0
649 runner.secret_store = cwltool.secrets.SecretStore()
650 runner.api._rootDesc = {"revision": "20210628"}
652 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
653 runner.api.collections().get().execute.return_value = {
654 "portable_data_hash": "99999999999999999999999999999994+99",
655 "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
657 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
666 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
668 "class": "CommandLineTool",
672 loadingContext, runtimeContext = self.helper(runner)
673 runtimeContext.name = "test_run_mounts"
675 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
676 arvtool.formatgraph = None
679 "class": "Directory",
680 "location": "keep:99999999999999999999999999999994+44",
681 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
685 "location": "keep:99999999999999999999999999999994+44/file1",
689 "location": "keep:99999999999999999999999999999994+44/file2",
694 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
695 j.run(runtimeContext)
696 runner.api.container_requests().create.assert_called_with(
697 body=JsonDiffMatcher({
699 'HOME': '/var/spool/cwl',
702 'name': 'test_run_mounts',
703 'runtime_constraints': {
707 'use_existing': True,
710 "/keep/99999999999999999999999999999994+44": {
711 "kind": "collection",
712 "portable_data_hash": "99999999999999999999999999999994+44",
713 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
715 '/tmp': {'kind': 'tmp',
716 "capacity": 1073741824 },
717 '/var/spool/cwl': {'kind': 'tmp',
718 "capacity": 1073741824 }
720 'state': 'Committed',
721 'output_name': 'Output for step test_run_mounts',
722 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
723 'output_path': '/var/spool/cwl',
725 'container_image': '99999999999999999999999999999994+99',
726 'command': ['ls', '/var/spool/cwl'],
727 'cwd': '/var/spool/cwl',
728 'scheduling_parameters': {},
731 'output_storage_classes': ["default"]
734 # The test passes no builder.resources
735 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
736 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
737 def test_secrets(self, keepdocker):
738 arvados_cwl.add_arv_hints()
739 runner = mock.MagicMock()
740 runner.ignore_docker_for_reuse = False
741 runner.intermediate_output_ttl = 0
742 runner.secret_store = cwltool.secrets.SecretStore()
743 runner.api._rootDesc = {"revision": "20210628"}
745 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
746 runner.api.collections().get().execute.return_value = {
747 "portable_data_hash": "99999999999999999999999999999993+99"}
749 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
751 tool = cmap({"arguments": ["md5sum", "example.conf"],
752 "class": "CommandLineTool",
753 "cwlVersion": "v1.2",
756 "class": "http://commonwl.org/cwltool#Secrets",
765 "id": "#secret_job.cwl/pw",
773 "class": "InitialWorkDirRequirement",
776 "entry": "username: user\npassword: $(inputs.pw)\n",
777 "entryname": "example.conf"
783 loadingContext, runtimeContext = self.helper(runner)
784 runtimeContext.name = "test_secrets"
786 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
787 arvtool.formatgraph = None
789 job_order = {"pw": "blorp"}
790 runner.secret_store.store(["pw"], job_order)
792 for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
793 j.run(runtimeContext)
794 runner.api.container_requests().create.assert_called_with(
795 body=JsonDiffMatcher({
797 'HOME': '/var/spool/cwl',
800 'name': 'test_secrets',
801 'runtime_constraints': {
805 'use_existing': True,
808 '/tmp': {'kind': 'tmp',
809 "capacity": 1073741824
811 '/var/spool/cwl': {'kind': 'tmp',
812 "capacity": 1073741824 }
814 'state': 'Committed',
815 'output_name': 'Output for step test_secrets',
816 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
817 'output_path': '/var/spool/cwl',
819 'container_image': '99999999999999999999999999999993+99',
820 'command': ['md5sum', 'example.conf'],
821 'cwd': '/var/spool/cwl',
822 'scheduling_parameters': {},
825 "/var/spool/cwl/example.conf": {
826 "content": "username: user\npassword: blorp\n",
830 'output_storage_classes': ["default"]
833 # The test passes no builder.resources
834 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
835 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
836 def test_timelimit(self, keepdocker):
837 runner = mock.MagicMock()
838 runner.ignore_docker_for_reuse = False
839 runner.intermediate_output_ttl = 0
840 runner.secret_store = cwltool.secrets.SecretStore()
841 runner.api._rootDesc = {"revision": "20210628"}
843 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
844 runner.api.collections().get().execute.return_value = {
845 "portable_data_hash": "99999999999999999999999999999993+99"}
851 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
853 "cwlVersion": "v1.2",
854 "class": "CommandLineTool",
857 "class": "ToolTimeLimit",
863 loadingContext, runtimeContext = self.helper(runner)
864 runtimeContext.name = "test_timelimit"
866 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
867 arvtool.formatgraph = None
869 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
870 j.run(runtimeContext)
872 _, kwargs = runner.api.container_requests().create.call_args
873 self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
876 # The test passes no builder.resources
877 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
878 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
879 def test_setting_storage_class(self, keepdocker):
880 arv_docker_clear_cache()
882 runner = mock.MagicMock()
883 runner.ignore_docker_for_reuse = False
884 runner.intermediate_output_ttl = 0
885 runner.secret_store = cwltool.secrets.SecretStore()
886 runner.api._rootDesc = {"revision": "20210628"}
888 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
889 runner.api.collections().get().execute.return_value = {
890 "portable_data_hash": "99999999999999999999999999999993+99"}
896 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
898 "cwlVersion": "v1.2",
899 "class": "CommandLineTool",
902 "class": "http://arvados.org/cwl#OutputStorageClass",
903 "finalStorageClass": ["baz_sc", "qux_sc"],
904 "intermediateStorageClass": ["foo_sc", "bar_sc"]
909 loadingContext, runtimeContext = self.helper(runner, True)
911 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
912 arvtool.formatgraph = None
914 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
915 j.run(runtimeContext)
916 runner.api.container_requests().create.assert_called_with(
917 body=JsonDiffMatcher({
919 'HOME': '/var/spool/cwl',
922 'name': 'test_run_True',
923 'runtime_constraints': {
927 'use_existing': True,
930 '/tmp': {'kind': 'tmp',
931 "capacity": 1073741824
933 '/var/spool/cwl': {'kind': 'tmp',
934 "capacity": 1073741824 }
936 'state': 'Committed',
937 'output_name': 'Output for step test_run_True',
938 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
939 'output_path': '/var/spool/cwl',
941 'container_image': '99999999999999999999999999999993+99',
942 'command': ['ls', '/var/spool/cwl'],
943 'cwd': '/var/spool/cwl',
944 'scheduling_parameters': {},
947 'output_storage_classes': ["foo_sc", "bar_sc"]
951 # The test passes no builder.resources
952 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
953 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
954 def test_setting_process_properties(self, keepdocker):
955 arv_docker_clear_cache()
957 runner = mock.MagicMock()
958 runner.ignore_docker_for_reuse = False
959 runner.intermediate_output_ttl = 0
960 runner.secret_store = cwltool.secrets.SecretStore()
961 runner.api._rootDesc = {"revision": "20210628"}
963 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
964 runner.api.collections().get().execute.return_value = {
965 "portable_data_hash": "99999999999999999999999999999993+99"}
969 {"id": "x", "type": "string"}],
972 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
974 "class": "CommandLineTool",
975 "cwlVersion": "v1.2",
978 "class": "http://arvados.org/cwl#ProcessProperties",
979 "processProperties": [
980 {"propertyName": "foo",
981 "propertyValue": "bar"},
982 {"propertyName": "baz",
983 "propertyValue": "$(inputs.x)"},
984 {"propertyName": "quux",
995 loadingContext, runtimeContext = self.helper(runner, True)
997 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
998 arvtool.formatgraph = None
1000 for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1001 j.run(runtimeContext)
1002 runner.api.container_requests().create.assert_called_with(
1003 body=JsonDiffMatcher({
1005 'HOME': '/var/spool/cwl',
1008 'name': 'test_run_True',
1009 'runtime_constraints': {
1013 'use_existing': True,
1016 '/tmp': {'kind': 'tmp',
1017 "capacity": 1073741824
1019 '/var/spool/cwl': {'kind': 'tmp',
1020 "capacity": 1073741824 }
1022 'state': 'Committed',
1023 'output_name': 'Output for step test_run_True',
1024 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1025 'output_path': '/var/spool/cwl',
1027 'container_image': '99999999999999999999999999999993+99',
1028 'command': ['ls', '/var/spool/cwl'],
1029 'cwd': '/var/spool/cwl',
1030 'scheduling_parameters': {},
1039 'secret_mounts': {},
1040 'output_storage_classes': ["default"]
1044 # The test passes no builder.resources
1045 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1046 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1047 def test_cuda_requirement(self, keepdocker):
1048 arvados_cwl.add_arv_hints()
1049 arv_docker_clear_cache()
1051 runner = mock.MagicMock()
1052 runner.ignore_docker_for_reuse = False
1053 runner.intermediate_output_ttl = 0
1054 runner.secret_store = cwltool.secrets.SecretStore()
1055 runner.api._rootDesc = {"revision": "20210628"}
1057 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1058 runner.api.collections().get().execute.return_value = {
1059 "portable_data_hash": "99999999999999999999999999999993+99"}
1062 "class": "http://commonwl.org/cwltool#CUDARequirement",
1063 "cudaVersionMin": "11.0",
1064 "cudaComputeCapability": "9.0",
1066 "class": "http://commonwl.org/cwltool#CUDARequirement",
1067 "cudaVersionMin": "11.0",
1068 "cudaComputeCapability": "9.0",
1069 "cudaDeviceCountMin": 2
1071 "class": "http://commonwl.org/cwltool#CUDARequirement",
1072 "cudaVersionMin": "11.0",
1073 "cudaComputeCapability": ["4.0", "5.0"],
1074 "cudaDeviceCountMin": 2
1079 'driver_version': "11.0",
1080 'hardware_capability': "9.0"
1083 'driver_version': "11.0",
1084 'hardware_capability': "9.0"
1087 'driver_version': "11.0",
1088 'hardware_capability': "4.0"
1091 for test_case in range(0, len(test_cwl_req)):
1096 "baseCommand": "nvidia-smi",
1099 "cwlVersion": "v1.2",
1100 "class": "CommandLineTool",
1101 "requirements": [test_cwl_req[test_case]]
1104 loadingContext, runtimeContext = self.helper(runner, True)
1106 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1107 arvtool.formatgraph = None
1109 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1110 j.run(runtimeContext)
1111 runner.api.container_requests().create.assert_called_with(
1112 body=JsonDiffMatcher({
1114 'HOME': '/var/spool/cwl',
1117 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1118 'runtime_constraints': {
1121 'cuda': test_arv_req[test_case]
1123 'use_existing': True,
1126 '/tmp': {'kind': 'tmp',
1127 "capacity": 1073741824
1129 '/var/spool/cwl': {'kind': 'tmp',
1130 "capacity": 1073741824 }
1132 'state': 'Committed',
1133 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1134 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1135 'output_path': '/var/spool/cwl',
1137 'container_image': '99999999999999999999999999999993+99',
1138 'command': ['nvidia-smi'],
1139 'cwd': '/var/spool/cwl',
1140 'scheduling_parameters': {},
1142 'secret_mounts': {},
1143 'output_storage_classes': ["default"]
1147 # The test passes no builder.resources
1148 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1149 @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1150 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1151 def test_match_local_docker(self, keepdocker, determine_image_id):
1152 arvados_cwl.add_arv_hints()
1153 arv_docker_clear_cache()
1155 runner = mock.MagicMock()
1156 runner.ignore_docker_for_reuse = False
1157 runner.intermediate_output_ttl = 0
1158 runner.secret_store = cwltool.secrets.SecretStore()
1159 runner.api._rootDesc = {"revision": "20210628"}
1161 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1162 ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1163 determine_image_id.side_effect = lambda x: "123"
1165 ex = mock.MagicMock()
1166 lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1167 "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1168 ex.execute.return_value = lookup[uuid]
1170 runner.api.collections().get.side_effect = execute
1175 "baseCommand": "echo",
1178 "cwlVersion": "v1.0",
1179 "class": "org.w3id.cwl.cwl.CommandLineTool"
1182 loadingContext, runtimeContext = self.helper(runner, True)
1184 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1185 arvtool.formatgraph = None
1187 container_request = {
1189 'HOME': '/var/spool/cwl',
1192 'name': 'test_run_True',
1193 'runtime_constraints': {
1197 'use_existing': True,
1200 '/tmp': {'kind': 'tmp',
1201 "capacity": 1073741824
1203 '/var/spool/cwl': {'kind': 'tmp',
1204 "capacity": 1073741824 }
1206 'state': 'Committed',
1207 'output_name': 'Output for step test_run_True',
1208 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1209 'output_path': '/var/spool/cwl',
1211 'container_image': '99999999999999999999999999999994+99',
1212 'command': ['echo'],
1213 'cwd': '/var/spool/cwl',
1214 'scheduling_parameters': {},
1216 'secret_mounts': {},
1217 'output_storage_classes': ["default"]
1220 runtimeContext.match_local_docker = False
1221 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1222 j.run(runtimeContext)
1223 runner.api.container_requests().create.assert_called_with(
1224 body=JsonDiffMatcher(container_request))
1226 arv_docker_clear_cache()
1227 runtimeContext.match_local_docker = True
1228 container_request['container_image'] = '99999999999999999999999999999993+99'
1229 container_request['name'] = 'test_run_True_2'
1230 container_request['output_name'] = 'Output for step test_run_True_2'
1231 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1232 j.run(runtimeContext)
1233 runner.api.container_requests().create.assert_called_with(
1234 body=JsonDiffMatcher(container_request))
1238 class TestWorkflow(unittest.TestCase):
1240 cwltool.process._names = set()
1241 arv_docker_clear_cache()
1243 def helper(self, runner, enable_reuse=True):
1244 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1246 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1247 collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1249 document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1250 document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1251 document_loader.fetch_text = document_loader.fetcher.fetch_text
1252 document_loader.check_exists = document_loader.fetcher.check_exists
1254 loadingContext = arvados_cwl.context.ArvLoadingContext(
1255 {"avsc_names": avsc_names,
1257 "make_fs_access": make_fs_access,
1258 "loader": document_loader,
1259 "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1260 "construct_tool_object": runner.arv_make_tool})
1261 runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1262 {"work_api": "containers",
1264 "name": "test_run_wf_"+str(enable_reuse),
1265 "make_fs_access": make_fs_access,
1267 "enable_reuse": enable_reuse,
1270 return loadingContext, runtimeContext
1272 # The test passes no builder.resources
1273 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1274 @mock.patch("arvados.collection.CollectionReader")
1275 @mock.patch("arvados.collection.Collection")
1276 @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1277 def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1278 arvados_cwl.add_arv_hints()
1280 api = mock.MagicMock()
1281 api._rootDesc = get_rootDesc()
1283 runner = arvados_cwl.executor.ArvCwlExecutor(api)
1284 self.assertEqual(runner.work_api, 'containers')
1286 list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1287 runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1288 runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1289 "portable_data_hash": "99999999999999999999999999999993+99"}]}
1291 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1292 runner.ignore_docker_for_reuse = False
1293 runner.num_retries = 0
1294 runner.secret_store = cwltool.secrets.SecretStore()
1296 loadingContext, runtimeContext = self.helper(runner)
1297 runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1299 mockcollectionreader().exists.return_value = True
1301 tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1302 metadata["cwlVersion"] = tool["cwlVersion"]
1304 mockc = mock.MagicMock()
1305 mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1306 mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1308 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1309 arvtool.formatgraph = None
1310 it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1312 next(it).run(runtimeContext)
1313 next(it).run(runtimeContext)
1315 with open("tests/wf/scatter2_subwf.cwl") as f:
1316 subwf = StripYAMLComments(f.read()).rstrip()
1318 runner.api.container_requests().create.assert_called_with(
1319 body=JsonDiffMatcher({
1324 "--preserve-entire-environment",
1328 "container_image": "99999999999999999999999999999993+99",
1329 "cwd": "/var/spool/cwl",
1331 "HOME": "/var/spool/cwl",
1335 "/keep/99999999999999999999999999999999+118": {
1336 "kind": "collection",
1337 "portable_data_hash": "99999999999999999999999999999999+118"
1340 "capacity": 1073741824,
1344 "capacity": 1073741824,
1347 "/var/spool/cwl/cwl.input.yml": {
1348 "kind": "collection",
1349 "path": "cwl.input.yml",
1350 "portable_data_hash": "99999999999999999999999999999996+99"
1352 "/var/spool/cwl/workflow.cwl": {
1353 "kind": "collection",
1354 "path": "workflow.cwl",
1355 "portable_data_hash": "99999999999999999999999999999996+99"
1359 "path": "/var/spool/cwl/cwl.output.json"
1362 "name": "scatterstep",
1363 "output_name": "Output for step scatterstep",
1364 "output_path": "/var/spool/cwl",
1368 "runtime_constraints": {
1372 "scheduling_parameters": {},
1373 "secret_mounts": {},
1374 "state": "Committed",
1375 "use_existing": True,
1376 'output_storage_classes': ["default"]
1378 mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1379 mockc.open().__enter__().write.assert_has_calls([mock.call(
1382 "basename": "token.txt",
1384 "location": "/keep/99999999999999999999999999999999+118/token.txt",
1390 # The test passes no builder.resources
1391 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1392 @mock.patch("arvados.collection.CollectionReader")
1393 @mock.patch("arvados.collection.Collection")
1394 @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1395 def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1396 arvados_cwl.add_arv_hints()
1398 api = mock.MagicMock()
1399 api._rootDesc = get_rootDesc()
1401 runner = arvados_cwl.executor.ArvCwlExecutor(api)
1402 self.assertEqual(runner.work_api, 'containers')
1404 list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1405 runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1406 "portable_data_hash": "99999999999999999999999999999993+99"}
1407 runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1408 "portable_data_hash": "99999999999999999999999999999993+99"}]}
1410 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1411 runner.ignore_docker_for_reuse = False
1412 runner.num_retries = 0
1413 runner.secret_store = cwltool.secrets.SecretStore()
1415 loadingContext, runtimeContext = self.helper(runner)
1416 runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1417 loadingContext.do_update = True
1418 tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1420 mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1422 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1423 arvtool.formatgraph = None
1424 it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1426 next(it).run(runtimeContext)
1427 next(it).run(runtimeContext)
1429 with open("tests/wf/echo-subwf.cwl") as f:
1430 subwf = StripYAMLComments(f.read())
1432 runner.api.container_requests().create.assert_called_with(
1433 body=JsonDiffMatcher({
1435 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1436 'scheduling_parameters': {},
1437 'name': u'echo-subwf',
1438 'secret_mounts': {},
1439 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1443 '/var/spool/cwl/cwl.input.yml': {
1444 'portable_data_hash': '99999999999999999999999999999996+99',
1445 'kind': 'collection',
1446 'path': 'cwl.input.yml'
1448 '/var/spool/cwl/workflow.cwl': {
1449 'portable_data_hash': '99999999999999999999999999999996+99',
1450 'kind': 'collection',
1451 'path': 'workflow.cwl'
1454 'path': '/var/spool/cwl/cwl.output.json',
1459 'capacity': 1073741824
1460 }, '/var/spool/cwl': {
1462 'capacity': 3221225472
1465 'state': 'Committed',
1466 'output_path': '/var/spool/cwl',
1467 'container_image': '99999999999999999999999999999993+99',
1472 u'--preserve-entire-environment',
1476 'use_existing': True,
1477 'output_name': u'Output for step echo-subwf',
1478 'cwd': '/var/spool/cwl',
1479 'output_storage_classes': ["default"]
1482 def test_default_work_api(self):
1483 arvados_cwl.add_arv_hints()
1485 api = mock.MagicMock()
1486 api._rootDesc = copy.deepcopy(get_rootDesc())
1487 runner = arvados_cwl.executor.ArvCwlExecutor(api)
1488 self.assertEqual(runner.work_api, 'containers')