X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/472090e584f8ce68c97e082aba08c79f2a242848..5d56a1af42f64df57ef7a1bcef6d016ff2310900:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index c270b9d383..09983f87a2 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -18,6 +18,7 @@ import os import functools import cwltool.process import cwltool.secrets +from cwltool.update import INTERNAL_VERSION from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap @@ -56,8 +57,12 @@ class CollectionMock(object): class TestContainer(unittest.TestCase): + def setUp(self): + cwltool.process._names = set() + arv_docker_clear_cache() + def helper(self, runner, enable_reuse=True): - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION) make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) @@ -66,7 +71,7 @@ class TestContainer(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "loader": Loader({}), - "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}}) + "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", "basedir": "", @@ -85,7 +90,6 @@ class TestContainer(unittest.TestCase): def setup_and_test_container_executor_and_logging(self, gcc_mock) : api = mock.MagicMock() api._rootDesc = copy.deepcopy(get_rootDesc()) - del api._rootDesc.get('resources')['jobs']['methods']['create'] # Make sure ArvCwlExecutor thinks it's running inside a container so it # adds the logging handler that will call runtime_status_update() mock @@ -166,7 +170,6 @@ class TestContainer(unittest.TestCase): # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_resource_requirements(self, keepdocker): - arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 3600 @@ -261,7 +264,6 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") @mock.patch("arvados.collection.Collection") def test_initial_work_dir(self, collection_mock, keepdocker): - arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 @@ -324,7 +326,7 @@ class TestContainer(unittest.TestCase): call_args, call_kwargs = runner.api.container_requests().create.call_args vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)]) - vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)]) + vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)]) @@ -389,8 +391,6 @@ class TestContainer(unittest.TestCase): # Test redirecting stdin/stdout/stderr @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_redirects(self, keepdocker): - arv_docker_clear_cache() - runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 @@ -400,7 +400,7 @@ class TestContainer(unittest.TestCase): runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99"} - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION) tool = cmap({ "inputs": [], @@ -532,6 +532,25 @@ class TestContainer(unittest.TestCase): except RuntimeError: self.fail("RuntimeStatusLoggingHandler should not be called recursively") + + # Test to make sure that an exception raised from + # get_current_container doesn't cause the logger to raise an + # exception + @mock.patch("arvados_cwl.util.get_current_container") + def test_runtime_status_get_current_container_exception(self, gcc_mock): + self.setup_and_test_container_executor_and_logging(gcc_mock) + root_logger = logging.getLogger('') + + # get_current_container is invoked when we call + # runtime_status_update, it is going to also raise an + # exception. + gcc_mock.side_effect = Exception("Second Error") + try: + root_logger.error("First Error") + except Exception: + self.fail("Exception in logger should not propagate") + self.assertTrue(gcc_mock.called) + @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update") @mock.patch("arvados_cwl.util.get_current_container") @mock.patch("arvados.collection.CollectionReader") @@ -595,8 +614,6 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_mounts(self, keepdocker): - arv_docker_clear_cache() - runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 @@ -686,8 +703,6 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_secrets(self, keepdocker): - arv_docker_clear_cache() - runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 @@ -783,8 +798,6 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_timelimit(self, keepdocker): - arv_docker_clear_cache() - runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 @@ -823,8 +836,12 @@ class TestContainer(unittest.TestCase): class TestWorkflow(unittest.TestCase): + def setUp(self): + cwltool.process._names = set() + arv_docker_clear_cache() + def helper(self, runner, enable_reuse=True): - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) @@ -839,7 +856,7 @@ class TestWorkflow(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "loader": document_loader, - "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}, + "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}, "construct_tool_object": runner.arv_make_tool}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", @@ -858,7 +875,6 @@ class TestWorkflow(unittest.TestCase): @mock.patch("arvados.collection.Collection") @mock.patch('arvados.commands.keepdocker.list_images_in_arv') def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader): - arv_docker_clear_cache() arvados_cwl.add_arv_hints() api = mock.MagicMock() @@ -880,6 +896,8 @@ class TestWorkflow(unittest.TestCase): loadingContext, runtimeContext = self.helper(runner) runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir) + mockcollectionreader().exists.return_value = True + tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl") metadata["cwlVersion"] = tool["cwlVersion"] @@ -904,7 +922,7 @@ class TestWorkflow(unittest.TestCase): "--no-container", "--move-outputs", "--preserve-entire-environment", - "workflow.cwl#main", + "workflow.cwl", "cwl.input.yml" ], "container_image": "99999999999999999999999999999993+99", @@ -958,7 +976,7 @@ class TestWorkflow(unittest.TestCase): })) mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)]) mockc.open().__enter__().write.assert_has_calls([mock.call( -bytes(b'''{ +'''{ "fileblub": { "basename": "token.txt", "class": "File", @@ -966,7 +984,7 @@ bytes(b'''{ "size": 0 }, "sleeptime": 5 -}'''))]) +}''')]) # The test passes no builder.resources # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @@ -974,7 +992,6 @@ bytes(b'''{ @mock.patch("arvados.collection.Collection") @mock.patch('arvados.commands.keepdocker.list_images_in_arv') def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader): - arv_docker_clear_cache() arvados_cwl.add_arv_hints() api = mock.MagicMock() @@ -1052,7 +1069,7 @@ bytes(b'''{ u'--no-container', u'--move-outputs', u'--preserve-entire-environment', - u'workflow.cwl#main', + u'workflow.cwl', u'cwl.input.yml' ], 'use_existing': True, @@ -1065,6 +1082,5 @@ bytes(b'''{ api = mock.MagicMock() api._rootDesc = copy.deepcopy(get_rootDesc()) - del api._rootDesc.get('resources')['jobs']['methods']['create'] runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers')