import functools
import cwltool.process
import cwltool.secrets
+from cwltool.update import INTERNAL_VERSION
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
class TestContainer(unittest.TestCase):
+ def setUp(self):
+ cwltool.process._names = set()
+ arv_docker_clear_cache()
+
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
def setup_and_test_container_executor_and_logging(self, gcc_mock) :
api = mock.MagicMock()
api._rootDesc = copy.deepcopy(get_rootDesc())
- del api._rootDesc.get('resources')['jobs']['methods']['create']
# Make sure ArvCwlExecutor thinks it's running inside a container so it
# adds the logging handler that will call runtime_status_update() mock
# For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_resource_requirements(self, keepdocker):
- arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@mock.patch("arvados.collection.Collection")
def test_initial_work_dir(self, collection_mock, keepdocker):
- arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
call_args, call_kwargs = runner.api.container_requests().create.call_args
vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
- vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
+ vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
# Test redirecting stdin/stdout/stderr
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_redirects(self, keepdocker):
- arv_docker_clear_cache()
-
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
tool = cmap({
"inputs": [],
except RuntimeError:
self.fail("RuntimeStatusLoggingHandler should not be called recursively")
+
+ # Test to make sure that an exception raised from
+ # get_current_container doesn't cause the logger to raise an
+ # exception
+ @mock.patch("arvados_cwl.util.get_current_container")
+ def test_runtime_status_get_current_container_exception(self, gcc_mock):
+ self.setup_and_test_container_executor_and_logging(gcc_mock)
+ root_logger = logging.getLogger('')
+
+ # get_current_container is invoked when we call
+ # runtime_status_update, it is going to also raise an
+ # exception.
+ gcc_mock.side_effect = Exception("Second Error")
+ try:
+ root_logger.error("First Error")
+ except Exception:
+ self.fail("Exception in logger should not propagate")
+ self.assertTrue(gcc_mock.called)
+
@mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
@mock.patch("arvados_cwl.util.get_current_container")
@mock.patch("arvados.collection.CollectionReader")
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_mounts(self, keepdocker):
- arv_docker_clear_cache()
-
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_secrets(self, keepdocker):
- arv_docker_clear_cache()
-
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_timelimit(self, keepdocker):
- arv_docker_clear_cache()
-
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
class TestWorkflow(unittest.TestCase):
+ def setUp(self):
+ cwltool.process._names = set()
+ arv_docker_clear_cache()
+
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
"basedir": "",
"make_fs_access": make_fs_access,
"loader": document_loader,
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"construct_tool_object": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
@mock.patch("arvados.collection.Collection")
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
- arv_docker_clear_cache()
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
loadingContext, runtimeContext = self.helper(runner)
runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
+ mockcollectionreader().exists.return_value = True
+
tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
"--no-container",
"--move-outputs",
"--preserve-entire-environment",
- "workflow.cwl#main",
+ "workflow.cwl",
"cwl.input.yml"
],
"container_image": "99999999999999999999999999999993+99",
}))
mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
mockc.open().__enter__().write.assert_has_calls([mock.call(
-bytes(b'''{
+'''{
"fileblub": {
"basename": "token.txt",
"class": "File",
"size": 0
},
"sleeptime": 5
-}'''))])
+}''')])
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.Collection")
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
- arv_docker_clear_cache()
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
u'--no-container',
u'--move-outputs',
u'--preserve-entire-environment',
- u'workflow.cwl#main',
+ u'workflow.cwl',
u'cwl.input.yml'
],
'use_existing': True,
api = mock.MagicMock()
api._rootDesc = copy.deepcopy(get_rootDesc())
- del api._rootDesc.get('resources')['jobs']['methods']['create']
runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
projects / arvados.git / blobdiff