#
# SPDX-License-Identifier: Apache-2.0
-from builtins import str
-from builtins import object
-
import arvados_cwl
import arvados_cwl.context
import arvados_cwl.util
-from arvados_cwl.arvdocker import arv_docker_clear_cache
+#from arvados_cwl.arvdocker import arv_docker_clear_cache
import copy
import arvados.config
import logging
-import mock
import unittest
import os
import functools
from cwltool.update import INTERNAL_VERSION
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
+import io
+
+from unittest import mock
from .matcher import JsonDiffMatcher, StripYAMLComments
from .mock_discovery import get_rootDesc
def setUp(self):
cwltool.process._names = set()
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
def tearDown(self):
root_logger = logging.getLogger('')
"construct_tool_object": runner.arv_make_tool,
"fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
"loader": Loader({}),
- "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
+ "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
+ "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_run(self, keepdocker):
for enable_reuse in (True, False):
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
- runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
+
runner.api.containers().get().execute.return_value = {"state":"Complete",
"output": "abc+123",
"exit_code": 0}
- col().open.return_value = []
+ # Need to noop-out the close method otherwise it gets
+ # discarded when closed and we can't call getvalue() to check
+ # it.
+ class NoopCloseStringIO(io.StringIO):
+ def close(self):
+ pass
+
+ usage_report = NoopCloseStringIO()
+ def colreader_action(name, mode):
+ nonlocal usage_report
+ if name == "node.json":
+ return io.StringIO("""{
+ "ProviderType": "c5.large",
+ "VCPUs": 2,
+ "RAM": 4294967296,
+ "IncludedScratch": 8000000000000,
+ "AddedScratch": 0,
+ "Price": 0.085,
+ "Preemptible": false,
+ "CUDA": {
+ "DriverVersion": "",
+ "HardwareCapability": "",
+ "DeviceCount": 0
+ }
+}""")
+ if name == 'crunchstat.txt':
+ return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
+ if name == 'arv-mount.txt':
+ return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
+ if name == 'usage_report.html':
+ return usage_report
+ return None
+
+ col().open.side_effect = colreader_action
+ col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
loadingContext, runtimeContext = self.helper(runner)
"uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
"container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
"modified_at": "2017-05-26T12:01:22Z",
- "properties": {}
+ "properties": {},
+ "name": "testjob"
})
self.assertFalse(api.collections().create.called)
self.assertFalse(runner.runtime_status_error.called)
+ # Assert that something was written to the usage report
+ self.assertTrue(len(usage_report.getvalue()) > 0)
+
arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
"properties": {}
})
- rts_mock.assert_called_with(
- 'error',
- 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
- ' ** log is empty **'
- )
+ rts_mock.assert_has_calls([
+ mock.call('error',
+ 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
+ ' ** log is empty **'
+ ),
+ mock.call('warning',
+ 'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
+ )])
arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
# The test passes no builder.resources
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_setting_storage_class(self, keepdocker):
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_setting_process_properties(self, keepdocker):
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_cuda_requirement(self, keepdocker):
arvados_cwl.add_arv_hints()
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
@mock.patch("arvados.commands.keepdocker.list_images_in_arv")
def test_match_local_docker(self, keepdocker, determine_image_id):
arvados_cwl.add_arv_hints()
- arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(container_request))
- arv_docker_clear_cache()
+ runtimeContext.cached_docker_lookups.clear()
runtimeContext.match_local_docker = True
container_request['container_image'] = '99999999999999999999999999999993+99'
container_request['name'] = 'test_run_True_2'
arvados_cwl.add_arv_hints()
for enable_preemptible in (None, True, False):
for preemptible_hint in (None, True, False):
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
runner = mock.MagicMock()
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": "20210628"}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
runner.api._rootDesc = {"revision": rev}
+ runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
class TestWorkflow(unittest.TestCase):
def setUp(self):
cwltool.process._names = set()
- arv_docker_clear_cache()
+ #arv_docker_clear_cache()
def helper(self, runner, enable_reuse=True):
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
"make_fs_access": make_fs_access,
"loader": document_loader,
"metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
- "construct_tool_object": runner.arv_make_tool})
+ "construct_tool_object": runner.arv_make_tool,
+ "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
+ api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
+ api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
projects / arvados.git / blobdiff