From: Tom Clegg <tom@curii.com>
Date: Tue, 13 Dec 2022 21:38:54 +0000 (-0500)
Subject: 19847: Let server apply default disk cache size.
X-Git-Tag: 2.5.0~7^2~1
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/42dd3d670f9e9030145ee73e97d3ed2ba040bca5?hp=745a7e253326a2b8e292dbd928c76320ce320216

19847: Let server apply default disk cache size.

Use disk cache when DefaultKeepCacheRAM==0 (instead of
DefaultKeepCacheDisk>0).

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index c6030d7237..a15c085da7 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -264,7 +264,7 @@ class ArvadosContainer(JobBase):
         if api_req:
             runtime_constraints["API"] = True
 
-        use_disk_cache = (self.arvrunner.api.config()["Containers"].get("DefaultKeepCacheDisk", 0) > 0)
+        use_disk_cache = (self.arvrunner.api.config()["Containers"].get("DefaultKeepCacheRAM", 0) == 0)
 
         keep_cache_type_req, _ = self.get_requirement("http://arvados.org/cwl#KeepCacheTypeRequirement")
         if keep_cache_type_req:
@@ -276,7 +276,7 @@ class ArvadosContainer(JobBase):
         if runtime_req:
             if "keep_cache" in runtime_req:
                 if use_disk_cache:
-                    # If DefaultKeepCacheDisk is non-zero it means we should use disk cache.
+                    # If DefaultKeepCacheRAM is zero it means we should use disk cache.
                     runtime_constraints["keep_cache_disk"] = math.ceil(runtime_req["keep_cache"] * 2**20)
                 else:
                     runtime_constraints["keep_cache_ram"] = math.ceil(runtime_req["keep_cache"] * 2**20)
@@ -290,11 +290,6 @@ class ArvadosContainer(JobBase):
                         "writable": True
                     }
 
-        if use_disk_cache and "keep_cache_disk" not in runtime_constraints:
-            # Cache size wasn't explicitly set so set the default to
-            # 2 GB <= the size of the RAM request <= 32 GiB
-            runtime_constraints["keep_cache_disk"] = min(max(2*1024*1024*1024, runtime_constraints["ram"]), 32*1024*1024*1024)
-
         partition_req, _ = self.get_requirement("http://arvados.org/cwl#PartitionRequirement")
         if partition_req:
             scheduling_parameters["partitions"] = aslist(partition_req["partition"])
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index b8bc707822..bad4d44087 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -135,7 +135,7 @@ class TestContainer(unittest.TestCase):
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
             runner.api._rootDesc = {"revision": "20210628"}
-            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -202,7 +202,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 3600
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -300,7 +300,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -432,7 +432,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -666,7 +666,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -791,7 +791,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -890,7 +890,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -936,7 +936,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -1012,7 +1012,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -1108,7 +1108,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -1213,7 +1213,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
-        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
@@ -1305,7 +1305,7 @@ class TestContainer(unittest.TestCase):
                 runner.intermediate_output_ttl = 0
                 runner.secret_store = cwltool.secrets.SecretStore()
                 runner.api._rootDesc = {"revision": "20210628"}
-                runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+                runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
                 runner.api.collections().get().execute.return_value = {
@@ -1398,7 +1398,7 @@ class TestContainer(unittest.TestCase):
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
             runner.api._rootDesc = {"revision": rev}
-            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -1486,7 +1486,7 @@ class TestWorkflow(unittest.TestCase):
 
         api = mock.MagicMock()
         api._rootDesc = get_rootDesc()
-        api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         runner = arvados_cwl.executor.ArvCwlExecutor(api)
         self.assertEqual(runner.work_api, 'containers')
@@ -1619,7 +1619,7 @@ class TestWorkflow(unittest.TestCase):
 
         api = mock.MagicMock()
         api._rootDesc = get_rootDesc()
-        api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
+        api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         runner = arvados_cwl.executor.ArvCwlExecutor(api)
         self.assertEqual(runner.work_api, 'containers')