Merge branch '20318-disk-cache'

closes #20318

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --combined doc/admin/upgrading.html.textile.liquid
index 6d37009e91d8a50f61cef5fa2267c20f73b348c2,7b8c20fdf9eaba632b0554dea7be38f5f30f560b..c80209822508e10c5dfd0d0c06b44e8c4e3dc12c
--- 1/doc/admin/upgrading.html.textile.liquid
--- 2/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@@ -32,10 -32,12 +32,16 @@@ h2(#main). development mai
  
  "previous: Upgrading to 2.7.1":#v2_7_1
  
+ h3. WebDAV service uses @/var/cache@ for file content
+ 
+ @keep-web@ now stores copies of recently accessed data blocks in @/var/cache/arvados/keep@ instead of in memory. That directory will be created automatically. The default cache size is 10% of the filesystem size. Use the new @Collections.WebDAVCache.DiskCacheSize@ config to specify a different percentage or an absolute size.
+ 
+ If the previously supported @MaxBlockEntries@ config is present, remove it to avoid warning messages at startup.
+ 
 +h3. Check MaxGatewayTunnels config
 +
 +If you use the LSF or Slurm dispatcher, ensure the new @API.MaxGatewayTunnels@ config entry is high enough to support the size of your cluster. See "LSF docs":{{site.baseurl}}/install/crunch2-lsf/install-dispatch.html#MaxGatewayTunnels or "Slurm docs":{{site.baseurl}}/install/crunch2-slurm/install-dispatch.html#MaxGatewayTunnels for details.
 +
  h2(#2_7_1). v2.7.1 (2023-12-12)
  
  "previous: Upgrading to 2.7.0":#v2_7_0

diff --combined lib/config/config.default.yml
index ddf7a01e90790909499bb9cd6cfa6ee9da0ccba0,872501f9153ea2201bfbea57aa65258a7c0b8b84..c8e854b7e26854a7cd24526cef82c66504957efa
--- 1/lib/config/config.default.yml
--- 2/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@@ -231,10 -231,6 +231,10 @@@ Clusters
        # also effectively limited by MaxConcurrentRailsRequests (see
        # below) because most controller requests proxy through to the
        # RailsAPI service.
 +      #
 +      # HTTP proxies and load balancers downstream of arvados services
 +      # should be configured to allow at least {MaxConcurrentRequest +
 +      # MaxQueuedRequests + MaxGatewayTunnels} concurrent requests.
        MaxConcurrentRequests: 64
  
        # Maximum number of concurrent requests to process concurrently
@@@ -254,13 -250,6 +254,13 @@@
        # the incoming request queue before returning 503.
        MaxQueueTimeForLockRequests: 2s
  
 +      # Maximum number of active gateway tunnel connections. One slot
 +      # is consumed by each "container shell" connection. If using an
 +      # HPC dispatcher (LSF or Slurm), one slot is consumed by each
 +      # running container.  These do not count toward
 +      # MaxConcurrentRequests.
 +      MaxGatewayTunnels: 1000
 +
        # Fraction of MaxConcurrentRequests that can be "log create"
        # messages at any given time.  This is to prevent logging
        # updates from crowding out more important requests.
@@@ -732,16 -721,18 +732,18 @@@
          # Time to cache manifests, permission checks, and sessions.
          TTL: 300s
  
-         # Block cache entries. Each block consumes up to 64 MiB RAM.
-         MaxBlockEntries: 20
+         # Maximum amount of data cached in /var/cache/arvados/keep.
+         # Can be given as a percentage ("10%") or a number of bytes
+         # ("10 GiB")
+         DiskCacheSize: 10%
  
          # Approximate memory limit (in bytes) for session cache.
          #
          # Note this applies to the in-memory representation of
          # projects and collections -- metadata, block locators,
-         # filenames, etc. -- excluding cached file content, which is
-         # limited by MaxBlockEntries.
-         MaxCollectionBytes: 100000000
+         # filenames, etc. -- not the file data itself (see
+         # DiskCacheSize).
+         MaxCollectionBytes: 100 MB
  
          # Persistent sessions.
          MaxSessions: 100

diff --combined sdk/go/arvados/config.go
index 16d789daf5163f49bc6fe4770565fcde9325fd35,acc091a90fdf8d458ac1ae6d181ef3c52d582139..9b2cf155b0b9e1ec712b1132c9c25ba11df6b6af
--- 1/sdk/go/arvados/config.go
--- 2/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -63,8 -63,8 +63,8 @@@ func (sc *Config) GetCluster(clusterID 
  
  type WebDAVCacheConfig struct {
        TTL                Duration
-       MaxBlockEntries    int
-       MaxCollectionBytes int64
+       DiskCacheSize      ByteSizeOrPercent
+       MaxCollectionBytes ByteSize
        MaxSessions        int
  }
  
@@@ -102,7 -102,6 +102,7 @@@ type Cluster struct 
                MaxConcurrentRailsRequests       int
                MaxConcurrentRequests            int
                MaxQueuedRequests                int
 +              MaxGatewayTunnels                int
                MaxQueueTimeForLockRequests      Duration
                LogCreateRequestFraction         float64
                MaxKeepBlobBuffers               int