18947: Refactor keep-web as arvados-server command.

[arvados.git] / services / keep-web / cache.go

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 16fbd0788cb3840f299d649aa73378aae579fce1..d5fdc4997ecee3c67d1cc8ea4003c2c1e3f6cac4 100644 (file)
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepweb
 
 import (
        "sync"
@@ -14,13 +14,14 @@ import (
        "git.arvados.org/arvados.git/sdk/go/keepclient"
        lru "github.com/hashicorp/golang-lru"
        "github.com/prometheus/client_golang/prometheus"
+       "github.com/sirupsen/logrus"
 )
 
 const metricsUpdateInterval = time.Second / 10
 
 type cache struct {
        cluster     *arvados.Cluster
-       config      *arvados.WebDAVCacheConfig // TODO: use cluster.Collections.WebDAV instead
+       logger      logrus.FieldLogger
        registry    *prometheus.Registry
        metrics     cacheMetrics
        pdhs        *lru.TwoQueueCache
@@ -136,15 +137,15 @@ type cachedSession struct {
 
 func (c *cache) setup() {
        var err error
-       c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries)
+       c.pdhs, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxUUIDEntries)
        if err != nil {
                panic(err)
        }
-       c.collections, err = lru.New2Q(c.config.MaxCollectionEntries)
+       c.collections, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxCollectionEntries)
        if err != nil {
                panic(err)
        }
-       c.sessions, err = lru.New2Q(c.config.MaxSessions)
+       c.sessions, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxSessions)
        if err != nil {
                panic(err)
        }
@@ -205,12 +206,12 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad
                return err
        }
        c.collections.Add(client.AuthToken+"\000"+updated.PortableDataHash, &cachedCollection{
-               expire:     time.Now().Add(time.Duration(c.config.TTL)),
+               expire:     time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
                collection: &updated,
        })
        c.pdhs.Add(coll.UUID, &cachedPDH{
-               expire:  time.Now().Add(time.Duration(c.config.TTL)),
-               refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+               expire:  time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
+               refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
                pdh:     updated.PortableDataHash,
        })
        return nil
@@ -235,7 +236,7 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, *cachedSessi
        if sess == nil {
                c.metrics.sessionMisses.Inc()
                sess = &cachedSession{
-                       expire: now.Add(c.config.TTL.Duration()),
+                       expire: now.Add(c.cluster.Collections.WebDAVCache.TTL.Duration()),
                }
                var err error
                sess.client, err = arvados.NewClientFromConfig(c.cluster)
@@ -332,12 +333,15 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
        if pdh == "" {
                // UUID->PDH mapping is not cached, might as well get
                // the whole collection record and be done (below).
+               c.logger.Debugf("cache(%s): have no pdh", targetID)
        } else if cached := c.lookupCollection(arv.ApiToken + "\000" + pdh); cached == nil {
                // PDH->manifest is not cached, might as well get the
                // whole collection record (below).
+               c.logger.Debugf("cache(%s): have pdh %s but manifest is not cached", targetID, pdh)
        } else if !pdhRefresh {
                // We looked up UUID->PDH very recently, and we still
                // have the manifest for that PDH.
+               c.logger.Debugf("cache(%s): have pdh %s and refresh not needed", targetID, pdh)
                return cached, nil
        } else {
                // Get current PDH for this UUID (and confirm we still
@@ -353,11 +357,13 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                if current.PortableDataHash == pdh {
                        // PDH has not changed, cached manifest is
                        // correct.
-                       return cached, err
+                       c.logger.Debugf("cache(%s): verified cached pdh %s is still correct", targetID, pdh)
+                       return cached, nil
                }
                if cached := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); cached != nil {
                        // PDH changed, and we already have the
                        // manifest for that new PDH.
+                       c.logger.Debugf("cache(%s): cached pdh %s was stale, new pdh is %s and manifest is already in cache", targetID, pdh, current.PortableDataHash)
                        return cached, nil
                }
        }
@@ -370,11 +376,12 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
        if err != nil {
                return nil, err
        }
-       exp := time.Now().Add(time.Duration(c.config.TTL))
+       c.logger.Debugf("cache(%s): retrieved manifest, caching with pdh %s", targetID, retrieved.PortableDataHash)
+       exp := time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL))
        if targetID != retrieved.PortableDataHash {
                c.pdhs.Add(targetID, &cachedPDH{
                        expire:  exp,
-                       refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+                       refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
                        pdh:     retrieved.PortableDataHash,
                })
        }
@@ -382,7 +389,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                expire:     exp,
                collection: &retrieved,
        })
-       if int64(len(retrieved.ManifestText)) > c.config.MaxCollectionBytes/int64(c.config.MaxCollectionEntries) {
+       if int64(len(retrieved.ManifestText)) > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/int64(c.cluster.Collections.WebDAVCache.MaxCollectionEntries) {
                select {
                case c.chPruneCollections <- struct{}{}:
                default:
@@ -422,7 +429,7 @@ func (c *cache) pruneCollections() {
                }
        }
        for i, k := range keys {
-               if size <= c.config.MaxCollectionBytes/2 {
+               if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
                        break
                }
                if expired[i] {