Merge branch '11917-dont-clear-cache'

[arvados.git] / services / keep-web / cache.go

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 26f6627424ce605d38675f5e6f1b7fa4ac53f7a8..d72effc075d65a62d41b722ec177dfa3d465ec5f 100644 (file)
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -1,7 +1,10 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
-       "fmt"
        "sync"
        "sync/atomic"
        "time"
@@ -12,11 +15,11 @@ import (
 )
 
 type cache struct {
-       TTL               arvados.Duration
-       CollectionEntries int
-       CollectionBytes   int64
-       PermissionEntries int
-       UUIDEntries       int
+       TTL                  arvados.Duration
+       MaxCollectionEntries int
+       MaxCollectionBytes   int64
+       MaxPermissionEntries int
+       MaxUUIDEntries       int
 
        stats       cacheStats
        pdhs        *lru.TwoQueueCache
@@ -42,7 +45,7 @@ type cachedPDH struct {
 
 type cachedCollection struct {
        expire     time.Time
-       collection map[string]interface{}
+       collection *arvados.Collection
 }
 
 type cachedPermission struct {
@@ -51,15 +54,15 @@ type cachedPermission struct {
 
 func (c *cache) setup() {
        var err error
-       c.pdhs, err = lru.New2Q(c.UUIDEntries)
+       c.pdhs, err = lru.New2Q(c.MaxUUIDEntries)
        if err != nil {
                panic(err)
        }
-       c.collections, err = lru.New2Q(c.CollectionEntries)
+       c.collections, err = lru.New2Q(c.MaxCollectionEntries)
        if err != nil {
                panic(err)
        }
-       c.permissions, err = lru.New2Q(c.PermissionEntries)
+       c.permissions, err = lru.New2Q(c.MaxPermissionEntries)
        if err != nil {
                panic(err)
        }
@@ -82,14 +85,15 @@ func (c *cache) Stats() cacheStats {
        }
 }
 
-func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (map[string]interface{}, error) {
+func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
        c.setupOnce.Do(c.setup)
 
        atomic.AddUint64(&c.stats.Requests, 1)
 
        permOK := false
        permKey := arv.ApiToken + "\000" + targetID
-       if ent, cached := c.permissions.Get(permKey); cached {
+       if forceReload {
+       } else if ent, cached := c.permissions.Get(permKey); cached {
                ent := ent.(*cachedPermission)
                if ent.expire.Before(time.Now()) {
                        c.permissions.Remove(permKey)
@@ -112,26 +116,25 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                }
        }
 
-       collection := c.lookupCollection(pdh)
-
-       if collection != nil && permOK && !forceReload {
-               return collection, nil
+       var collection *arvados.Collection
+       if pdh != "" {
+               collection = c.lookupCollection(pdh)
        }
 
-       if collection != nil {
+       if collection != nil && permOK {
+               return collection, nil
+       } else if collection != nil {
                // Ask API for current PDH for this targetID. Most
                // likely, the cached PDH is still correct; if so,
                // _and_ the current token has permission, we can
                // use our cached manifest.
                atomic.AddUint64(&c.stats.APICalls, 1)
-               var current map[string]interface{}
+               var current arvados.Collection
                err := arv.Get("collections", targetID, selectPDH, &current)
                if err != nil {
                        return nil, err
                }
-               if checkPDH, ok := current["portable_data_hash"].(string); !ok {
-                       return nil, fmt.Errorf("API response for %q had no PDH", targetID)
-               } else if checkPDH == pdh {
+               if current.PortableDataHash == pdh {
                        exp := time.Now().Add(time.Duration(c.TTL))
                        c.permissions.Add(permKey, &cachedPermission{
                                expire: exp,
@@ -147,7 +150,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                        // PDH changed, but now we know we have
                        // permission -- and maybe we already have the
                        // new PDH in the cache.
-                       if coll := c.lookupCollection(checkPDH); coll != nil {
+                       if coll := c.lookupCollection(current.PortableDataHash); coll != nil {
                                return coll, nil
                        }
                }
@@ -159,25 +162,22 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
        if err != nil {
                return nil, err
        }
-       pdh, ok := collection["portable_data_hash"].(string)
-       if !ok {
-               return nil, fmt.Errorf("API response for %q had no PDH", targetID)
-       }
        exp := time.Now().Add(time.Duration(c.TTL))
        c.permissions.Add(permKey, &cachedPermission{
                expire: exp,
        })
        c.pdhs.Add(targetID, &cachedPDH{
                expire: exp,
-               pdh:    pdh,
+               pdh:    collection.PortableDataHash,
        })
-       c.collections.Add(pdh, &cachedCollection{
-               expire:     exp,
-               collection: collection,
-       })
-       if int64(len(collection["manifest_text"].(string))) > c.CollectionBytes/int64(c.CollectionEntries) {
-               c.pruneCollections()
-       }
+       // Disabled, see #11945
+       // c.collections.Add(collection.PortableDataHash, &cachedCollection{
+       //      expire:     exp,
+       //      collection: collection,
+       // })
+       // if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
+       //      go c.pruneCollections()
+       // }
        return collection, nil
 }
 
@@ -200,7 +200,7 @@ func (c *cache) pruneCollections() {
                        continue
                }
                ent := v.(*cachedCollection)
-               n := len(ent.collection["manifest_text"].(string))
+               n := len(ent.collection.ManifestText)
                size += int64(n)
                entsize[i] = n
                expired[i] = ent.expire.Before(now)
@@ -212,7 +212,7 @@ func (c *cache) pruneCollections() {
                }
        }
        for i, k := range keys {
-               if size <= c.CollectionBytes {
+               if size <= c.MaxCollectionBytes {
                        break
                }
                if expired[i] {
@@ -233,12 +233,12 @@ func (c *cache) collectionBytes() uint64 {
                if !ok {
                        continue
                }
-               size += uint64(len(v.(*cachedCollection).collection["manifest_text"].(string)))
+               size += uint64(len(v.(*cachedCollection).collection.ManifestText))
        }
        return size
 }
 
-func (c *cache) lookupCollection(pdh string) map[string]interface{} {
+func (c *cache) lookupCollection(pdh string) *arvados.Collection {
        if pdh == "" {
                return nil
        } else if ent, cached := c.collections.Get(pdh); !cached {