Merge branch '12483-writable-fs'

[arvados.git] / services / keep-web / cache.go

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index ab7c65310b0abbf6b9ba1c5eb2b8a22142ab8347..9ee99903c8d1e537d487a67d1c77d848fc93c807 100644 (file)
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -1,7 +1,10 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
-       "fmt"
        "sync"
        "sync/atomic"
        "time"
@@ -13,6 +16,7 @@ import (
 
 type cache struct {
        TTL                  arvados.Duration
+       UUIDTTL              arvados.Duration
        MaxCollectionEntries int
        MaxCollectionBytes   int64
        MaxPermissionEntries int
@@ -42,7 +46,7 @@ type cachedPDH struct {
 
 type cachedCollection struct {
        expire     time.Time
-       collection map[string]interface{}
+       collection *arvados.Collection
 }
 
 type cachedPermission struct {
@@ -82,7 +86,30 @@ func (c *cache) Stats() cacheStats {
        }
 }
 
-func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (map[string]interface{}, error) {
+// Update saves a modified version (fs) to an existing collection
+// (coll) and, if successful, updates the relevant cache entries so
+// subsequent calls to Get() reflect the modifications.
+func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error {
+       c.setupOnce.Do(c.setup)
+
+       if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText {
+               return err
+       } else {
+               coll.ManifestText = m
+       }
+       var updated arvados.Collection
+       defer c.pdhs.Remove(coll.UUID)
+       err := client.RequestAndDecode(&updated, "PATCH", "/arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil)
+       if err == nil {
+               c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{
+                       expire:     time.Now().Add(time.Duration(c.TTL)),
+                       collection: &updated,
+               })
+       }
+       return err
+}
+
+func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
        c.setupOnce.Do(c.setup)
 
        atomic.AddUint64(&c.stats.Requests, 1)
@@ -103,7 +130,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
        var pdh string
        if arvadosclient.PDHMatch(targetID) {
                pdh = targetID
-       } else if forceReload {
        } else if ent, cached := c.pdhs.Get(targetID); cached {
                ent := ent.(*cachedPDH)
                if ent.expire.Before(time.Now()) {
@@ -114,9 +140,9 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                }
        }
 
-       var collection map[string]interface{}
+       var collection *arvados.Collection
        if pdh != "" {
-               collection = c.lookupCollection(pdh)
+               collection = c.lookupCollection(arv.ApiToken + "\000" + pdh)
        }
 
        if collection != nil && permOK {
@@ -127,21 +153,18 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                // _and_ the current token has permission, we can
                // use our cached manifest.
                atomic.AddUint64(&c.stats.APICalls, 1)
-               var current map[string]interface{}
+               var current arvados.Collection
                err := arv.Get("collections", targetID, selectPDH, &current)
                if err != nil {
                        return nil, err
                }
-               if checkPDH, ok := current["portable_data_hash"].(string); !ok {
-                       return nil, fmt.Errorf("API response for %q had no PDH", targetID)
-               } else if checkPDH == pdh {
-                       exp := time.Now().Add(time.Duration(c.TTL))
+               if current.PortableDataHash == pdh {
                        c.permissions.Add(permKey, &cachedPermission{
-                               expire: exp,
+                               expire: time.Now().Add(time.Duration(c.TTL)),
                        })
                        if pdh != targetID {
                                c.pdhs.Add(targetID, &cachedPDH{
-                                       expire: exp,
+                                       expire: time.Now().Add(time.Duration(c.UUIDTTL)),
                                        pdh:    pdh,
                                })
                        }
@@ -150,7 +173,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
                        // PDH changed, but now we know we have
                        // permission -- and maybe we already have the
                        // new PDH in the cache.
-                       if coll := c.lookupCollection(checkPDH); coll != nil {
+                       if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil {
                                return coll, nil
                        }
                }
@@ -162,23 +185,19 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
        if err != nil {
                return nil, err
        }
-       pdh, ok := collection["portable_data_hash"].(string)
-       if !ok {
-               return nil, fmt.Errorf("API response for %q had no PDH", targetID)
-       }
        exp := time.Now().Add(time.Duration(c.TTL))
        c.permissions.Add(permKey, &cachedPermission{
                expire: exp,
        })
        c.pdhs.Add(targetID, &cachedPDH{
-               expire: exp,
-               pdh:    pdh,
+               expire: time.Now().Add(time.Duration(c.UUIDTTL)),
+               pdh:    collection.PortableDataHash,
        })
-       c.collections.Add(pdh, &cachedCollection{
+       c.collections.Add(arv.ApiToken+"\000"+collection.PortableDataHash, &cachedCollection{
                expire:     exp,
                collection: collection,
        })
-       if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
+       if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
                go c.pruneCollections()
        }
        return collection, nil
@@ -203,7 +222,7 @@ func (c *cache) pruneCollections() {
                        continue
                }
                ent := v.(*cachedCollection)
-               n := len(ent.collection["manifest_text"].(string))
+               n := len(ent.collection.ManifestText)
                size += int64(n)
                entsize[i] = n
                expired[i] = ent.expire.Before(now)
@@ -236,20 +255,18 @@ func (c *cache) collectionBytes() uint64 {
                if !ok {
                        continue
                }
-               size += uint64(len(v.(*cachedCollection).collection["manifest_text"].(string)))
+               size += uint64(len(v.(*cachedCollection).collection.ManifestText))
        }
        return size
 }
 
-func (c *cache) lookupCollection(pdh string) map[string]interface{} {
-       if pdh == "" {
-               return nil
-       } else if ent, cached := c.collections.Get(pdh); !cached {
+func (c *cache) lookupCollection(key string) *arvados.Collection {
+       if ent, cached := c.collections.Get(key); !cached {
                return nil
        } else {
                ent := ent.(*cachedCollection)
                if ent.expire.Before(time.Now()) {
-                       c.collections.Remove(pdh)
+                       c.collections.Remove(key)
                        return nil
                } else {
                        atomic.AddUint64(&c.stats.CollectionHits, 1)