X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/123153139bbee3674c81325653d87fa19ffbe0e4..641ef213571f793bb290a182dee3c4325bc85096:/services/keep-web/cache.go diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index ab7c65310b..59e8de3bc9 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -1,7 +1,10 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( - "fmt" "sync" "sync/atomic" "time" @@ -13,6 +16,7 @@ import ( type cache struct { TTL arvados.Duration + UUIDTTL arvados.Duration MaxCollectionEntries int MaxCollectionBytes int64 MaxPermissionEntries int @@ -42,7 +46,7 @@ type cachedPDH struct { type cachedCollection struct { expire time.Time - collection map[string]interface{} + collection *arvados.Collection } type cachedPermission struct { @@ -82,7 +86,30 @@ func (c *cache) Stats() cacheStats { } } -func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (map[string]interface{}, error) { +// Update saves a modified version (fs) to an existing collection +// (coll) and, if successful, updates the relevant cache entries so +// subsequent calls to Get() reflect the modifications. +func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error { + c.setupOnce.Do(c.setup) + + if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText { + return err + } else { + coll.ManifestText = m + } + var updated arvados.Collection + defer c.pdhs.Remove(coll.UUID) + err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil) + if err == nil { + c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{ + expire: time.Now().Add(time.Duration(c.TTL)), + collection: &updated, + }) + } + return err +} + +func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) { c.setupOnce.Do(c.setup) atomic.AddUint64(&c.stats.Requests, 1) @@ -103,7 +130,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo var pdh string if arvadosclient.PDHMatch(targetID) { pdh = targetID - } else if forceReload { } else if ent, cached := c.pdhs.Get(targetID); cached { ent := ent.(*cachedPDH) if ent.expire.Before(time.Now()) { @@ -114,9 +140,9 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo } } - var collection map[string]interface{} + var collection *arvados.Collection if pdh != "" { - collection = c.lookupCollection(pdh) + collection = c.lookupCollection(arv.ApiToken + "\000" + pdh) } if collection != nil && permOK { @@ -127,21 +153,18 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // _and_ the current token has permission, we can // use our cached manifest. atomic.AddUint64(&c.stats.APICalls, 1) - var current map[string]interface{} + var current arvados.Collection err := arv.Get("collections", targetID, selectPDH, ¤t) if err != nil { return nil, err } - if checkPDH, ok := current["portable_data_hash"].(string); !ok { - return nil, fmt.Errorf("API response for %q had no PDH", targetID) - } else if checkPDH == pdh { - exp := time.Now().Add(time.Duration(c.TTL)) + if current.PortableDataHash == pdh { c.permissions.Add(permKey, &cachedPermission{ - expire: exp, + expire: time.Now().Add(time.Duration(c.TTL)), }) if pdh != targetID { c.pdhs.Add(targetID, &cachedPDH{ - expire: exp, + expire: time.Now().Add(time.Duration(c.UUIDTTL)), pdh: pdh, }) } @@ -150,7 +173,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // PDH changed, but now we know we have // permission -- and maybe we already have the // new PDH in the cache. - if coll := c.lookupCollection(checkPDH); coll != nil { + if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { return coll, nil } } @@ -162,23 +185,19 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo if err != nil { return nil, err } - pdh, ok := collection["portable_data_hash"].(string) - if !ok { - return nil, fmt.Errorf("API response for %q had no PDH", targetID) - } exp := time.Now().Add(time.Duration(c.TTL)) c.permissions.Add(permKey, &cachedPermission{ expire: exp, }) c.pdhs.Add(targetID, &cachedPDH{ - expire: exp, - pdh: pdh, + expire: time.Now().Add(time.Duration(c.UUIDTTL)), + pdh: collection.PortableDataHash, }) - c.collections.Add(pdh, &cachedCollection{ + c.collections.Add(arv.ApiToken+"\000"+collection.PortableDataHash, &cachedCollection{ expire: exp, collection: collection, }) - if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { + if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { go c.pruneCollections() } return collection, nil @@ -203,7 +222,7 @@ func (c *cache) pruneCollections() { continue } ent := v.(*cachedCollection) - n := len(ent.collection["manifest_text"].(string)) + n := len(ent.collection.ManifestText) size += int64(n) entsize[i] = n expired[i] = ent.expire.Before(now) @@ -236,20 +255,18 @@ func (c *cache) collectionBytes() uint64 { if !ok { continue } - size += uint64(len(v.(*cachedCollection).collection["manifest_text"].(string))) + size += uint64(len(v.(*cachedCollection).collection.ManifestText)) } return size } -func (c *cache) lookupCollection(pdh string) map[string]interface{} { - if pdh == "" { - return nil - } else if ent, cached := c.collections.Get(pdh); !cached { +func (c *cache) lookupCollection(key string) *arvados.Collection { + if ent, cached := c.collections.Get(key); !cached { return nil } else { ent := ent.(*cachedCollection) if ent.expire.Before(time.Now()) { - c.collections.Remove(pdh) + c.collections.Remove(key) return nil } else { atomic.AddUint64(&c.stats.CollectionHits, 1)