X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/413db07b4c81ea08663f90f31ee03227349d2be4..2af7b0336b2b92e38f6966b8bbc233c05704815d:/services/keep-web/cache.go diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index b2bab78216..eeb78ad905 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -6,11 +6,10 @@ package main import ( "sync" - "sync/atomic" "time" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "github.com/hashicorp/golang-lru" "github.com/prometheus/client_golang/prometheus" ) @@ -18,15 +17,8 @@ import ( const metricsUpdateInterval = time.Second / 10 type cache struct { - TTL arvados.Duration - UUIDTTL arvados.Duration - MaxCollectionEntries int - MaxCollectionBytes int64 - MaxPermissionEntries int - MaxUUIDEntries int - + config *arvados.WebDAVCacheConfig registry *prometheus.Registry - stats cacheStats metrics cacheMetrics pdhs *lru.TwoQueueCache collections *lru.TwoQueueCache @@ -34,17 +26,6 @@ type cache struct { setupOnce sync.Once } -// cacheStats is EOL - add new metrics to cacheMetrics instead -type cacheStats struct { - Requests uint64 `json:"Cache.Requests"` - CollectionBytes uint64 `json:"Cache.CollectionBytes"` - CollectionEntries int `json:"Cache.CollectionEntries"` - CollectionHits uint64 `json:"Cache.CollectionHits"` - PDHHits uint64 `json:"Cache.UUIDHits"` - PermissionHits uint64 `json:"Cache.PermissionHits"` - APICalls uint64 `json:"Cache.APICalls"` -} - type cacheMetrics struct { requests prometheus.Counter collectionBytes prometheus.Gauge @@ -123,15 +104,15 @@ type cachedPermission struct { func (c *cache) setup() { var err error - c.pdhs, err = lru.New2Q(c.MaxUUIDEntries) + c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries) if err != nil { panic(err) } - c.collections, err = lru.New2Q(c.MaxCollectionEntries) + c.collections, err = lru.New2Q(c.config.MaxCollectionEntries) if err != nil { panic(err) } - c.permissions, err = lru.New2Q(c.MaxPermissionEntries) + c.permissions, err = lru.New2Q(c.config.MaxPermissionEntries) if err != nil { panic(err) } @@ -157,36 +138,27 @@ var selectPDH = map[string]interface{}{ "select": []string{"portable_data_hash"}, } -func (c *cache) Stats() cacheStats { - c.setupOnce.Do(c.setup) - return cacheStats{ - Requests: atomic.LoadUint64(&c.stats.Requests), - CollectionBytes: c.collectionBytes(), - CollectionEntries: c.collections.Len(), - CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits), - PDHHits: atomic.LoadUint64(&c.stats.PDHHits), - PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits), - APICalls: atomic.LoadUint64(&c.stats.APICalls), - } -} - // Update saves a modified version (fs) to an existing collection // (coll) and, if successful, updates the relevant cache entries so // subsequent calls to Get() reflect the modifications. func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error { c.setupOnce.Do(c.setup) - if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText { + m, err := fs.MarshalManifest(".") + if err != nil || m == coll.ManifestText { return err - } else { - coll.ManifestText = m } + coll.ManifestText = m var updated arvados.Collection defer c.pdhs.Remove(coll.UUID) - err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil) + err = client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": coll.ManifestText, + }, + }) if err == nil { c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{ - expire: time.Now().Add(time.Duration(c.TTL)), + expire: time.Now().Add(time.Duration(c.config.TTL)), collection: &updated, }) } @@ -195,8 +167,6 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) { c.setupOnce.Do(c.setup) - - atomic.AddUint64(&c.stats.Requests, 1) c.metrics.requests.Inc() permOK := false @@ -208,7 +178,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo c.permissions.Remove(permKey) } else { permOK = true - atomic.AddUint64(&c.stats.PermissionHits, 1) c.metrics.permissionHits.Inc() } } @@ -222,7 +191,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo c.pdhs.Remove(targetID) } else { pdh = ent.pdh - atomic.AddUint64(&c.stats.PDHHits, 1) c.metrics.pdhHits.Inc() } } @@ -239,7 +207,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // likely, the cached PDH is still correct; if so, // _and_ the current token has permission, we can // use our cached manifest. - atomic.AddUint64(&c.stats.APICalls, 1) c.metrics.apiCalls.Inc() var current arvados.Collection err := arv.Get("collections", targetID, selectPDH, ¤t) @@ -248,45 +215,43 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo } if current.PortableDataHash == pdh { c.permissions.Add(permKey, &cachedPermission{ - expire: time.Now().Add(time.Duration(c.TTL)), + expire: time.Now().Add(time.Duration(c.config.TTL)), }) if pdh != targetID { c.pdhs.Add(targetID, &cachedPDH{ - expire: time.Now().Add(time.Duration(c.UUIDTTL)), + expire: time.Now().Add(time.Duration(c.config.UUIDTTL)), pdh: pdh, }) } return collection, err - } else { - // PDH changed, but now we know we have - // permission -- and maybe we already have the - // new PDH in the cache. - if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { - return coll, nil - } + } + // PDH changed, but now we know we have + // permission -- and maybe we already have the + // new PDH in the cache. + if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { + return coll, nil } } // Collection manifest is not cached. - atomic.AddUint64(&c.stats.APICalls, 1) c.metrics.apiCalls.Inc() err := arv.Get("collections", targetID, nil, &collection) if err != nil { return nil, err } - exp := time.Now().Add(time.Duration(c.TTL)) + exp := time.Now().Add(time.Duration(c.config.TTL)) c.permissions.Add(permKey, &cachedPermission{ expire: exp, }) c.pdhs.Add(targetID, &cachedPDH{ - expire: time.Now().Add(time.Duration(c.UUIDTTL)), + expire: time.Now().Add(time.Duration(c.config.UUIDTTL)), pdh: collection.PortableDataHash, }) c.collections.Add(arv.ApiToken+"\000"+collection.PortableDataHash, &cachedCollection{ expire: exp, collection: collection, }) - if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { + if int64(len(collection.ManifestText)) > c.config.MaxCollectionBytes/int64(c.config.MaxCollectionEntries) { go c.pruneCollections() } return collection, nil @@ -323,7 +288,7 @@ func (c *cache) pruneCollections() { } } for i, k := range keys { - if size <= c.MaxCollectionBytes { + if size <= c.config.MaxCollectionBytes { break } if expired[i] { @@ -359,7 +324,6 @@ func (c *cache) lookupCollection(key string) *arvados.Collection { c.collections.Remove(key) return nil } - atomic.AddUint64(&c.stats.CollectionHits, 1) c.metrics.collectionHits.Inc() return ent.collection }