X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0eb72b526bf8bbb011551ecf019f604e17a534f1..ccfad8f850f6d3edc43e7757fbeed112864efd18:/services/keep-web/cache.go diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index 174e370c13..b2bab78216 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -12,22 +12,29 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "github.com/hashicorp/golang-lru" + "github.com/prometheus/client_golang/prometheus" ) +const metricsUpdateInterval = time.Second / 10 + type cache struct { TTL arvados.Duration + UUIDTTL arvados.Duration MaxCollectionEntries int MaxCollectionBytes int64 MaxPermissionEntries int MaxUUIDEntries int + registry *prometheus.Registry stats cacheStats + metrics cacheMetrics pdhs *lru.TwoQueueCache collections *lru.TwoQueueCache permissions *lru.TwoQueueCache setupOnce sync.Once } +// cacheStats is EOL - add new metrics to cacheMetrics instead type cacheStats struct { Requests uint64 `json:"Cache.Requests"` CollectionBytes uint64 `json:"Cache.CollectionBytes"` @@ -38,6 +45,68 @@ type cacheStats struct { APICalls uint64 `json:"Cache.APICalls"` } +type cacheMetrics struct { + requests prometheus.Counter + collectionBytes prometheus.Gauge + collectionEntries prometheus.Gauge + collectionHits prometheus.Counter + pdhHits prometheus.Counter + permissionHits prometheus.Counter + apiCalls prometheus.Counter +} + +func (m *cacheMetrics) setup(reg *prometheus.Registry) { + m.requests = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "requests", + Help: "Number of targetID-to-manifest lookups handled.", + }) + reg.MustRegister(m.requests) + m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "hits", + Help: "Number of pdh-to-manifest cache hits.", + }) + reg.MustRegister(m.collectionHits) + m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "pdh_hits", + Help: "Number of uuid-to-pdh cache hits.", + }) + reg.MustRegister(m.pdhHits) + m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "permission_hits", + Help: "Number of targetID-to-permission cache hits.", + }) + reg.MustRegister(m.permissionHits) + m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "api_calls", + Help: "Number of outgoing API calls made by cache.", + }) + reg.MustRegister(m.apiCalls) + m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "cached_manifest_bytes", + Help: "Total size of all manifests in cache.", + }) + reg.MustRegister(m.collectionBytes) + m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "keepweb_collectioncache", + Name: "cached_manifests", + Help: "Number of manifests in cache.", + }) + reg.MustRegister(m.collectionEntries) +} + type cachedPDH struct { expire time.Time pdh string @@ -66,6 +135,22 @@ func (c *cache) setup() { if err != nil { panic(err) } + + reg := c.registry + if reg == nil { + reg = prometheus.NewRegistry() + } + c.metrics.setup(reg) + go func() { + for range time.Tick(metricsUpdateInterval) { + c.updateGauges() + } + }() +} + +func (c *cache) updateGauges() { + c.metrics.collectionBytes.Set(float64(c.collectionBytes())) + c.metrics.collectionEntries.Set(float64(c.collections.Len())) } var selectPDH = map[string]interface{}{ @@ -85,10 +170,34 @@ func (c *cache) Stats() cacheStats { } } +// Update saves a modified version (fs) to an existing collection +// (coll) and, if successful, updates the relevant cache entries so +// subsequent calls to Get() reflect the modifications. +func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error { + c.setupOnce.Do(c.setup) + + if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText { + return err + } else { + coll.ManifestText = m + } + var updated arvados.Collection + defer c.pdhs.Remove(coll.UUID) + err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, client.UpdateBody(coll), nil) + if err == nil { + c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{ + expire: time.Now().Add(time.Duration(c.TTL)), + collection: &updated, + }) + } + return err +} + func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) { c.setupOnce.Do(c.setup) atomic.AddUint64(&c.stats.Requests, 1) + c.metrics.requests.Inc() permOK := false permKey := arv.ApiToken + "\000" + targetID @@ -100,6 +209,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo } else { permOK = true atomic.AddUint64(&c.stats.PermissionHits, 1) + c.metrics.permissionHits.Inc() } } @@ -113,12 +223,13 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo } else { pdh = ent.pdh atomic.AddUint64(&c.stats.PDHHits, 1) + c.metrics.pdhHits.Inc() } } var collection *arvados.Collection if pdh != "" { - collection = c.lookupCollection(pdh) + collection = c.lookupCollection(arv.ApiToken + "\000" + pdh) } if collection != nil && permOK { @@ -129,19 +240,19 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // _and_ the current token has permission, we can // use our cached manifest. atomic.AddUint64(&c.stats.APICalls, 1) + c.metrics.apiCalls.Inc() var current arvados.Collection err := arv.Get("collections", targetID, selectPDH, ¤t) if err != nil { return nil, err } if current.PortableDataHash == pdh { - exp := time.Now().Add(time.Duration(c.TTL)) c.permissions.Add(permKey, &cachedPermission{ - expire: exp, + expire: time.Now().Add(time.Duration(c.TTL)), }) if pdh != targetID { c.pdhs.Add(targetID, &cachedPDH{ - expire: exp, + expire: time.Now().Add(time.Duration(c.UUIDTTL)), pdh: pdh, }) } @@ -150,7 +261,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // PDH changed, but now we know we have // permission -- and maybe we already have the // new PDH in the cache. - if coll := c.lookupCollection(current.PortableDataHash); coll != nil { + if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { return coll, nil } } @@ -158,6 +269,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo // Collection manifest is not cached. atomic.AddUint64(&c.stats.APICalls, 1) + c.metrics.apiCalls.Inc() err := arv.Get("collections", targetID, nil, &collection) if err != nil { return nil, err @@ -167,10 +279,10 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo expire: exp, }) c.pdhs.Add(targetID, &cachedPDH{ - expire: exp, + expire: time.Now().Add(time.Duration(c.UUIDTTL)), pdh: collection.PortableDataHash, }) - c.collections.Add(collection.PortableDataHash, &cachedCollection{ + c.collections.Add(arv.ApiToken+"\000"+collection.PortableDataHash, &cachedCollection{ expire: exp, collection: collection, }) @@ -237,19 +349,17 @@ func (c *cache) collectionBytes() uint64 { return size } -func (c *cache) lookupCollection(pdh string) *arvados.Collection { - if pdh == "" { +func (c *cache) lookupCollection(key string) *arvados.Collection { + e, cached := c.collections.Get(key) + if !cached { return nil - } else if ent, cached := c.collections.Get(pdh); !cached { + } + ent := e.(*cachedCollection) + if ent.expire.Before(time.Now()) { + c.collections.Remove(key) return nil - } else { - ent := ent.(*cachedCollection) - if ent.expire.Before(time.Now()) { - c.collections.Remove(pdh) - return nil - } else { - atomic.AddUint64(&c.stats.CollectionHits, 1) - return ent.collection - } } + atomic.AddUint64(&c.stats.CollectionHits, 1) + c.metrics.collectionHits.Inc() + return ent.collection }