services/keepstore/mounts_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "encoding/json"
  11         "net/http"
  12         "net/http/httptest"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  16         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  17         "github.com/prometheus/client_golang/prometheus"
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 func (s *HandlerSuite) TestMounts(c *check.C) {
  22         c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
  23
  24         vols := s.handler.volmgr.AllWritable()
  25         vols[0].Put(context.Background(), TestHash, TestBlock)
  26         vols[1].Put(context.Background(), TestHash2, TestBlock2)
  27
  28         resp := s.call("GET", "/mounts", "", nil)
  29         c.Check(resp.Code, check.Equals, http.StatusOK)
  30         var mntList []struct {
  31                 UUID           string          `json:"uuid"`
  32                 DeviceID       string          `json:"device_id"`
  33                 ReadOnly       bool            `json:"read_only"`
  34                 Replication    int             `json:"replication"`
  35                 StorageClasses map[string]bool `json:"storage_classes"`
  36         }
  37         c.Log(resp.Body.String())
  38         err := json.Unmarshal(resp.Body.Bytes(), &mntList)
  39         c.Assert(err, check.IsNil)
  40         c.Assert(len(mntList), check.Equals, 2)
  41         for _, m := range mntList {
  42                 c.Check(len(m.UUID), check.Equals, 27)
  43                 c.Check(m.UUID[:12], check.Equals, "zzzzz-nyw5e-")
  44                 c.Check(m.DeviceID, check.Equals, "mock-device-id")
  45                 c.Check(m.ReadOnly, check.Equals, false)
  46                 c.Check(m.Replication, check.Equals, 1)
  47                 c.Check(m.StorageClasses, check.DeepEquals, map[string]bool{"default": true})
  48         }
  49         c.Check(mntList[0].UUID, check.Not(check.Equals), mntList[1].UUID)
  50
  51         // Bad auth
  52         for _, tok := range []string{"", "xyzzy"} {
  53                 resp = s.call("GET", "/mounts/"+mntList[1].UUID+"/blocks", tok, nil)
  54                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
  55                 c.Check(resp.Body.String(), check.Equals, "Unauthorized\n")
  56         }
  57
  58         tok := arvadostest.DataManagerToken
  59
  60         // Nonexistent mount UUID
  61         resp = s.call("GET", "/mounts/X/blocks", tok, nil)
  62         c.Check(resp.Code, check.Equals, http.StatusNotFound)
  63         c.Check(resp.Body.String(), check.Equals, "mount not found\n")
  64
  65         // Complete index of first mount
  66         resp = s.call("GET", "/mounts/"+mntList[0].UUID+"/blocks", tok, nil)
  67         c.Check(resp.Code, check.Equals, http.StatusOK)
  68         c.Check(resp.Body.String(), check.Matches, TestHash+`\+[0-9]+ [0-9]+\n\n`)
  69
  70         // Partial index of first mount (one block matches prefix)
  71         resp = s.call("GET", "/mounts/"+mntList[0].UUID+"/blocks?prefix="+TestHash[:2], tok, nil)
  72         c.Check(resp.Code, check.Equals, http.StatusOK)
  73         c.Check(resp.Body.String(), check.Matches, TestHash+`\+[0-9]+ [0-9]+\n\n`)
  74
  75         // Complete index of second mount (note trailing slash)
  76         resp = s.call("GET", "/mounts/"+mntList[1].UUID+"/blocks/", tok, nil)
  77         c.Check(resp.Code, check.Equals, http.StatusOK)
  78         c.Check(resp.Body.String(), check.Matches, TestHash2+`\+[0-9]+ [0-9]+\n\n`)
  79
  80         // Partial index of second mount (no blocks match prefix)
  81         resp = s.call("GET", "/mounts/"+mntList[1].UUID+"/blocks/?prefix="+TestHash[:2], tok, nil)
  82         c.Check(resp.Code, check.Equals, http.StatusOK)
  83         c.Check(resp.Body.String(), check.Equals, "\n")
  84 }
  85
  86 func (s *HandlerSuite) TestMetrics(c *check.C) {
  87         reg := prometheus.NewRegistry()
  88         c.Assert(s.handler.setup(context.Background(), s.cluster, "", reg, testServiceURL), check.IsNil)
  89         instrumented := httpserver.Instrument(reg, ctxlog.TestLogger(c), s.handler.Handler)
  90         s.handler.Handler = instrumented.ServeAPI(s.cluster.ManagementToken, instrumented)
  91
  92         s.call("PUT", "/"+TestHash, "", TestBlock)
  93         s.call("PUT", "/"+TestHash2, "", TestBlock2)
  94         resp := s.call("GET", "/metrics.json", "", nil)
  95         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
  96         resp = s.call("GET", "/metrics.json", "foobar", nil)
  97         c.Check(resp.Code, check.Equals, http.StatusForbidden)
  98         resp = s.call("GET", "/metrics.json", arvadostest.ManagementToken, nil)
  99         c.Check(resp.Code, check.Equals, http.StatusOK)
 100         var j []struct {
 101                 Name   string
 102                 Help   string
 103                 Type   string
 104                 Metric []struct {
 105                         Label []struct {
 106                                 Name  string
 107                                 Value string
 108                         }
 109                         Summary struct {
 110                                 SampleCount string  `json:"sample_count"`
 111                                 SampleSum   float64 `json:"sample_sum"`
 112                                 Quantile    []struct {
 113                                         Quantile float64
 114                                         Value    float64
 115                                 }
 116                         }
 117                 }
 118         }
 119         json.NewDecoder(resp.Body).Decode(&j)
 120         found := make(map[string]bool)
 121         names := map[string]bool{}
 122         for _, g := range j {
 123                 names[g.Name] = true
 124                 for _, m := range g.Metric {
 125                         if len(m.Label) == 2 && m.Label[0].Name == "code" && m.Label[0].Value == "200" && m.Label[1].Name == "method" && m.Label[1].Value == "put" {
 126                                 c.Check(m.Summary.SampleCount, check.Equals, "2")
 127                                 c.Check(len(m.Summary.Quantile), check.Not(check.Equals), 0)
 128                                 c.Check(m.Summary.Quantile[0].Value, check.Not(check.Equals), float64(0))
 129                                 found[g.Name] = true
 130                         }
 131                 }
 132         }
 133
 134         metricsNames := []string{
 135                 "arvados_keepstore_bufferpool_inuse_buffers",
 136                 "arvados_keepstore_bufferpool_max_buffers",
 137                 "arvados_keepstore_bufferpool_allocated_bytes",
 138                 "arvados_keepstore_pull_queue_inprogress_entries",
 139                 "arvados_keepstore_pull_queue_pending_entries",
 140                 "arvados_keepstore_trash_queue_inprogress_entries",
 141                 "arvados_keepstore_trash_queue_pending_entries",
 142                 "request_duration_seconds",
 143         }
 144         for _, m := range metricsNames {
 145                 _, ok := names[m]
 146                 c.Check(ok, check.Equals, true, check.Commentf("checking metric %q", m))
 147         }
 148 }
 149
 150 func (s *HandlerSuite) call(method, path, tok string, body []byte) *httptest.ResponseRecorder {
 151         resp := httptest.NewRecorder()
 152         req, _ := http.NewRequest(method, path, bytes.NewReader(body))
 153         if tok != "" {
 154                 req.Header.Set("Authorization", "Bearer "+tok)
 155         }
 156         s.handler.ServeHTTP(resp, req)
 157         return resp
 158 }