+// Run a full-featured server, including the metrics/health routes
+// that are added by service.Command.
+func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
+ logbuf = &bytes.Buffer{}
+ cluster = *s.handler.Cluster
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+
+ var configjson bytes.Buffer
+ json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
+ go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
+ for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
+ if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
+ srvaddr = "http://" + m[1]
+ break
+ }
+ }
+ if srvaddr == "" {
+ c.Fatal("timed out")
+ }
+ return
+}
+
+// Ensure uploads can take longer than API.RequestTimeout.
+//
+// Currently, this works only by accident: service.Command cancels the
+// request context as usual (there is no exemption), but
+// webdav.Handler doesn't notice if the request context is cancelled
+// while waiting to send or receive file data.
+func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
+ s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
+ _, srvaddr, _ := s.runServer(c)
+
+ var coll arvados.Collection
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveTokenV2
+ err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+ c.Assert(err, check.IsNil)
+
+ pr, pw := io.Pipe()
+ go func() {
+ time.Sleep(time.Second)
+ pw.Write(make([]byte, 10000000))
+ pw.Close()
+ }()
+ req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ time.Sleep(time.Second)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(len(body), check.Equals, 10000000)
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+ cluster, srvaddr, _ := s.runServer(c)
+ req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
+ req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ body, _ := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
+}
+
+func (s *IntegrationSuite) TestMetrics(c *check.C) {
+ cluster, srvaddr, _ := s.runServer(c)
+
+ req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ _, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ time.Sleep(metricsUpdateInterval * 2)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ req.Header.Set("Authorization", "Bearer badtoken")
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ type summary struct {
+ SampleCount string
+ SampleSum float64
+ }
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
+ var ents []struct {
+ Name string
+ Help string
+ Type string
+ Metric []struct {
+ Label []struct {
+ Name string
+ Value string
+ }
+ Counter counter
+ Gauge gauge
+ Summary summary
+ }
+ }
+ json.NewDecoder(resp.Body).Decode(&ents)
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
+ for _, e := range ents {
+ for _, m := range e.Metric {
+ labels := map[string]string{}
+ for _, lbl := range m.Label {
+ labels[lbl.Name] = lbl.Value
+ }
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
+ }
+ }
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(445))
+
+ // If the Host header indicates a collection, /metrics.json
+ // refers to a file in the collection -- the metrics handler
+ // must not intercept that route. Ditto health check paths.
+ for _, path := range []string{"/metrics.json", "/_health/ping"} {
+ c.Logf("path: %q", path)
+ req, _ = http.NewRequest("GET", srvaddr+path, nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ }
+}
+