services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "net"
  16         "net/http"
  17         "os"
  18         "os/exec"
  19         "strings"
  20         "testing"
  21
  22         "git.arvados.org/arvados.git/lib/config"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  27         "git.arvados.org/arvados.git/sdk/go/keepclient"
  28         check "gopkg.in/check.v1"
  29 )
  30
  31 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  32
  33 var _ = check.Suite(&IntegrationSuite{})
  34
  35 // IntegrationSuite tests need an API server and a keep-web server
  36 type IntegrationSuite struct {
  37         testServer *server
  38         ArvConfig  *arvados.Config
  39 }
  40
  41 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  42         for _, token := range []string{
  43                 "",
  44                 "bogustoken",
  45         } {
  46                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  47                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  48                 c.Check(body, check.Equals, notFoundMessage+"\n")
  49
  50                 if token != "" {
  51                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  52                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  53                         c.Check(body, check.Equals, notFoundMessage+"\n")
  54                 }
  55
  56                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  57                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  58                 c.Check(body, check.Equals, notFoundMessage+"\n")
  59         }
  60 }
  61
  62 // TODO: Move most cases to functional tests -- at least use Go's own
  63 // http client instead of forking curl. Just leave enough of an
  64 // integration test to assure that the documented way of invoking curl
  65 // really works against the server.
  66 func (s *IntegrationSuite) Test404(c *check.C) {
  67         for _, uri := range []string{
  68                 // Routing errors (always 404 regardless of what's stored in Keep)
  69                 "/foo",
  70                 "/download",
  71                 "/collections",
  72                 "/collections/",
  73                 // Implicit/generated index is not implemented yet;
  74                 // until then, return 404.
  75                 "/collections/" + arvadostest.FooCollection,
  76                 "/collections/" + arvadostest.FooCollection + "/",
  77                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  78                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  79                 // Non-existent file in collection
  80                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  81                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  82                 // Non-existent collection
  83                 "/collections/" + arvadostest.NonexistentCollection,
  84                 "/collections/" + arvadostest.NonexistentCollection + "/",
  85                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  86                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  87         } {
  88                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  89                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  90                 if len(body) > 0 {
  91                         c.Check(body, check.Equals, notFoundMessage+"\n")
  92                 }
  93         }
  94 }
  95
  96 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  97         if testing.Short() {
  98                 c.Skip("skipping 1GB integration test in short mode")
  99         }
 100         s.test100BlockFile(c, 10000000)
 101 }
 102
 103 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 104         if testing.Short() {
 105                 // 3 MB
 106                 s.test100BlockFile(c, 30000)
 107         } else {
 108                 // 300 MB
 109                 s.test100BlockFile(c, 3000000)
 110         }
 111 }
 112
 113 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 114         testdata := make([]byte, blocksize)
 115         for i := 0; i < blocksize; i++ {
 116                 testdata[i] = byte(' ')
 117         }
 118         arv, err := arvadosclient.MakeArvadosClient()
 119         c.Assert(err, check.Equals, nil)
 120         arv.ApiToken = arvadostest.ActiveToken
 121         kc, err := keepclient.MakeKeepClient(arv)
 122         c.Assert(err, check.Equals, nil)
 123         loc, _, err := kc.PutB(testdata[:])
 124         c.Assert(err, check.Equals, nil)
 125         mtext := "."
 126         for i := 0; i < 100; i++ {
 127                 mtext = mtext + " " + loc
 128         }
 129         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 130         coll := map[string]interface{}{}
 131         err = arv.Create("collections",
 132                 map[string]interface{}{
 133                         "collection": map[string]interface{}{
 134                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 135                                 "manifest_text": mtext,
 136                         },
 137                 }, &coll)
 138         c.Assert(err, check.Equals, nil)
 139         uuid := coll["uuid"].(string)
 140
 141         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 142         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 143         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 144         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 145         c.Check(size, check.Equals, int64(blocksize)*100)
 146 }
 147
 148 type curlCase struct {
 149         auth    string
 150         host    string
 151         path    string
 152         dataMD5 string
 153 }
 154
 155 func (s *IntegrationSuite) Test200(c *check.C) {
 156         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 157         for _, spec := range []curlCase{
 158                 // My collection
 159                 {
 160                         auth:    arvadostest.ActiveToken,
 161                         host:    arvadostest.FooCollection + "--collections.example.com",
 162                         path:    "/foo",
 163                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 164                 },
 165                 {
 166                         auth:    arvadostest.ActiveToken,
 167                         host:    arvadostest.FooCollection + ".collections.example.com",
 168                         path:    "/foo",
 169                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 170                 },
 171                 {
 172                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 173                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 174                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 175                 },
 176                 {
 177                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 178                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 179                 },
 180                 {
 181                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 182                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 183                 },
 184                 {
 185                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 186                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 187                 },
 188                 {
 189                         auth:    "tokensobogus",
 190                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 191                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 192                 },
 193                 {
 194                         auth:    arvadostest.ActiveToken,
 195                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 196                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 197                 },
 198                 {
 199                         auth:    arvadostest.AnonymousToken,
 200                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 201                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 202                 },
 203
 204                 // Anonymously accessible data
 205                 {
 206                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 207                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 208                 },
 209                 {
 210                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 211                         path:    "/Hello%20world.txt",
 212                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 213                 },
 214                 {
 215                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 216                         path:    "/_/Hello%20world.txt",
 217                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 218                 },
 219                 {
 220                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 221                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 222                 },
 223                 {
 224                         auth:    arvadostest.ActiveToken,
 225                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 226                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 227                 },
 228                 {
 229                         auth:    arvadostest.SpectatorToken,
 230                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 231                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 232                 },
 233                 {
 234                         auth:    arvadostest.SpectatorToken,
 235                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 236                         path:    "/Hello%20world.txt",
 237                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 238                 },
 239                 {
 240                         auth:    arvadostest.SpectatorToken,
 241                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 242                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 243                 },
 244         } {
 245                 host := spec.host
 246                 if host == "" {
 247                         host = "collections.example.com"
 248                 }
 249                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 250                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 251                 if strings.HasSuffix(spec.path, ".txt") {
 252                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 253                         // TODO: Check some types that aren't
 254                         // automatically detected by Go's http server
 255                         // by sniffing the content.
 256                 }
 257                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 258         }
 259 }
 260
 261 // Return header block and body.
 262 func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 263         curlArgs := []string{"--silent", "--show-error", "--include"}
 264         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 265         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 266         if strings.Contains(auth, " ") {
 267                 // caller supplied entire Authorization header value
 268                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
 269         } else if auth != "" {
 270                 // caller supplied Arvados token
 271                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 272         }
 273         curlArgs = append(curlArgs, args...)
 274         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 275         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 276         cmd := exec.Command("curl", curlArgs...)
 277         stdout, err := cmd.StdoutPipe()
 278         c.Assert(err, check.IsNil)
 279         cmd.Stderr = os.Stderr
 280         err = cmd.Start()
 281         c.Assert(err, check.IsNil)
 282         buf := make([]byte, 2<<27)
 283         n, err := io.ReadFull(stdout, buf)
 284         // Discard (but measure size of) anything past 128 MiB.
 285         var discarded int64
 286         if err == io.ErrUnexpectedEOF {
 287                 buf = buf[:n]
 288         } else {
 289                 c.Assert(err, check.IsNil)
 290                 discarded, err = io.Copy(ioutil.Discard, stdout)
 291                 c.Assert(err, check.IsNil)
 292         }
 293         err = cmd.Wait()
 294         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 295         // response from the server, even if the response status
 296         // indicates that the request failed. In our test suite, we
 297         // always expect a valid HTTP response, and we parse the
 298         // headers ourselves. If curl exits non-zero, our testing
 299         // environment is broken.
 300         c.Assert(err, check.Equals, nil)
 301         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 302         c.Assert(len(hdrsAndBody), check.Equals, 2)
 303         hdr = hdrsAndBody[0]
 304         bodyPart = hdrsAndBody[1]
 305         bodySize = int64(len(bodyPart)) + discarded
 306         return
 307 }
 308
 309 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 310         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
 311         origin := "http://" + s.testServer.Addr
 312         req, _ := http.NewRequest("GET", origin+"/notfound", nil)
 313         _, err := http.DefaultClient.Do(req)
 314         c.Assert(err, check.IsNil)
 315         req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
 316         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 317         resp, err := http.DefaultClient.Do(req)
 318         c.Assert(err, check.IsNil)
 319         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 320         for i := 0; i < 2; i++ {
 321                 req, _ = http.NewRequest("GET", origin+"/foo", nil)
 322                 req.Host = arvadostest.FooCollection + ".example.com"
 323                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 324                 resp, err = http.DefaultClient.Do(req)
 325                 c.Assert(err, check.IsNil)
 326                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 327                 buf, _ := ioutil.ReadAll(resp.Body)
 328                 c.Check(buf, check.DeepEquals, []byte("foo"))
 329                 resp.Body.Close()
 330         }
 331
 332         s.testServer.Config.Cache.updateGauges()
 333
 334         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 335         resp, err = http.DefaultClient.Do(req)
 336         c.Assert(err, check.IsNil)
 337         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 338
 339         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 340         req.Header.Set("Authorization", "Bearer badtoken")
 341         resp, err = http.DefaultClient.Do(req)
 342         c.Assert(err, check.IsNil)
 343         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 344
 345         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 346         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 347         resp, err = http.DefaultClient.Do(req)
 348         c.Assert(err, check.IsNil)
 349         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 350         type summary struct {
 351                 SampleCount string
 352                 SampleSum   float64
 353         }
 354         type counter struct {
 355                 Value int64
 356         }
 357         type gauge struct {
 358                 Value float64
 359         }
 360         var ents []struct {
 361                 Name   string
 362                 Help   string
 363                 Type   string
 364                 Metric []struct {
 365                         Label []struct {
 366                                 Name  string
 367                                 Value string
 368                         }
 369                         Counter counter
 370                         Gauge   gauge
 371                         Summary summary
 372                 }
 373         }
 374         json.NewDecoder(resp.Body).Decode(&ents)
 375         summaries := map[string]summary{}
 376         gauges := map[string]gauge{}
 377         counters := map[string]counter{}
 378         for _, e := range ents {
 379                 for _, m := range e.Metric {
 380                         labels := map[string]string{}
 381                         for _, lbl := range m.Label {
 382                                 labels[lbl.Name] = lbl.Value
 383                         }
 384                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 385                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 386                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 387                 }
 388         }
 389         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 390         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 391         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 392         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 393         c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
 394         c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
 395         c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
 396         c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
 397         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
 398         // FooCollection's cached manifest size is 45 ("1f4b0....+45")
 399         // plus one 51-byte blob signature; session fs counts 3 inodes
 400         // * 64 bytes.
 401         c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51+64*3))
 402
 403         // If the Host header indicates a collection, /metrics.json
 404         // refers to a file in the collection -- the metrics handler
 405         // must not intercept that route.
 406         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 407         req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 408         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 409         resp, err = http.DefaultClient.Do(req)
 410         c.Assert(err, check.IsNil)
 411         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 412 }
 413
 414 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 415         arvadostest.ResetDB(c)
 416         arvadostest.StartKeep(2, true)
 417
 418         arv, err := arvadosclient.MakeArvadosClient()
 419         c.Assert(err, check.Equals, nil)
 420         arv.ApiToken = arvadostest.ActiveToken
 421         kc, err := keepclient.MakeKeepClient(arv)
 422         c.Assert(err, check.Equals, nil)
 423         kc.PutB([]byte("Hello world\n"))
 424         kc.PutB([]byte("foo"))
 425         kc.PutB([]byte("foobar"))
 426         kc.PutB([]byte("waz"))
 427 }
 428
 429 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 430         arvadostest.StopKeep(2)
 431 }
 432
 433 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 434         arvadostest.ResetEnv()
 435         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
 436         ldr.Path = "-"
 437         arvCfg, err := ldr.Load()
 438         c.Check(err, check.IsNil)
 439         cfg := newConfig(ctxlog.TestLogger(c), arvCfg)
 440         c.Assert(err, check.IsNil)
 441         cfg.Client = arvados.Client{
 442                 APIHost:  testAPIHost,
 443                 Insecure: true,
 444         }
 445         listen := "127.0.0.1:0"
 446         cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 447         cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 448         cfg.cluster.ManagementToken = arvadostest.ManagementToken
 449         cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
 450         cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 451         s.ArvConfig = arvCfg
 452         s.testServer = &server{Config: cfg}
 453         logger := ctxlog.TestLogger(c)
 454         ctx := ctxlog.Context(context.Background(), logger)
 455         err = s.testServer.Start(ctx, logger)
 456         c.Assert(err, check.Equals, nil)
 457 }
 458
 459 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 460         var err error
 461         if s.testServer != nil {
 462                 err = s.testServer.Close()
 463         }
 464         c.Check(err, check.Equals, nil)
 465 }
 466
 467 // Gocheck boilerplate
 468 func Test(t *testing.T) {
 469         check.TestingT(t)
 470 }