Merge branch '18184-singularity-374'
[arvados.git] / services / keep-web / server_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "encoding/json"
11         "fmt"
12         "io"
13         "io/ioutil"
14         "net"
15         "net/http"
16         "os"
17         "os/exec"
18         "strings"
19         "testing"
20
21         "git.arvados.org/arvados.git/lib/config"
22         "git.arvados.org/arvados.git/sdk/go/arvados"
23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
26         "git.arvados.org/arvados.git/sdk/go/keepclient"
27         check "gopkg.in/check.v1"
28 )
29
30 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
31
32 var _ = check.Suite(&IntegrationSuite{})
33
34 // IntegrationSuite tests need an API server and a keep-web server
35 type IntegrationSuite struct {
36         testServer *server
37         ArvConfig  *arvados.Config
38 }
39
40 func (s *IntegrationSuite) TestNoToken(c *check.C) {
41         for _, token := range []string{
42                 "",
43                 "bogustoken",
44         } {
45                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
46                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
47                 c.Check(body, check.Equals, notFoundMessage+"\n")
48
49                 if token != "" {
50                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
51                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
52                         c.Check(body, check.Equals, notFoundMessage+"\n")
53                 }
54
55                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
56                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
57                 c.Check(body, check.Equals, notFoundMessage+"\n")
58         }
59 }
60
61 // TODO: Move most cases to functional tests -- at least use Go's own
62 // http client instead of forking curl. Just leave enough of an
63 // integration test to assure that the documented way of invoking curl
64 // really works against the server.
65 func (s *IntegrationSuite) Test404(c *check.C) {
66         for _, uri := range []string{
67                 // Routing errors (always 404 regardless of what's stored in Keep)
68                 "/foo",
69                 "/download",
70                 "/collections",
71                 "/collections/",
72                 // Implicit/generated index is not implemented yet;
73                 // until then, return 404.
74                 "/collections/" + arvadostest.FooCollection,
75                 "/collections/" + arvadostest.FooCollection + "/",
76                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
77                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
78                 // Non-existent file in collection
79                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
80                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
81                 // Non-existent collection
82                 "/collections/" + arvadostest.NonexistentCollection,
83                 "/collections/" + arvadostest.NonexistentCollection + "/",
84                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
85                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
86         } {
87                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
88                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
89                 if len(body) > 0 {
90                         c.Check(body, check.Equals, notFoundMessage+"\n")
91                 }
92         }
93 }
94
95 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
96         if testing.Short() {
97                 c.Skip("skipping 1GB integration test in short mode")
98         }
99         s.test100BlockFile(c, 10000000)
100 }
101
102 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
103         if testing.Short() {
104                 // 3 MB
105                 s.test100BlockFile(c, 30000)
106         } else {
107                 // 300 MB
108                 s.test100BlockFile(c, 3000000)
109         }
110 }
111
112 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
113         testdata := make([]byte, blocksize)
114         for i := 0; i < blocksize; i++ {
115                 testdata[i] = byte(' ')
116         }
117         arv, err := arvadosclient.MakeArvadosClient()
118         c.Assert(err, check.Equals, nil)
119         arv.ApiToken = arvadostest.ActiveToken
120         kc, err := keepclient.MakeKeepClient(arv)
121         c.Assert(err, check.Equals, nil)
122         loc, _, err := kc.PutB(testdata[:])
123         c.Assert(err, check.Equals, nil)
124         mtext := "."
125         for i := 0; i < 100; i++ {
126                 mtext = mtext + " " + loc
127         }
128         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
129         coll := map[string]interface{}{}
130         err = arv.Create("collections",
131                 map[string]interface{}{
132                         "collection": map[string]interface{}{
133                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
134                                 "manifest_text": mtext,
135                         },
136                 }, &coll)
137         c.Assert(err, check.Equals, nil)
138         uuid := coll["uuid"].(string)
139
140         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
141         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
142         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
143         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
144         c.Check(size, check.Equals, int64(blocksize)*100)
145 }
146
147 type curlCase struct {
148         auth    string
149         host    string
150         path    string
151         dataMD5 string
152 }
153
154 func (s *IntegrationSuite) Test200(c *check.C) {
155         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
156         for _, spec := range []curlCase{
157                 // My collection
158                 {
159                         auth:    arvadostest.ActiveToken,
160                         host:    arvadostest.FooCollection + "--collections.example.com",
161                         path:    "/foo",
162                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
163                 },
164                 {
165                         auth:    arvadostest.ActiveToken,
166                         host:    arvadostest.FooCollection + ".collections.example.com",
167                         path:    "/foo",
168                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
169                 },
170                 {
171                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
172                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
173                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
174                 },
175                 {
176                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
177                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
178                 },
179                 {
180                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
181                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
182                 },
183                 {
184                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
185                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
186                 },
187                 {
188                         auth:    "tokensobogus",
189                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
190                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
191                 },
192                 {
193                         auth:    arvadostest.ActiveToken,
194                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
195                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
196                 },
197                 {
198                         auth:    arvadostest.AnonymousToken,
199                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
200                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
201                 },
202
203                 // Anonymously accessible data
204                 {
205                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
206                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
207                 },
208                 {
209                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
210                         path:    "/Hello%20world.txt",
211                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
212                 },
213                 {
214                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
215                         path:    "/_/Hello%20world.txt",
216                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
217                 },
218                 {
219                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
220                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
221                 },
222                 {
223                         auth:    arvadostest.ActiveToken,
224                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
225                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
226                 },
227                 {
228                         auth:    arvadostest.SpectatorToken,
229                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
230                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
231                 },
232                 {
233                         auth:    arvadostest.SpectatorToken,
234                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
235                         path:    "/Hello%20world.txt",
236                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
237                 },
238                 {
239                         auth:    arvadostest.SpectatorToken,
240                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
241                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
242                 },
243         } {
244                 host := spec.host
245                 if host == "" {
246                         host = "collections.example.com"
247                 }
248                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
249                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
250                 if strings.HasSuffix(spec.path, ".txt") {
251                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
252                         // TODO: Check some types that aren't
253                         // automatically detected by Go's http server
254                         // by sniffing the content.
255                 }
256                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
257         }
258 }
259
260 // Return header block and body.
261 func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
262         curlArgs := []string{"--silent", "--show-error", "--include"}
263         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
264         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
265         if strings.Contains(auth, " ") {
266                 // caller supplied entire Authorization header value
267                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
268         } else if auth != "" {
269                 // caller supplied Arvados token
270                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
271         }
272         curlArgs = append(curlArgs, args...)
273         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
274         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
275         cmd := exec.Command("curl", curlArgs...)
276         stdout, err := cmd.StdoutPipe()
277         c.Assert(err, check.IsNil)
278         cmd.Stderr = os.Stderr
279         err = cmd.Start()
280         c.Assert(err, check.IsNil)
281         buf := make([]byte, 2<<27)
282         n, err := io.ReadFull(stdout, buf)
283         // Discard (but measure size of) anything past 128 MiB.
284         var discarded int64
285         if err == io.ErrUnexpectedEOF {
286                 buf = buf[:n]
287         } else {
288                 c.Assert(err, check.IsNil)
289                 discarded, err = io.Copy(ioutil.Discard, stdout)
290                 c.Assert(err, check.IsNil)
291         }
292         err = cmd.Wait()
293         // Without "-f", curl exits 0 as long as it gets a valid HTTP
294         // response from the server, even if the response status
295         // indicates that the request failed. In our test suite, we
296         // always expect a valid HTTP response, and we parse the
297         // headers ourselves. If curl exits non-zero, our testing
298         // environment is broken.
299         c.Assert(err, check.Equals, nil)
300         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
301         c.Assert(len(hdrsAndBody), check.Equals, 2)
302         hdr = hdrsAndBody[0]
303         bodyPart = hdrsAndBody[1]
304         bodySize = int64(len(bodyPart)) + discarded
305         return
306 }
307
308 func (s *IntegrationSuite) TestMetrics(c *check.C) {
309         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
310         origin := "http://" + s.testServer.Addr
311         req, _ := http.NewRequest("GET", origin+"/notfound", nil)
312         _, err := http.DefaultClient.Do(req)
313         c.Assert(err, check.IsNil)
314         req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
315         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
316         resp, err := http.DefaultClient.Do(req)
317         c.Assert(err, check.IsNil)
318         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
319         for i := 0; i < 2; i++ {
320                 req, _ = http.NewRequest("GET", origin+"/foo", nil)
321                 req.Host = arvadostest.FooCollection + ".example.com"
322                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
323                 resp, err = http.DefaultClient.Do(req)
324                 c.Assert(err, check.IsNil)
325                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
326                 buf, _ := ioutil.ReadAll(resp.Body)
327                 c.Check(buf, check.DeepEquals, []byte("foo"))
328                 resp.Body.Close()
329         }
330
331         s.testServer.Config.Cache.updateGauges()
332
333         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
334         resp, err = http.DefaultClient.Do(req)
335         c.Assert(err, check.IsNil)
336         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
337
338         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
339         req.Header.Set("Authorization", "Bearer badtoken")
340         resp, err = http.DefaultClient.Do(req)
341         c.Assert(err, check.IsNil)
342         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
343
344         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
345         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
346         resp, err = http.DefaultClient.Do(req)
347         c.Assert(err, check.IsNil)
348         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
349         type summary struct {
350                 SampleCount string
351                 SampleSum   float64
352         }
353         type counter struct {
354                 Value int64
355         }
356         type gauge struct {
357                 Value float64
358         }
359         var ents []struct {
360                 Name   string
361                 Help   string
362                 Type   string
363                 Metric []struct {
364                         Label []struct {
365                                 Name  string
366                                 Value string
367                         }
368                         Counter counter
369                         Gauge   gauge
370                         Summary summary
371                 }
372         }
373         json.NewDecoder(resp.Body).Decode(&ents)
374         summaries := map[string]summary{}
375         gauges := map[string]gauge{}
376         counters := map[string]counter{}
377         for _, e := range ents {
378                 for _, m := range e.Metric {
379                         labels := map[string]string{}
380                         for _, lbl := range m.Label {
381                                 labels[lbl.Name] = lbl.Value
382                         }
383                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
384                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
385                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
386                 }
387         }
388         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
389         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
390         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
391         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
392         c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
393         c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
394         c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
395         c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
396         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
397         // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
398         c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
399
400         // If the Host header indicates a collection, /metrics.json
401         // refers to a file in the collection -- the metrics handler
402         // must not intercept that route.
403         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
404         req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
405         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
406         resp, err = http.DefaultClient.Do(req)
407         c.Assert(err, check.IsNil)
408         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
409 }
410
411 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
412         arvadostest.ResetDB(c)
413         arvadostest.StartKeep(2, true)
414
415         arv, err := arvadosclient.MakeArvadosClient()
416         c.Assert(err, check.Equals, nil)
417         arv.ApiToken = arvadostest.ActiveToken
418         kc, err := keepclient.MakeKeepClient(arv)
419         c.Assert(err, check.Equals, nil)
420         kc.PutB([]byte("Hello world\n"))
421         kc.PutB([]byte("foo"))
422         kc.PutB([]byte("foobar"))
423         kc.PutB([]byte("waz"))
424 }
425
426 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
427         arvadostest.StopKeep(2)
428 }
429
430 func (s *IntegrationSuite) SetUpTest(c *check.C) {
431         arvadostest.ResetEnv()
432         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
433         ldr.Path = "-"
434         arvCfg, err := ldr.Load()
435         c.Check(err, check.IsNil)
436         cfg := newConfig(ctxlog.TestLogger(c), arvCfg)
437         c.Assert(err, check.IsNil)
438         cfg.Client = arvados.Client{
439                 APIHost:  testAPIHost,
440                 Insecure: true,
441         }
442         listen := "127.0.0.1:0"
443         cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
444         cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
445         cfg.cluster.ManagementToken = arvadostest.ManagementToken
446         cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
447         cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
448         s.ArvConfig = arvCfg
449         s.testServer = &server{Config: cfg}
450         err = s.testServer.Start(ctxlog.TestLogger(c))
451         c.Assert(err, check.Equals, nil)
452 }
453
454 func (s *IntegrationSuite) TearDownTest(c *check.C) {
455         var err error
456         if s.testServer != nil {
457                 err = s.testServer.Close()
458         }
459         c.Check(err, check.Equals, nil)
460 }
461
462 // Gocheck boilerplate
463 func Test(t *testing.T) {
464         check.TestingT(t)
465 }