Merge branch '16723-kill-vs-requeue'
[arvados.git] / services / keepproxy / keepproxy_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "errors"
11         "fmt"
12         "io/ioutil"
13         "math/rand"
14         "net/http"
15         "net/http/httptest"
16         "strings"
17         "sync"
18         "testing"
19         "time"
20
21         "git.arvados.org/arvados.git/lib/config"
22         "git.arvados.org/arvados.git/sdk/go/arvados"
23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
26         "git.arvados.org/arvados.git/sdk/go/keepclient"
27         log "github.com/sirupsen/logrus"
28
29         . "gopkg.in/check.v1"
30 )
31
32 // Gocheck boilerplate
33 func Test(t *testing.T) {
34         TestingT(t)
35 }
36
37 // Gocheck boilerplate
38 var _ = Suite(&ServerRequiredSuite{})
39
40 // Tests that require the Keep server running
41 type ServerRequiredSuite struct{}
42
43 // Gocheck boilerplate
44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
45
46 // Tests that require the Keep servers running as defined in config.yml
47 type ServerRequiredConfigYmlSuite struct{}
48
49 // Gocheck boilerplate
50 var _ = Suite(&NoKeepServerSuite{})
51
52 // Test with no keepserver to simulate errors
53 type NoKeepServerSuite struct{}
54
55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
56
57 // Wait (up to 1 second) for keepproxy to listen on a port. This
58 // avoids a race condition where we hit a "connection refused" error
59 // because we start testing the proxy too soon.
60 func waitForListener() {
61         const (
62                 ms = 5
63         )
64         for i := 0; listener == nil && i < 10000; i += ms {
65                 time.Sleep(ms * time.Millisecond)
66         }
67         if listener == nil {
68                 panic("Timed out waiting for listener to start")
69         }
70 }
71
72 func closeListener() {
73         if listener != nil {
74                 listener.Close()
75         }
76 }
77
78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
79         arvadostest.StartAPI()
80         arvadostest.StartKeep(2, false)
81 }
82
83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
84         arvadostest.ResetEnv()
85 }
86
87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
88         arvadostest.StopKeep(2)
89         arvadostest.StopAPI()
90 }
91
92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
93         arvadostest.StartAPI()
94         // config.yml defines 4 keepstores
95         arvadostest.StartKeep(4, false)
96 }
97
98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
99         arvadostest.ResetEnv()
100 }
101
102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
103         arvadostest.StopKeep(4)
104         arvadostest.StopAPI()
105 }
106
107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
108         arvadostest.StartAPI()
109         // We need API to have some keep services listed, but the
110         // services themselves should be unresponsive.
111         arvadostest.StartKeep(2, false)
112         arvadostest.StopKeep(2)
113 }
114
115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
116         arvadostest.ResetEnv()
117 }
118
119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
120         arvadostest.StopAPI()
121 }
122
123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
124         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
125         c.Assert(err, Equals, nil)
126         cluster, err := cfg.GetCluster("")
127         c.Assert(err, Equals, nil)
128
129         if !loadKeepstoresFromConfig {
130                 // Do not load Keepstore InternalURLs from the config file
131                 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
132         }
133
134         cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
135
136         listener = nil
137         go func() {
138                 run(log.New(), cluster)
139                 defer closeListener()
140         }()
141         waitForListener()
142
143         client := arvados.NewClientFromEnv()
144         arv, err := arvadosclient.New(client)
145         c.Assert(err, Equals, nil)
146         if bogusClientToken {
147                 arv.ApiToken = "bogus-token"
148         }
149         kc := keepclient.New(arv)
150         sr := map[string]string{
151                 TestProxyUUID: "http://" + listener.Addr().String(),
152         }
153         kc.SetServiceRoots(sr, sr, sr)
154         kc.Arvados.External = true
155
156         return kc
157 }
158
159 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
160         runProxy(c, false, false)
161         defer closeListener()
162
163         req, err := http.NewRequest("POST",
164                 "http://"+listener.Addr().String()+"/",
165                 strings.NewReader("TestViaHeader"))
166         c.Assert(err, Equals, nil)
167         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
168         resp, err := (&http.Client{}).Do(req)
169         c.Assert(err, Equals, nil)
170         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
171         c.Assert(resp.StatusCode, Equals, http.StatusOK)
172         locator, err := ioutil.ReadAll(resp.Body)
173         c.Assert(err, Equals, nil)
174         resp.Body.Close()
175
176         req, err = http.NewRequest("GET",
177                 "http://"+listener.Addr().String()+"/"+string(locator),
178                 nil)
179         c.Assert(err, Equals, nil)
180         resp, err = (&http.Client{}).Do(req)
181         c.Assert(err, Equals, nil)
182         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
183         resp.Body.Close()
184 }
185
186 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
187         kc := runProxy(c, false, false)
188         defer closeListener()
189
190         sr := map[string]string{
191                 TestProxyUUID: "http://" + listener.Addr().String(),
192         }
193         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
194
195         content := []byte("TestLoopDetection")
196         _, _, err := kc.PutB(content)
197         c.Check(err, ErrorMatches, `.*loop detected.*`)
198
199         hash := fmt.Sprintf("%x", md5.Sum(content))
200         _, _, _, err = kc.Get(hash)
201         c.Check(err, ErrorMatches, `.*loop detected.*`)
202 }
203
204 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
205         kc := runProxy(c, false, false)
206         defer closeListener()
207
208         // Set up fake keepstore to record request headers
209         var hdr http.Header
210         ts := httptest.NewServer(http.HandlerFunc(
211                 func(w http.ResponseWriter, r *http.Request) {
212                         hdr = r.Header
213                         http.Error(w, "Error", http.StatusInternalServerError)
214                 }))
215         defer ts.Close()
216
217         // Point keepproxy router's keepclient to the fake keepstore
218         sr := map[string]string{
219                 TestProxyUUID: ts.URL,
220         }
221         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
222
223         // Set up client to ask for storage classes to keepproxy
224         kc.StorageClasses = []string{"secure"}
225         content := []byte("Very important data")
226         _, _, err := kc.PutB(content)
227         c.Check(err, NotNil)
228         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
229 }
230
231 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
232         kc := runProxy(c, false, false)
233         defer closeListener()
234
235         content := []byte("TestDesiredReplicas")
236         hash := fmt.Sprintf("%x", md5.Sum(content))
237
238         for _, kc.Want_replicas = range []int{0, 1, 2} {
239                 locator, rep, err := kc.PutB(content)
240                 c.Check(err, Equals, nil)
241                 c.Check(rep, Equals, kc.Want_replicas)
242                 if rep > 0 {
243                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
244                 }
245         }
246 }
247
248 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
249         kc := runProxy(c, false, false)
250         defer closeListener()
251
252         content := []byte("TestPutWrongContentLength")
253         hash := fmt.Sprintf("%x", md5.Sum(content))
254
255         // If we use http.Client to send these requests to the network
256         // server we just started, the Go http library automatically
257         // fixes the invalid Content-Length header. In order to test
258         // our server behavior, we have to call the handler directly
259         // using an httptest.ResponseRecorder.
260         rtr := MakeRESTRouter(kc, 10*time.Second, "")
261
262         type testcase struct {
263                 sendLength   string
264                 expectStatus int
265         }
266
267         for _, t := range []testcase{
268                 {"1", http.StatusBadRequest},
269                 {"", http.StatusLengthRequired},
270                 {"-1", http.StatusLengthRequired},
271                 {"abcdef", http.StatusLengthRequired},
272         } {
273                 req, err := http.NewRequest("PUT",
274                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
275                         bytes.NewReader(content))
276                 c.Assert(err, IsNil)
277                 req.Header.Set("Content-Length", t.sendLength)
278                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
279                 req.Header.Set("Content-Type", "application/octet-stream")
280
281                 resp := httptest.NewRecorder()
282                 rtr.ServeHTTP(resp, req)
283                 c.Check(resp.Code, Equals, t.expectStatus)
284         }
285 }
286
287 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
288         kc := runProxy(c, false, false)
289         defer closeListener()
290         router.(*proxyHandler).timeout = time.Nanosecond
291
292         buf := make([]byte, 1<<20)
293         rand.Read(buf)
294         var wg sync.WaitGroup
295         for i := 0; i < 128; i++ {
296                 wg.Add(1)
297                 go func() {
298                         defer wg.Done()
299                         kc.PutB(buf)
300                 }()
301         }
302         done := make(chan bool)
303         go func() {
304                 wg.Wait()
305                 close(done)
306         }()
307         select {
308         case <-done:
309         case <-time.After(10 * time.Second):
310                 c.Error("timeout")
311         }
312 }
313
314 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
315         kc := runProxy(c, false, false)
316         defer closeListener()
317
318         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
319         var hash2 string
320
321         {
322                 _, _, err := kc.Ask(hash)
323                 c.Check(err, Equals, keepclient.BlockNotFound)
324                 c.Log("Finished Ask (expected BlockNotFound)")
325         }
326
327         {
328                 reader, _, _, err := kc.Get(hash)
329                 c.Check(reader, Equals, nil)
330                 c.Check(err, Equals, keepclient.BlockNotFound)
331                 c.Log("Finished Get (expected BlockNotFound)")
332         }
333
334         // Note in bug #5309 among other errors keepproxy would set
335         // Content-Length incorrectly on the 404 BlockNotFound response, this
336         // would result in a protocol violation that would prevent reuse of the
337         // connection, which would manifest by the next attempt to use the
338         // connection (in this case the PutB below) failing.  So to test for
339         // that bug it's necessary to trigger an error response (such as
340         // BlockNotFound) and then do something else with the same httpClient
341         // connection.
342
343         {
344                 var rep int
345                 var err error
346                 hash2, rep, err = kc.PutB([]byte("foo"))
347                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
348                 c.Check(rep, Equals, 2)
349                 c.Check(err, Equals, nil)
350                 c.Log("Finished PutB (expected success)")
351         }
352
353         {
354                 blocklen, _, err := kc.Ask(hash2)
355                 c.Assert(err, Equals, nil)
356                 c.Check(blocklen, Equals, int64(3))
357                 c.Log("Finished Ask (expected success)")
358         }
359
360         {
361                 reader, blocklen, _, err := kc.Get(hash2)
362                 c.Assert(err, Equals, nil)
363                 all, err := ioutil.ReadAll(reader)
364                 c.Check(err, IsNil)
365                 c.Check(all, DeepEquals, []byte("foo"))
366                 c.Check(blocklen, Equals, int64(3))
367                 c.Log("Finished Get (expected success)")
368         }
369
370         {
371                 var rep int
372                 var err error
373                 hash2, rep, err = kc.PutB([]byte(""))
374                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
375                 c.Check(rep, Equals, 2)
376                 c.Check(err, Equals, nil)
377                 c.Log("Finished PutB zero block")
378         }
379
380         {
381                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
382                 c.Assert(err, Equals, nil)
383                 all, err := ioutil.ReadAll(reader)
384                 c.Check(err, IsNil)
385                 c.Check(all, DeepEquals, []byte(""))
386                 c.Check(blocklen, Equals, int64(0))
387                 c.Log("Finished Get zero block")
388         }
389 }
390
391 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
392         kc := runProxy(c, true, false)
393         defer closeListener()
394
395         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
396
397         _, _, err := kc.Ask(hash)
398         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
399
400         hash2, rep, err := kc.PutB([]byte("bar"))
401         c.Check(hash2, Equals, "")
402         c.Check(rep, Equals, 0)
403         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
404
405         blocklen, _, err := kc.Ask(hash)
406         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
407         c.Check(err, ErrorMatches, ".*not found.*")
408         c.Check(blocklen, Equals, int64(0))
409
410         _, blocklen, _, err = kc.Get(hash)
411         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
412         c.Check(err, ErrorMatches, ".*not found.*")
413         c.Check(blocklen, Equals, int64(0))
414
415 }
416
417 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
418         runProxy(c, false, false)
419         defer closeListener()
420
421         {
422                 client := http.Client{}
423                 req, err := http.NewRequest("OPTIONS",
424                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
425                         nil)
426                 c.Assert(err, IsNil)
427                 req.Header.Add("Access-Control-Request-Method", "PUT")
428                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
429                 resp, err := client.Do(req)
430                 c.Check(err, Equals, nil)
431                 c.Check(resp.StatusCode, Equals, 200)
432                 body, err := ioutil.ReadAll(resp.Body)
433                 c.Check(err, IsNil)
434                 c.Check(string(body), Equals, "")
435                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
436                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
437         }
438
439         {
440                 resp, err := http.Get(
441                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
442                 c.Check(err, Equals, nil)
443                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
444                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
445         }
446 }
447
448 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
449         runProxy(c, false, false)
450         defer closeListener()
451
452         {
453                 client := http.Client{}
454                 req, err := http.NewRequest("POST",
455                         "http://"+listener.Addr().String()+"/",
456                         strings.NewReader("qux"))
457                 c.Check(err, IsNil)
458                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
459                 req.Header.Add("Content-Type", "application/octet-stream")
460                 resp, err := client.Do(req)
461                 c.Check(err, Equals, nil)
462                 body, err := ioutil.ReadAll(resp.Body)
463                 c.Check(err, Equals, nil)
464                 c.Check(string(body), Matches,
465                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
466         }
467 }
468
469 func (s *ServerRequiredSuite) TestStripHint(c *C) {
470         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
471                 Equals,
472                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
473         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
474                 Equals,
475                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
476         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
477                 Equals,
478                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
479         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
480                 Equals,
481                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
482
483 }
484
485 // Test GetIndex
486 //   Put one block, with 2 replicas
487 //   With no prefix (expect the block locator, twice)
488 //   With an existing prefix (expect the block locator, twice)
489 //   With a valid but non-existing prefix (expect "\n")
490 //   With an invalid prefix (expect error)
491 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
492         getIndexWorker(c, false)
493 }
494
495 // Test GetIndex
496 //   Uses config.yml
497 //   Put one block, with 2 replicas
498 //   With no prefix (expect the block locator, twice)
499 //   With an existing prefix (expect the block locator, twice)
500 //   With a valid but non-existing prefix (expect "\n")
501 //   With an invalid prefix (expect error)
502 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
503         getIndexWorker(c, true)
504 }
505
506 func getIndexWorker(c *C, useConfig bool) {
507         kc := runProxy(c, false, useConfig)
508         defer closeListener()
509
510         // Put "index-data" blocks
511         data := []byte("index-data")
512         hash := fmt.Sprintf("%x", md5.Sum(data))
513
514         hash2, rep, err := kc.PutB(data)
515         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
516         c.Check(rep, Equals, 2)
517         c.Check(err, Equals, nil)
518
519         reader, blocklen, _, err := kc.Get(hash)
520         c.Assert(err, IsNil)
521         c.Check(blocklen, Equals, int64(10))
522         all, err := ioutil.ReadAll(reader)
523         c.Assert(err, IsNil)
524         c.Check(all, DeepEquals, data)
525
526         // Put some more blocks
527         _, _, err = kc.PutB([]byte("some-more-index-data"))
528         c.Check(err, IsNil)
529
530         kc.Arvados.ApiToken = arvadostest.SystemRootToken
531
532         // Invoke GetIndex
533         for _, spec := range []struct {
534                 prefix         string
535                 expectTestHash bool
536                 expectOther    bool
537         }{
538                 {"", true, true},         // with no prefix
539                 {hash[:3], true, false},  // with matching prefix
540                 {"abcdef", false, false}, // with no such prefix
541         } {
542                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
543                 c.Assert(err, Equals, nil)
544                 indexResp, err := ioutil.ReadAll(indexReader)
545                 c.Assert(err, Equals, nil)
546                 locators := strings.Split(string(indexResp), "\n")
547                 gotTestHash := 0
548                 gotOther := 0
549                 for _, locator := range locators {
550                         if locator == "" {
551                                 continue
552                         }
553                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
554                         if locator[:32] == hash {
555                                 gotTestHash++
556                         } else {
557                                 gotOther++
558                         }
559                 }
560                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
561                 c.Check(gotOther > 0, Equals, spec.expectOther)
562         }
563
564         // GetIndex with invalid prefix
565         _, err = kc.GetIndex(TestProxyUUID, "xyz")
566         c.Assert((err != nil), Equals, true)
567 }
568
569 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
570         kc := runProxy(c, false, false)
571         defer closeListener()
572         hash, _, err := kc.PutB([]byte("shareddata"))
573         c.Check(err, IsNil)
574         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
575         rdr, _, _, err := kc.Get(hash)
576         c.Assert(err, IsNil)
577         data, err := ioutil.ReadAll(rdr)
578         c.Check(err, IsNil)
579         c.Check(data, DeepEquals, []byte("shareddata"))
580 }
581
582 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
583         kc := runProxy(c, false, false)
584         defer closeListener()
585
586         // Put a test block
587         hash, rep, err := kc.PutB([]byte("foo"))
588         c.Check(err, IsNil)
589         c.Check(rep, Equals, 2)
590
591         for _, badToken := range []string{
592                 "nosuchtoken",
593                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
594         } {
595                 kc.Arvados.ApiToken = badToken
596
597                 // Ask and Get will fail only if the upstream
598                 // keepstore server checks for valid signatures.
599                 // Without knowing the blob signing key, there is no
600                 // way for keepproxy to know whether a given token is
601                 // permitted to read a block.  So these tests fail:
602                 if false {
603                         _, _, err = kc.Ask(hash)
604                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
605                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
606                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
607
608                         _, _, _, err = kc.Get(hash)
609                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
610                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
611                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
612                 }
613
614                 _, _, err = kc.PutB([]byte("foo"))
615                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
616         }
617 }
618
619 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
620         kc := runProxy(c, false, false)
621         defer closeListener()
622
623         // Point keepproxy at a non-existent keepstore
624         locals := map[string]string{
625                 TestProxyUUID: "http://localhost:12345",
626         }
627         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
628
629         // Ask should result in temporary bad gateway error
630         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
631         _, _, err := kc.Ask(hash)
632         c.Check(err, NotNil)
633         errNotFound, _ := err.(*keepclient.ErrNotFound)
634         c.Check(errNotFound.Temporary(), Equals, true)
635         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
636
637         // Get should result in temporary bad gateway error
638         _, _, _, err = kc.Get(hash)
639         c.Check(err, NotNil)
640         errNotFound, _ = err.(*keepclient.ErrNotFound)
641         c.Check(errNotFound.Temporary(), Equals, true)
642         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
643 }
644
645 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
646         kc := runProxy(c, false, false)
647         defer closeListener()
648
649         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
650         for _, f := range []func() error{
651                 func() error {
652                         _, _, err := kc.Ask(hash)
653                         return err
654                 },
655                 func() error {
656                         _, _, _, err := kc.Get(hash)
657                         return err
658                 },
659         } {
660                 err := f()
661                 c.Assert(err, NotNil)
662                 errNotFound, _ := err.(*keepclient.ErrNotFound)
663                 c.Check(errNotFound.Temporary(), Equals, true)
664                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
665         }
666 }
667
668 func (s *ServerRequiredSuite) TestPing(c *C) {
669         kc := runProxy(c, false, false)
670         defer closeListener()
671
672         rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
673
674         req, err := http.NewRequest("GET",
675                 "http://"+listener.Addr().String()+"/_health/ping",
676                 nil)
677         c.Assert(err, IsNil)
678         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
679
680         resp := httptest.NewRecorder()
681         rtr.ServeHTTP(resp, req)
682         c.Check(resp.Code, Equals, 200)
683         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
684 }