Merge branch '13879-arvados-server' closes #13879
[arvados.git] / services / keepproxy / keepproxy_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "errors"
11         "fmt"
12         "io/ioutil"
13         "math/rand"
14         "net/http"
15         "net/http/httptest"
16         "os"
17         "strings"
18         "sync"
19         "testing"
20         "time"
21
22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
25
26         . "gopkg.in/check.v1"
27 )
28
29 // Gocheck boilerplate
30 func Test(t *testing.T) {
31         TestingT(t)
32 }
33
34 // Gocheck boilerplate
35 var _ = Suite(&ServerRequiredSuite{})
36
37 // Tests that require the Keep server running
38 type ServerRequiredSuite struct{}
39
40 // Gocheck boilerplate
41 var _ = Suite(&NoKeepServerSuite{})
42
43 // Test with no keepserver to simulate errors
44 type NoKeepServerSuite struct{}
45
46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
47
48 // Wait (up to 1 second) for keepproxy to listen on a port. This
49 // avoids a race condition where we hit a "connection refused" error
50 // because we start testing the proxy too soon.
51 func waitForListener() {
52         const (
53                 ms = 5
54         )
55         for i := 0; listener == nil && i < 10000; i += ms {
56                 time.Sleep(ms * time.Millisecond)
57         }
58         if listener == nil {
59                 panic("Timed out waiting for listener to start")
60         }
61 }
62
63 func closeListener() {
64         if listener != nil {
65                 listener.Close()
66         }
67 }
68
69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
70         arvadostest.StartAPI()
71         arvadostest.StartKeep(2, false)
72 }
73
74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
75         arvadostest.ResetEnv()
76 }
77
78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
79         arvadostest.StopKeep(2)
80         arvadostest.StopAPI()
81 }
82
83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
84         arvadostest.StartAPI()
85         // We need API to have some keep services listed, but the
86         // services themselves should be unresponsive.
87         arvadostest.StartKeep(2, false)
88         arvadostest.StopKeep(2)
89 }
90
91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
92         arvadostest.ResetEnv()
93 }
94
95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
96         arvadostest.StopAPI()
97 }
98
99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
100         args = append([]string{"keepproxy"}, args...)
101         os.Args = append(args, "-listen=:0")
102         listener = nil
103         go main()
104         waitForListener()
105
106         arv, err := arvadosclient.MakeArvadosClient()
107         c.Assert(err, Equals, nil)
108         if bogusClientToken {
109                 arv.ApiToken = "bogus-token"
110         }
111         kc := keepclient.New(arv)
112         sr := map[string]string{
113                 TestProxyUUID: "http://" + listener.Addr().String(),
114         }
115         kc.SetServiceRoots(sr, sr, sr)
116         kc.Arvados.External = true
117
118         return kc
119 }
120
121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
122         runProxy(c, nil, false)
123         defer closeListener()
124
125         req, err := http.NewRequest("POST",
126                 "http://"+listener.Addr().String()+"/",
127                 strings.NewReader("TestViaHeader"))
128         c.Assert(err, Equals, nil)
129         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
130         resp, err := (&http.Client{}).Do(req)
131         c.Assert(err, Equals, nil)
132         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
133         locator, err := ioutil.ReadAll(resp.Body)
134         c.Assert(err, Equals, nil)
135         resp.Body.Close()
136
137         req, err = http.NewRequest("GET",
138                 "http://"+listener.Addr().String()+"/"+string(locator),
139                 nil)
140         c.Assert(err, Equals, nil)
141         resp, err = (&http.Client{}).Do(req)
142         c.Assert(err, Equals, nil)
143         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
144         resp.Body.Close()
145 }
146
147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
148         kc := runProxy(c, nil, false)
149         defer closeListener()
150
151         sr := map[string]string{
152                 TestProxyUUID: "http://" + listener.Addr().String(),
153         }
154         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
155
156         content := []byte("TestLoopDetection")
157         _, _, err := kc.PutB(content)
158         c.Check(err, ErrorMatches, `.*loop detected.*`)
159
160         hash := fmt.Sprintf("%x", md5.Sum(content))
161         _, _, _, err = kc.Get(hash)
162         c.Check(err, ErrorMatches, `.*loop detected.*`)
163 }
164
165 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
166         kc := runProxy(c, nil, false)
167         defer closeListener()
168
169         // Set up fake keepstore to record request headers
170         var hdr http.Header
171         ts := httptest.NewServer(http.HandlerFunc(
172                 func(w http.ResponseWriter, r *http.Request) {
173                         hdr = r.Header
174                         http.Error(w, "Error", http.StatusInternalServerError)
175                 }))
176         defer ts.Close()
177
178         // Point keepproxy router's keepclient to the fake keepstore
179         sr := map[string]string{
180                 TestProxyUUID: ts.URL,
181         }
182         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
183
184         // Set up client to ask for storage classes to keepproxy
185         kc.StorageClasses = []string{"secure"}
186         content := []byte("Very important data")
187         _, _, err := kc.PutB(content)
188         c.Check(err, NotNil)
189         c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
190 }
191
192 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
193         kc := runProxy(c, nil, false)
194         defer closeListener()
195
196         content := []byte("TestDesiredReplicas")
197         hash := fmt.Sprintf("%x", md5.Sum(content))
198
199         for _, kc.Want_replicas = range []int{0, 1, 2} {
200                 locator, rep, err := kc.PutB(content)
201                 c.Check(err, Equals, nil)
202                 c.Check(rep, Equals, kc.Want_replicas)
203                 if rep > 0 {
204                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
205                 }
206         }
207 }
208
209 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
210         kc := runProxy(c, nil, false)
211         defer closeListener()
212
213         content := []byte("TestPutWrongContentLength")
214         hash := fmt.Sprintf("%x", md5.Sum(content))
215
216         // If we use http.Client to send these requests to the network
217         // server we just started, the Go http library automatically
218         // fixes the invalid Content-Length header. In order to test
219         // our server behavior, we have to call the handler directly
220         // using an httptest.ResponseRecorder.
221         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
222
223         type testcase struct {
224                 sendLength   string
225                 expectStatus int
226         }
227
228         for _, t := range []testcase{
229                 {"1", http.StatusBadRequest},
230                 {"", http.StatusLengthRequired},
231                 {"-1", http.StatusLengthRequired},
232                 {"abcdef", http.StatusLengthRequired},
233         } {
234                 req, err := http.NewRequest("PUT",
235                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
236                         bytes.NewReader(content))
237                 c.Assert(err, IsNil)
238                 req.Header.Set("Content-Length", t.sendLength)
239                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
240                 req.Header.Set("Content-Type", "application/octet-stream")
241
242                 resp := httptest.NewRecorder()
243                 rtr.ServeHTTP(resp, req)
244                 c.Check(resp.Code, Equals, t.expectStatus)
245         }
246 }
247
248 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
249         kc := runProxy(c, nil, false)
250         defer closeListener()
251         router.(*proxyHandler).timeout = time.Nanosecond
252
253         buf := make([]byte, 1<<20)
254         rand.Read(buf)
255         var wg sync.WaitGroup
256         for i := 0; i < 128; i++ {
257                 wg.Add(1)
258                 go func() {
259                         defer wg.Done()
260                         kc.PutB(buf)
261                 }()
262         }
263         done := make(chan bool)
264         go func() {
265                 wg.Wait()
266                 close(done)
267         }()
268         select {
269         case <-done:
270         case <-time.After(10 * time.Second):
271                 c.Error("timeout")
272         }
273 }
274
275 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
276         kc := runProxy(c, nil, false)
277         defer closeListener()
278
279         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
280         var hash2 string
281
282         {
283                 _, _, err := kc.Ask(hash)
284                 c.Check(err, Equals, keepclient.BlockNotFound)
285                 c.Log("Finished Ask (expected BlockNotFound)")
286         }
287
288         {
289                 reader, _, _, err := kc.Get(hash)
290                 c.Check(reader, Equals, nil)
291                 c.Check(err, Equals, keepclient.BlockNotFound)
292                 c.Log("Finished Get (expected BlockNotFound)")
293         }
294
295         // Note in bug #5309 among other errors keepproxy would set
296         // Content-Length incorrectly on the 404 BlockNotFound response, this
297         // would result in a protocol violation that would prevent reuse of the
298         // connection, which would manifest by the next attempt to use the
299         // connection (in this case the PutB below) failing.  So to test for
300         // that bug it's necessary to trigger an error response (such as
301         // BlockNotFound) and then do something else with the same httpClient
302         // connection.
303
304         {
305                 var rep int
306                 var err error
307                 hash2, rep, err = kc.PutB([]byte("foo"))
308                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
309                 c.Check(rep, Equals, 2)
310                 c.Check(err, Equals, nil)
311                 c.Log("Finished PutB (expected success)")
312         }
313
314         {
315                 blocklen, _, err := kc.Ask(hash2)
316                 c.Assert(err, Equals, nil)
317                 c.Check(blocklen, Equals, int64(3))
318                 c.Log("Finished Ask (expected success)")
319         }
320
321         {
322                 reader, blocklen, _, err := kc.Get(hash2)
323                 c.Assert(err, Equals, nil)
324                 all, err := ioutil.ReadAll(reader)
325                 c.Check(err, IsNil)
326                 c.Check(all, DeepEquals, []byte("foo"))
327                 c.Check(blocklen, Equals, int64(3))
328                 c.Log("Finished Get (expected success)")
329         }
330
331         {
332                 var rep int
333                 var err error
334                 hash2, rep, err = kc.PutB([]byte(""))
335                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
336                 c.Check(rep, Equals, 2)
337                 c.Check(err, Equals, nil)
338                 c.Log("Finished PutB zero block")
339         }
340
341         {
342                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
343                 c.Assert(err, Equals, nil)
344                 all, err := ioutil.ReadAll(reader)
345                 c.Check(err, IsNil)
346                 c.Check(all, DeepEquals, []byte(""))
347                 c.Check(blocklen, Equals, int64(0))
348                 c.Log("Finished Get zero block")
349         }
350 }
351
352 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
353         kc := runProxy(c, nil, true)
354         defer closeListener()
355
356         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
357
358         _, _, err := kc.Ask(hash)
359         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
360
361         hash2, rep, err := kc.PutB([]byte("bar"))
362         c.Check(hash2, Equals, "")
363         c.Check(rep, Equals, 0)
364         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
365
366         blocklen, _, err := kc.Ask(hash)
367         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
368         c.Check(err, ErrorMatches, ".*not found.*")
369         c.Check(blocklen, Equals, int64(0))
370
371         _, blocklen, _, err = kc.Get(hash)
372         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
373         c.Check(err, ErrorMatches, ".*not found.*")
374         c.Check(blocklen, Equals, int64(0))
375
376 }
377
378 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
379         kc := runProxy(c, []string{"-no-get"}, false)
380         defer closeListener()
381
382         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
383
384         {
385                 _, _, err := kc.Ask(hash)
386                 errNotFound, _ := err.(keepclient.ErrNotFound)
387                 c.Check(errNotFound, NotNil)
388                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
389                 c.Log("Ask 1")
390         }
391
392         {
393                 hash2, rep, err := kc.PutB([]byte("baz"))
394                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
395                 c.Check(rep, Equals, 2)
396                 c.Check(err, Equals, nil)
397                 c.Log("PutB")
398         }
399
400         {
401                 blocklen, _, err := kc.Ask(hash)
402                 errNotFound, _ := err.(keepclient.ErrNotFound)
403                 c.Check(errNotFound, NotNil)
404                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
405                 c.Check(blocklen, Equals, int64(0))
406                 c.Log("Ask 2")
407         }
408
409         {
410                 _, blocklen, _, err := kc.Get(hash)
411                 errNotFound, _ := err.(keepclient.ErrNotFound)
412                 c.Check(errNotFound, NotNil)
413                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
414                 c.Check(blocklen, Equals, int64(0))
415                 c.Log("Get")
416         }
417 }
418
419 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
420         kc := runProxy(c, []string{"-no-put"}, false)
421         defer closeListener()
422
423         hash2, rep, err := kc.PutB([]byte("quux"))
424         c.Check(hash2, Equals, "")
425         c.Check(rep, Equals, 0)
426         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
427 }
428
429 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
430         runProxy(c, nil, false)
431         defer closeListener()
432
433         {
434                 client := http.Client{}
435                 req, err := http.NewRequest("OPTIONS",
436                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
437                         nil)
438                 c.Assert(err, IsNil)
439                 req.Header.Add("Access-Control-Request-Method", "PUT")
440                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
441                 resp, err := client.Do(req)
442                 c.Check(err, Equals, nil)
443                 c.Check(resp.StatusCode, Equals, 200)
444                 body, err := ioutil.ReadAll(resp.Body)
445                 c.Check(err, IsNil)
446                 c.Check(string(body), Equals, "")
447                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
448                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
449         }
450
451         {
452                 resp, err := http.Get(
453                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
454                 c.Check(err, Equals, nil)
455                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
456                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
457         }
458 }
459
460 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
461         runProxy(c, nil, false)
462         defer closeListener()
463
464         {
465                 client := http.Client{}
466                 req, err := http.NewRequest("POST",
467                         "http://"+listener.Addr().String()+"/",
468                         strings.NewReader("qux"))
469                 c.Check(err, IsNil)
470                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
471                 req.Header.Add("Content-Type", "application/octet-stream")
472                 resp, err := client.Do(req)
473                 c.Check(err, Equals, nil)
474                 body, err := ioutil.ReadAll(resp.Body)
475                 c.Check(err, Equals, nil)
476                 c.Check(string(body), Matches,
477                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
478         }
479 }
480
481 func (s *ServerRequiredSuite) TestStripHint(c *C) {
482         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
483                 Equals,
484                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
485         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
486                 Equals,
487                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
488         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
489                 Equals,
490                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
491         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
492                 Equals,
493                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
494
495 }
496
497 // Test GetIndex
498 //   Put one block, with 2 replicas
499 //   With no prefix (expect the block locator, twice)
500 //   With an existing prefix (expect the block locator, twice)
501 //   With a valid but non-existing prefix (expect "\n")
502 //   With an invalid prefix (expect error)
503 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
504         kc := runProxy(c, nil, false)
505         defer closeListener()
506
507         // Put "index-data" blocks
508         data := []byte("index-data")
509         hash := fmt.Sprintf("%x", md5.Sum(data))
510
511         hash2, rep, err := kc.PutB(data)
512         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
513         c.Check(rep, Equals, 2)
514         c.Check(err, Equals, nil)
515
516         reader, blocklen, _, err := kc.Get(hash)
517         c.Assert(err, IsNil)
518         c.Check(blocklen, Equals, int64(10))
519         all, err := ioutil.ReadAll(reader)
520         c.Assert(err, IsNil)
521         c.Check(all, DeepEquals, data)
522
523         // Put some more blocks
524         _, _, err = kc.PutB([]byte("some-more-index-data"))
525         c.Check(err, IsNil)
526
527         kc.Arvados.ApiToken = arvadostest.DataManagerToken
528
529         // Invoke GetIndex
530         for _, spec := range []struct {
531                 prefix         string
532                 expectTestHash bool
533                 expectOther    bool
534         }{
535                 {"", true, true},         // with no prefix
536                 {hash[:3], true, false},  // with matching prefix
537                 {"abcdef", false, false}, // with no such prefix
538         } {
539                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
540                 c.Assert(err, Equals, nil)
541                 indexResp, err := ioutil.ReadAll(indexReader)
542                 c.Assert(err, Equals, nil)
543                 locators := strings.Split(string(indexResp), "\n")
544                 gotTestHash := 0
545                 gotOther := 0
546                 for _, locator := range locators {
547                         if locator == "" {
548                                 continue
549                         }
550                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
551                         if locator[:32] == hash {
552                                 gotTestHash++
553                         } else {
554                                 gotOther++
555                         }
556                 }
557                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
558                 c.Check(gotOther > 0, Equals, spec.expectOther)
559         }
560
561         // GetIndex with invalid prefix
562         _, err = kc.GetIndex(TestProxyUUID, "xyz")
563         c.Assert((err != nil), Equals, true)
564 }
565
566 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
567         kc := runProxy(c, nil, false)
568         defer closeListener()
569         hash, _, err := kc.PutB([]byte("shareddata"))
570         c.Check(err, IsNil)
571         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
572         rdr, _, _, err := kc.Get(hash)
573         c.Assert(err, IsNil)
574         data, err := ioutil.ReadAll(rdr)
575         c.Check(err, IsNil)
576         c.Check(data, DeepEquals, []byte("shareddata"))
577 }
578
579 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
580         kc := runProxy(c, nil, false)
581         defer closeListener()
582
583         // Put a test block
584         hash, rep, err := kc.PutB([]byte("foo"))
585         c.Check(err, IsNil)
586         c.Check(rep, Equals, 2)
587
588         for _, badToken := range []string{
589                 "nosuchtoken",
590                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
591         } {
592                 kc.Arvados.ApiToken = badToken
593
594                 // Ask and Get will fail only if the upstream
595                 // keepstore server checks for valid signatures.
596                 // Without knowing the blob signing key, there is no
597                 // way for keepproxy to know whether a given token is
598                 // permitted to read a block.  So these tests fail:
599                 if false {
600                         _, _, err = kc.Ask(hash)
601                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
602                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
603                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
604
605                         _, _, _, err = kc.Get(hash)
606                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
607                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
608                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
609                 }
610
611                 _, _, err = kc.PutB([]byte("foo"))
612                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
613         }
614 }
615
616 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
617         kc := runProxy(c, nil, false)
618         defer closeListener()
619
620         // Point keepproxy at a non-existent keepstore
621         locals := map[string]string{
622                 TestProxyUUID: "http://localhost:12345",
623         }
624         router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
625
626         // Ask should result in temporary bad gateway error
627         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
628         _, _, err := kc.Ask(hash)
629         c.Check(err, NotNil)
630         errNotFound, _ := err.(*keepclient.ErrNotFound)
631         c.Check(errNotFound.Temporary(), Equals, true)
632         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
633
634         // Get should result in temporary bad gateway error
635         _, _, _, err = kc.Get(hash)
636         c.Check(err, NotNil)
637         errNotFound, _ = err.(*keepclient.ErrNotFound)
638         c.Check(errNotFound.Temporary(), Equals, true)
639         c.Assert(err, ErrorMatches, ".*HTTP 502.*")
640 }
641
642 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
643         kc := runProxy(c, nil, false)
644         defer closeListener()
645
646         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
647         for _, f := range []func() error{
648                 func() error {
649                         _, _, err := kc.Ask(hash)
650                         return err
651                 },
652                 func() error {
653                         _, _, _, err := kc.Get(hash)
654                         return err
655                 },
656         } {
657                 err := f()
658                 c.Assert(err, NotNil)
659                 errNotFound, _ := err.(*keepclient.ErrNotFound)
660                 c.Check(errNotFound.Temporary(), Equals, true)
661                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
662         }
663 }
664
665 func (s *ServerRequiredSuite) TestPing(c *C) {
666         kc := runProxy(c, nil, false)
667         defer closeListener()
668
669         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
670
671         req, err := http.NewRequest("GET",
672                 "http://"+listener.Addr().String()+"/_health/ping",
673                 nil)
674         c.Assert(err, IsNil)
675         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
676
677         resp := httptest.NewRecorder()
678         rtr.ServeHTTP(resp, req)
679         c.Check(resp.Code, Equals, 200)
680         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
681 }