Merge branch 'master' of git.curoverse.com:arvados into 11876-r-sdk
[arvados.git] / services / keepproxy / keepproxy_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "errors"
11         "fmt"
12         "io/ioutil"
13         "math/rand"
14         "net/http"
15         "net/http/httptest"
16         "os"
17         "strings"
18         "sync"
19         "testing"
20         "time"
21
22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
25
26         . "gopkg.in/check.v1"
27 )
28
29 // Gocheck boilerplate
30 func Test(t *testing.T) {
31         TestingT(t)
32 }
33
34 // Gocheck boilerplate
35 var _ = Suite(&ServerRequiredSuite{})
36
37 // Tests that require the Keep server running
38 type ServerRequiredSuite struct{}
39
40 // Gocheck boilerplate
41 var _ = Suite(&NoKeepServerSuite{})
42
43 // Test with no keepserver to simulate errors
44 type NoKeepServerSuite struct{}
45
46 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
47
48 // Wait (up to 1 second) for keepproxy to listen on a port. This
49 // avoids a race condition where we hit a "connection refused" error
50 // because we start testing the proxy too soon.
51 func waitForListener() {
52         const (
53                 ms = 5
54         )
55         for i := 0; listener == nil && i < 10000; i += ms {
56                 time.Sleep(ms * time.Millisecond)
57         }
58         if listener == nil {
59                 panic("Timed out waiting for listener to start")
60         }
61 }
62
63 func closeListener() {
64         if listener != nil {
65                 listener.Close()
66         }
67 }
68
69 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
70         arvadostest.StartAPI()
71         arvadostest.StartKeep(2, false)
72 }
73
74 func (s *ServerRequiredSuite) SetUpTest(c *C) {
75         arvadostest.ResetEnv()
76 }
77
78 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
79         arvadostest.StopKeep(2)
80         arvadostest.StopAPI()
81 }
82
83 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
84         arvadostest.StartAPI()
85         // We need API to have some keep services listed, but the
86         // services themselves should be unresponsive.
87         arvadostest.StartKeep(2, false)
88         arvadostest.StopKeep(2)
89 }
90
91 func (s *NoKeepServerSuite) SetUpTest(c *C) {
92         arvadostest.ResetEnv()
93 }
94
95 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
96         arvadostest.StopAPI()
97 }
98
99 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
100         args = append([]string{"keepproxy"}, args...)
101         os.Args = append(args, "-listen=:0")
102         listener = nil
103         go main()
104         waitForListener()
105
106         arv, err := arvadosclient.MakeArvadosClient()
107         c.Assert(err, Equals, nil)
108         if bogusClientToken {
109                 arv.ApiToken = "bogus-token"
110         }
111         kc := keepclient.New(arv)
112         sr := map[string]string{
113                 TestProxyUUID: "http://" + listener.Addr().String(),
114         }
115         kc.SetServiceRoots(sr, sr, sr)
116         kc.Arvados.External = true
117
118         return kc
119 }
120
121 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
122         runProxy(c, nil, false)
123         defer closeListener()
124
125         req, err := http.NewRequest("POST",
126                 "http://"+listener.Addr().String()+"/",
127                 strings.NewReader("TestViaHeader"))
128         req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
129         resp, err := (&http.Client{}).Do(req)
130         c.Assert(err, Equals, nil)
131         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
132         locator, err := ioutil.ReadAll(resp.Body)
133         c.Assert(err, Equals, nil)
134         resp.Body.Close()
135
136         req, err = http.NewRequest("GET",
137                 "http://"+listener.Addr().String()+"/"+string(locator),
138                 nil)
139         c.Assert(err, Equals, nil)
140         resp, err = (&http.Client{}).Do(req)
141         c.Assert(err, Equals, nil)
142         c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
143         resp.Body.Close()
144 }
145
146 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
147         kc := runProxy(c, nil, false)
148         defer closeListener()
149
150         sr := map[string]string{
151                 TestProxyUUID: "http://" + listener.Addr().String(),
152         }
153         router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
154
155         content := []byte("TestLoopDetection")
156         _, _, err := kc.PutB(content)
157         c.Check(err, ErrorMatches, `.*loop detected.*`)
158
159         hash := fmt.Sprintf("%x", md5.Sum(content))
160         _, _, _, err = kc.Get(hash)
161         c.Check(err, ErrorMatches, `.*loop detected.*`)
162 }
163
164 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
165         kc := runProxy(c, nil, false)
166         defer closeListener()
167
168         content := []byte("TestDesiredReplicas")
169         hash := fmt.Sprintf("%x", md5.Sum(content))
170
171         for _, kc.Want_replicas = range []int{0, 1, 2} {
172                 locator, rep, err := kc.PutB(content)
173                 c.Check(err, Equals, nil)
174                 c.Check(rep, Equals, kc.Want_replicas)
175                 if rep > 0 {
176                         c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
177                 }
178         }
179 }
180
181 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
182         kc := runProxy(c, nil, false)
183         defer closeListener()
184
185         content := []byte("TestPutWrongContentLength")
186         hash := fmt.Sprintf("%x", md5.Sum(content))
187
188         // If we use http.Client to send these requests to the network
189         // server we just started, the Go http library automatically
190         // fixes the invalid Content-Length header. In order to test
191         // our server behavior, we have to call the handler directly
192         // using an httptest.ResponseRecorder.
193         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
194
195         type testcase struct {
196                 sendLength   string
197                 expectStatus int
198         }
199
200         for _, t := range []testcase{
201                 {"1", http.StatusBadRequest},
202                 {"", http.StatusLengthRequired},
203                 {"-1", http.StatusLengthRequired},
204                 {"abcdef", http.StatusLengthRequired},
205         } {
206                 req, err := http.NewRequest("PUT",
207                         fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
208                         bytes.NewReader(content))
209                 c.Assert(err, IsNil)
210                 req.Header.Set("Content-Length", t.sendLength)
211                 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
212                 req.Header.Set("Content-Type", "application/octet-stream")
213
214                 resp := httptest.NewRecorder()
215                 rtr.ServeHTTP(resp, req)
216                 c.Check(resp.Code, Equals, t.expectStatus)
217         }
218 }
219
220 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
221         kc := runProxy(c, nil, false)
222         defer closeListener()
223         router.(*proxyHandler).timeout = time.Nanosecond
224
225         buf := make([]byte, 1<<20)
226         rand.Read(buf)
227         var wg sync.WaitGroup
228         for i := 0; i < 128; i++ {
229                 wg.Add(1)
230                 go func() {
231                         defer wg.Done()
232                         kc.PutB(buf)
233                 }()
234         }
235         done := make(chan bool)
236         go func() {
237                 wg.Wait()
238                 close(done)
239         }()
240         select {
241         case <-done:
242         case <-time.After(10 * time.Second):
243                 c.Error("timeout")
244         }
245 }
246
247 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
248         kc := runProxy(c, nil, false)
249         defer closeListener()
250
251         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
252         var hash2 string
253
254         {
255                 _, _, err := kc.Ask(hash)
256                 c.Check(err, Equals, keepclient.BlockNotFound)
257                 c.Log("Finished Ask (expected BlockNotFound)")
258         }
259
260         {
261                 reader, _, _, err := kc.Get(hash)
262                 c.Check(reader, Equals, nil)
263                 c.Check(err, Equals, keepclient.BlockNotFound)
264                 c.Log("Finished Get (expected BlockNotFound)")
265         }
266
267         // Note in bug #5309 among other errors keepproxy would set
268         // Content-Length incorrectly on the 404 BlockNotFound response, this
269         // would result in a protocol violation that would prevent reuse of the
270         // connection, which would manifest by the next attempt to use the
271         // connection (in this case the PutB below) failing.  So to test for
272         // that bug it's necessary to trigger an error response (such as
273         // BlockNotFound) and then do something else with the same httpClient
274         // connection.
275
276         {
277                 var rep int
278                 var err error
279                 hash2, rep, err = kc.PutB([]byte("foo"))
280                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
281                 c.Check(rep, Equals, 2)
282                 c.Check(err, Equals, nil)
283                 c.Log("Finished PutB (expected success)")
284         }
285
286         {
287                 blocklen, _, err := kc.Ask(hash2)
288                 c.Assert(err, Equals, nil)
289                 c.Check(blocklen, Equals, int64(3))
290                 c.Log("Finished Ask (expected success)")
291         }
292
293         {
294                 reader, blocklen, _, err := kc.Get(hash2)
295                 c.Assert(err, Equals, nil)
296                 all, err := ioutil.ReadAll(reader)
297                 c.Check(all, DeepEquals, []byte("foo"))
298                 c.Check(blocklen, Equals, int64(3))
299                 c.Log("Finished Get (expected success)")
300         }
301
302         {
303                 var rep int
304                 var err error
305                 hash2, rep, err = kc.PutB([]byte(""))
306                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
307                 c.Check(rep, Equals, 2)
308                 c.Check(err, Equals, nil)
309                 c.Log("Finished PutB zero block")
310         }
311
312         {
313                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
314                 c.Assert(err, Equals, nil)
315                 all, err := ioutil.ReadAll(reader)
316                 c.Check(all, DeepEquals, []byte(""))
317                 c.Check(blocklen, Equals, int64(0))
318                 c.Log("Finished Get zero block")
319         }
320 }
321
322 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
323         kc := runProxy(c, nil, true)
324         defer closeListener()
325
326         hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
327
328         _, _, err := kc.Ask(hash)
329         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
330
331         hash2, rep, err := kc.PutB([]byte("bar"))
332         c.Check(hash2, Equals, "")
333         c.Check(rep, Equals, 0)
334         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
335
336         blocklen, _, err := kc.Ask(hash)
337         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
338         c.Check(err, ErrorMatches, ".*not found.*")
339         c.Check(blocklen, Equals, int64(0))
340
341         _, blocklen, _, err = kc.Get(hash)
342         c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
343         c.Check(err, ErrorMatches, ".*not found.*")
344         c.Check(blocklen, Equals, int64(0))
345
346 }
347
348 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
349         kc := runProxy(c, []string{"-no-get"}, false)
350         defer closeListener()
351
352         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
353
354         {
355                 _, _, err := kc.Ask(hash)
356                 errNotFound, _ := err.(keepclient.ErrNotFound)
357                 c.Check(errNotFound, NotNil)
358                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
359                 c.Log("Ask 1")
360         }
361
362         {
363                 hash2, rep, err := kc.PutB([]byte("baz"))
364                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
365                 c.Check(rep, Equals, 2)
366                 c.Check(err, Equals, nil)
367                 c.Log("PutB")
368         }
369
370         {
371                 blocklen, _, err := kc.Ask(hash)
372                 errNotFound, _ := err.(keepclient.ErrNotFound)
373                 c.Check(errNotFound, NotNil)
374                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
375                 c.Check(blocklen, Equals, int64(0))
376                 c.Log("Ask 2")
377         }
378
379         {
380                 _, blocklen, _, err := kc.Get(hash)
381                 errNotFound, _ := err.(keepclient.ErrNotFound)
382                 c.Check(errNotFound, NotNil)
383                 c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
384                 c.Check(blocklen, Equals, int64(0))
385                 c.Log("Get")
386         }
387 }
388
389 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
390         kc := runProxy(c, []string{"-no-put"}, false)
391         defer closeListener()
392
393         hash2, rep, err := kc.PutB([]byte("quux"))
394         c.Check(hash2, Equals, "")
395         c.Check(rep, Equals, 0)
396         c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
397 }
398
399 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
400         runProxy(c, nil, false)
401         defer closeListener()
402
403         {
404                 client := http.Client{}
405                 req, err := http.NewRequest("OPTIONS",
406                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
407                         nil)
408                 req.Header.Add("Access-Control-Request-Method", "PUT")
409                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
410                 resp, err := client.Do(req)
411                 c.Check(err, Equals, nil)
412                 c.Check(resp.StatusCode, Equals, 200)
413                 body, err := ioutil.ReadAll(resp.Body)
414                 c.Check(string(body), Equals, "")
415                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
416                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
417         }
418
419         {
420                 resp, err := http.Get(
421                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
422                 c.Check(err, Equals, nil)
423                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
424                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
425         }
426 }
427
428 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
429         runProxy(c, nil, false)
430         defer closeListener()
431
432         {
433                 client := http.Client{}
434                 req, err := http.NewRequest("POST",
435                         "http://"+listener.Addr().String()+"/",
436                         strings.NewReader("qux"))
437                 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
438                 req.Header.Add("Content-Type", "application/octet-stream")
439                 resp, err := client.Do(req)
440                 c.Check(err, Equals, nil)
441                 body, err := ioutil.ReadAll(resp.Body)
442                 c.Check(err, Equals, nil)
443                 c.Check(string(body), Matches,
444                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
445         }
446 }
447
448 func (s *ServerRequiredSuite) TestStripHint(c *C) {
449         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
450                 Equals,
451                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
452         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
453                 Equals,
454                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
455         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
456                 Equals,
457                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
458         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
459                 Equals,
460                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
461
462 }
463
464 // Test GetIndex
465 //   Put one block, with 2 replicas
466 //   With no prefix (expect the block locator, twice)
467 //   With an existing prefix (expect the block locator, twice)
468 //   With a valid but non-existing prefix (expect "\n")
469 //   With an invalid prefix (expect error)
470 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
471         kc := runProxy(c, nil, false)
472         defer closeListener()
473
474         // Put "index-data" blocks
475         data := []byte("index-data")
476         hash := fmt.Sprintf("%x", md5.Sum(data))
477
478         hash2, rep, err := kc.PutB(data)
479         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
480         c.Check(rep, Equals, 2)
481         c.Check(err, Equals, nil)
482
483         reader, blocklen, _, err := kc.Get(hash)
484         c.Assert(err, Equals, nil)
485         c.Check(blocklen, Equals, int64(10))
486         all, err := ioutil.ReadAll(reader)
487         c.Check(all, DeepEquals, data)
488
489         // Put some more blocks
490         _, rep, err = kc.PutB([]byte("some-more-index-data"))
491         c.Check(err, Equals, nil)
492
493         kc.Arvados.ApiToken = arvadostest.DataManagerToken
494
495         // Invoke GetIndex
496         for _, spec := range []struct {
497                 prefix         string
498                 expectTestHash bool
499                 expectOther    bool
500         }{
501                 {"", true, true},         // with no prefix
502                 {hash[:3], true, false},  // with matching prefix
503                 {"abcdef", false, false}, // with no such prefix
504         } {
505                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
506                 c.Assert(err, Equals, nil)
507                 indexResp, err := ioutil.ReadAll(indexReader)
508                 c.Assert(err, Equals, nil)
509                 locators := strings.Split(string(indexResp), "\n")
510                 gotTestHash := 0
511                 gotOther := 0
512                 for _, locator := range locators {
513                         if locator == "" {
514                                 continue
515                         }
516                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
517                         if locator[:32] == hash {
518                                 gotTestHash++
519                         } else {
520                                 gotOther++
521                         }
522                 }
523                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
524                 c.Check(gotOther > 0, Equals, spec.expectOther)
525         }
526
527         // GetIndex with invalid prefix
528         _, err = kc.GetIndex(TestProxyUUID, "xyz")
529         c.Assert((err != nil), Equals, true)
530 }
531
532 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
533         kc := runProxy(c, nil, false)
534         defer closeListener()
535         hash, _, err := kc.PutB([]byte("shareddata"))
536         c.Check(err, IsNil)
537         kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
538         rdr, _, _, err := kc.Get(hash)
539         c.Assert(err, IsNil)
540         data, err := ioutil.ReadAll(rdr)
541         c.Check(err, IsNil)
542         c.Check(data, DeepEquals, []byte("shareddata"))
543 }
544
545 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
546         kc := runProxy(c, nil, false)
547         defer closeListener()
548
549         // Put a test block
550         hash, rep, err := kc.PutB([]byte("foo"))
551         c.Check(err, IsNil)
552         c.Check(rep, Equals, 2)
553
554         for _, badToken := range []string{
555                 "nosuchtoken",
556                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
557         } {
558                 kc.Arvados.ApiToken = badToken
559
560                 // Ask and Get will fail only if the upstream
561                 // keepstore server checks for valid signatures.
562                 // Without knowing the blob signing key, there is no
563                 // way for keepproxy to know whether a given token is
564                 // permitted to read a block.  So these tests fail:
565                 if false {
566                         _, _, err = kc.Ask(hash)
567                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
568                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
569                         c.Check(err, ErrorMatches, ".*HTTP 403.*")
570
571                         _, _, _, err = kc.Get(hash)
572                         c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
573                         c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
574                         c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
575                 }
576
577                 _, _, err = kc.PutB([]byte("foo"))
578                 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
579         }
580 }
581
582 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
583         arv, err := arvadosclient.MakeArvadosClient()
584         c.Assert(err, Equals, nil)
585
586         // keepclient with no such keep server
587         kc := keepclient.New(arv)
588         locals := map[string]string{
589                 TestProxyUUID: "http://localhost:12345",
590         }
591         kc.SetServiceRoots(locals, nil, nil)
592
593         // Ask should result in temporary connection refused error
594         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
595         _, _, err = kc.Ask(hash)
596         c.Check(err, NotNil)
597         errNotFound, _ := err.(*keepclient.ErrNotFound)
598         c.Check(errNotFound.Temporary(), Equals, true)
599         c.Assert(err, ErrorMatches, ".*connection refused.*")
600
601         // Get should result in temporary connection refused error
602         _, _, _, err = kc.Get(hash)
603         c.Check(err, NotNil)
604         errNotFound, _ = err.(*keepclient.ErrNotFound)
605         c.Check(errNotFound.Temporary(), Equals, true)
606         c.Assert(err, ErrorMatches, ".*connection refused.*")
607 }
608
609 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
610         kc := runProxy(c, nil, false)
611         defer closeListener()
612
613         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
614         for _, f := range []func() error{
615                 func() error {
616                         _, _, err := kc.Ask(hash)
617                         return err
618                 },
619                 func() error {
620                         _, _, _, err := kc.Get(hash)
621                         return err
622                 },
623         } {
624                 err := f()
625                 c.Assert(err, NotNil)
626                 errNotFound, _ := err.(*keepclient.ErrNotFound)
627                 c.Check(errNotFound.Temporary(), Equals, true)
628                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
629         }
630 }
631
632 func (s *ServerRequiredSuite) TestPing(c *C) {
633         kc := runProxy(c, nil, false)
634         defer closeListener()
635
636         rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
637
638         req, err := http.NewRequest("GET",
639                 "http://"+listener.Addr().String()+"/_health/ping",
640                 nil)
641         c.Assert(err, IsNil)
642         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
643
644         resp := httptest.NewRecorder()
645         rtr.ServeHTTP(resp, req)
646         c.Check(resp.Code, Equals, 200)
647         c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
648 }