Add 'tools/arvbox/' from commit 'd3d368758db1f4a9fa5b89f77b5ee61d68ef5b72'
[arvados.git] / services / keepproxy / keepproxy_test.go
1 package main
2
3 import (
4         "crypto/md5"
5         "fmt"
6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
7         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
8         "git.curoverse.com/arvados.git/sdk/go/keepclient"
9         "io/ioutil"
10         "log"
11         "net/http"
12         "os"
13         "strings"
14         "testing"
15         "time"
16
17         . "gopkg.in/check.v1"
18 )
19
20 // Gocheck boilerplate
21 func Test(t *testing.T) {
22         TestingT(t)
23 }
24
25 // Gocheck boilerplate
26 var _ = Suite(&ServerRequiredSuite{})
27
28 // Tests that require the Keep server running
29 type ServerRequiredSuite struct{}
30
31 // Gocheck boilerplate
32 var _ = Suite(&NoKeepServerSuite{})
33
34 // Test with no keepserver to simulate errors
35 type NoKeepServerSuite struct{}
36
37 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
38
39 // Wait (up to 1 second) for keepproxy to listen on a port. This
40 // avoids a race condition where we hit a "connection refused" error
41 // because we start testing the proxy too soon.
42 func waitForListener() {
43         const (
44                 ms = 5
45         )
46         for i := 0; listener == nil && i < 10000; i += ms {
47                 time.Sleep(ms * time.Millisecond)
48         }
49         if listener == nil {
50                 log.Fatalf("Timed out waiting for listener to start")
51         }
52 }
53
54 func closeListener() {
55         if listener != nil {
56                 listener.Close()
57         }
58 }
59
60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
61         arvadostest.StartAPI()
62         arvadostest.StartKeep(2, false)
63 }
64
65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
66         arvadostest.ResetEnv()
67 }
68
69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
70         arvadostest.StopKeep(2)
71         arvadostest.StopAPI()
72 }
73
74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
75         arvadostest.StartAPI()
76         // We need API to have some keep services listed, but the
77         // services themselves should be unresponsive.
78         arvadostest.StartKeep(2, false)
79         arvadostest.StopKeep(2)
80 }
81
82 func (s *NoKeepServerSuite) SetUpTest(c *C) {
83         arvadostest.ResetEnv()
84 }
85
86 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
87         arvadostest.StopAPI()
88 }
89
90 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
91         args = append([]string{"keepproxy"}, args...)
92         os.Args = append(args, "-listen=:0")
93         listener = nil
94         go main()
95         waitForListener()
96
97         arv, err := arvadosclient.MakeArvadosClient()
98         c.Assert(err, Equals, nil)
99         if bogusClientToken {
100                 arv.ApiToken = "bogus-token"
101         }
102         kc := keepclient.New(&arv)
103         sr := map[string]string{
104                 TestProxyUUID: "http://" + listener.Addr().String(),
105         }
106         kc.SetServiceRoots(sr, sr, sr)
107         kc.Arvados.External = true
108
109         return kc
110 }
111
112 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
113         kc := runProxy(c, nil, false)
114         defer closeListener()
115
116         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
117         var hash2 string
118
119         {
120                 _, _, err := kc.Ask(hash)
121                 c.Check(err, Equals, keepclient.BlockNotFound)
122                 log.Print("Finished Ask (expected BlockNotFound)")
123         }
124
125         {
126                 reader, _, _, err := kc.Get(hash)
127                 c.Check(reader, Equals, nil)
128                 c.Check(err, Equals, keepclient.BlockNotFound)
129                 log.Print("Finished Get (expected BlockNotFound)")
130         }
131
132         // Note in bug #5309 among other errors keepproxy would set
133         // Content-Length incorrectly on the 404 BlockNotFound response, this
134         // would result in a protocol violation that would prevent reuse of the
135         // connection, which would manifest by the next attempt to use the
136         // connection (in this case the PutB below) failing.  So to test for
137         // that bug it's necessary to trigger an error response (such as
138         // BlockNotFound) and then do something else with the same httpClient
139         // connection.
140
141         {
142                 var rep int
143                 var err error
144                 hash2, rep, err = kc.PutB([]byte("foo"))
145                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
146                 c.Check(rep, Equals, 2)
147                 c.Check(err, Equals, nil)
148                 log.Print("Finished PutB (expected success)")
149         }
150
151         {
152                 blocklen, _, err := kc.Ask(hash2)
153                 c.Assert(err, Equals, nil)
154                 c.Check(blocklen, Equals, int64(3))
155                 log.Print("Finished Ask (expected success)")
156         }
157
158         {
159                 reader, blocklen, _, err := kc.Get(hash2)
160                 c.Assert(err, Equals, nil)
161                 all, err := ioutil.ReadAll(reader)
162                 c.Check(all, DeepEquals, []byte("foo"))
163                 c.Check(blocklen, Equals, int64(3))
164                 log.Print("Finished Get (expected success)")
165         }
166
167         {
168                 var rep int
169                 var err error
170                 hash2, rep, err = kc.PutB([]byte(""))
171                 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
172                 c.Check(rep, Equals, 2)
173                 c.Check(err, Equals, nil)
174                 log.Print("Finished PutB zero block")
175         }
176
177         {
178                 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
179                 c.Assert(err, Equals, nil)
180                 all, err := ioutil.ReadAll(reader)
181                 c.Check(all, DeepEquals, []byte(""))
182                 c.Check(blocklen, Equals, int64(0))
183                 log.Print("Finished Get zero block")
184         }
185 }
186
187 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
188         kc := runProxy(c, nil, true)
189         defer closeListener()
190
191         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
192
193         {
194                 _, _, err := kc.Ask(hash)
195                 errNotFound, _ := err.(keepclient.ErrNotFound)
196                 c.Check(errNotFound, NotNil)
197                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
198                 log.Print("Ask 1")
199         }
200
201         {
202                 hash2, rep, err := kc.PutB([]byte("bar"))
203                 c.Check(hash2, Equals, "")
204                 c.Check(rep, Equals, 0)
205                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
206                 log.Print("PutB")
207         }
208
209         {
210                 blocklen, _, err := kc.Ask(hash)
211                 errNotFound, _ := err.(keepclient.ErrNotFound)
212                 c.Check(errNotFound, NotNil)
213                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
214                 c.Check(blocklen, Equals, int64(0))
215                 log.Print("Ask 2")
216         }
217
218         {
219                 _, blocklen, _, err := kc.Get(hash)
220                 errNotFound, _ := err.(keepclient.ErrNotFound)
221                 c.Check(errNotFound, NotNil)
222                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
223                 c.Check(blocklen, Equals, int64(0))
224                 log.Print("Get")
225         }
226 }
227
228 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
229         kc := runProxy(c, []string{"-no-get"}, false)
230         defer closeListener()
231
232         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
233
234         {
235                 _, _, err := kc.Ask(hash)
236                 errNotFound, _ := err.(keepclient.ErrNotFound)
237                 c.Check(errNotFound, NotNil)
238                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
239                 log.Print("Ask 1")
240         }
241
242         {
243                 hash2, rep, err := kc.PutB([]byte("baz"))
244                 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
245                 c.Check(rep, Equals, 2)
246                 c.Check(err, Equals, nil)
247                 log.Print("PutB")
248         }
249
250         {
251                 blocklen, _, err := kc.Ask(hash)
252                 errNotFound, _ := err.(keepclient.ErrNotFound)
253                 c.Check(errNotFound, NotNil)
254                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
255                 c.Check(blocklen, Equals, int64(0))
256                 log.Print("Ask 2")
257         }
258
259         {
260                 _, blocklen, _, err := kc.Get(hash)
261                 errNotFound, _ := err.(keepclient.ErrNotFound)
262                 c.Check(errNotFound, NotNil)
263                 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
264                 c.Check(blocklen, Equals, int64(0))
265                 log.Print("Get")
266         }
267 }
268
269 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
270         kc := runProxy(c, []string{"-no-put"}, false)
271         defer closeListener()
272
273         hash2, rep, err := kc.PutB([]byte("quux"))
274         c.Check(hash2, Equals, "")
275         c.Check(rep, Equals, 0)
276         c.Check(err, Equals, keepclient.InsufficientReplicasError)
277 }
278
279 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
280         runProxy(c, nil, false)
281         defer closeListener()
282
283         {
284                 client := http.Client{}
285                 req, err := http.NewRequest("OPTIONS",
286                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
287                         nil)
288                 req.Header.Add("Access-Control-Request-Method", "PUT")
289                 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
290                 resp, err := client.Do(req)
291                 c.Check(err, Equals, nil)
292                 c.Check(resp.StatusCode, Equals, 200)
293                 body, err := ioutil.ReadAll(resp.Body)
294                 c.Check(string(body), Equals, "")
295                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
296                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
297         }
298
299         {
300                 resp, err := http.Get(
301                         fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
302                 c.Check(err, Equals, nil)
303                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
304                 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
305         }
306 }
307
308 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
309         runProxy(c, nil, false)
310         defer closeListener()
311
312         {
313                 client := http.Client{}
314                 req, err := http.NewRequest("POST",
315                         "http://"+listener.Addr().String()+"/",
316                         strings.NewReader("qux"))
317                 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
318                 req.Header.Add("Content-Type", "application/octet-stream")
319                 resp, err := client.Do(req)
320                 c.Check(err, Equals, nil)
321                 body, err := ioutil.ReadAll(resp.Body)
322                 c.Check(err, Equals, nil)
323                 c.Check(string(body), Matches,
324                         fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
325         }
326 }
327
328 func (s *ServerRequiredSuite) TestStripHint(c *C) {
329         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
330                 Equals,
331                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
332         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
333                 Equals,
334                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
335         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
336                 Equals,
337                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
338         c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
339                 Equals,
340                 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
341
342 }
343
344 // Test GetIndex
345 //   Put one block, with 2 replicas
346 //   With no prefix (expect the block locator, twice)
347 //   With an existing prefix (expect the block locator, twice)
348 //   With a valid but non-existing prefix (expect "\n")
349 //   With an invalid prefix (expect error)
350 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
351         kc := runProxy(c, nil, false)
352         defer closeListener()
353
354         // Put "index-data" blocks
355         data := []byte("index-data")
356         hash := fmt.Sprintf("%x", md5.Sum(data))
357
358         hash2, rep, err := kc.PutB(data)
359         c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
360         c.Check(rep, Equals, 2)
361         c.Check(err, Equals, nil)
362
363         reader, blocklen, _, err := kc.Get(hash)
364         c.Assert(err, Equals, nil)
365         c.Check(blocklen, Equals, int64(10))
366         all, err := ioutil.ReadAll(reader)
367         c.Check(all, DeepEquals, data)
368
369         // Put some more blocks
370         _, rep, err = kc.PutB([]byte("some-more-index-data"))
371         c.Check(err, Equals, nil)
372
373         kc.Arvados.ApiToken = arvadostest.DataManagerToken
374
375         // Invoke GetIndex
376         for _, spec := range []struct {
377                 prefix         string
378                 expectTestHash bool
379                 expectOther    bool
380         }{
381                 {"", true, true},         // with no prefix
382                 {hash[:3], true, false},  // with matching prefix
383                 {"abcdef", false, false}, // with no such prefix
384         } {
385                 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
386                 c.Assert(err, Equals, nil)
387                 indexResp, err := ioutil.ReadAll(indexReader)
388                 c.Assert(err, Equals, nil)
389                 locators := strings.Split(string(indexResp), "\n")
390                 gotTestHash := 0
391                 gotOther := 0
392                 for _, locator := range locators {
393                         if locator == "" {
394                                 continue
395                         }
396                         c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
397                         if locator[:32] == hash {
398                                 gotTestHash++
399                         } else {
400                                 gotOther++
401                         }
402                 }
403                 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
404                 c.Check(gotOther > 0, Equals, spec.expectOther)
405         }
406
407         // GetIndex with invalid prefix
408         _, err = kc.GetIndex(TestProxyUUID, "xyz")
409         c.Assert((err != nil), Equals, true)
410 }
411
412 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
413         kc := runProxy(c, nil, false)
414         defer closeListener()
415
416         // Put a test block
417         hash, rep, err := kc.PutB([]byte("foo"))
418         c.Check(err, Equals, nil)
419         c.Check(rep, Equals, 2)
420
421         for _, token := range []string{
422                 "nosuchtoken",
423                 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
424         } {
425                 // Change token to given bad token
426                 kc.Arvados.ApiToken = token
427
428                 // Ask should result in error
429                 _, _, err = kc.Ask(hash)
430                 c.Check(err, NotNil)
431                 errNotFound, _ := err.(keepclient.ErrNotFound)
432                 c.Check(errNotFound.Temporary(), Equals, false)
433                 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
434
435                 // Get should result in error
436                 _, _, _, err = kc.Get(hash)
437                 c.Check(err, NotNil)
438                 errNotFound, _ = err.(keepclient.ErrNotFound)
439                 c.Check(errNotFound.Temporary(), Equals, false)
440                 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
441         }
442 }
443
444 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
445         arv, err := arvadosclient.MakeArvadosClient()
446         c.Assert(err, Equals, nil)
447
448         // keepclient with no such keep server
449         kc := keepclient.New(&arv)
450         locals := map[string]string{
451                 TestProxyUUID: "http://localhost:12345",
452         }
453         kc.SetServiceRoots(locals, nil, nil)
454
455         // Ask should result in temporary connection refused error
456         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
457         _, _, err = kc.Ask(hash)
458         c.Check(err, NotNil)
459         errNotFound, _ := err.(*keepclient.ErrNotFound)
460         c.Check(errNotFound.Temporary(), Equals, true)
461         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
462
463         // Get should result in temporary connection refused error
464         _, _, _, err = kc.Get(hash)
465         c.Check(err, NotNil)
466         errNotFound, _ = err.(*keepclient.ErrNotFound)
467         c.Check(errNotFound.Temporary(), Equals, true)
468         c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
469 }
470
471 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
472         kc := runProxy(c, nil, false)
473         defer closeListener()
474
475         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
476         for _, f := range []func() error{
477                 func() error {
478                         _, _, err := kc.Ask(hash)
479                         return err
480                 },
481                 func() error {
482                         _, _, _, err := kc.Get(hash)
483                         return err
484                 },
485         } {
486                 err := f()
487                 c.Assert(err, NotNil)
488                 errNotFound, _ := err.(*keepclient.ErrNotFound)
489                 c.Check(errNotFound.Temporary(), Equals, true)
490                 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
491         }
492 }