// Copyright (C) The Arvados Authors. All rights reserved. // // SPDX-License-Identifier: Apache-2.0 package keepclient import ( "bytes" "context" "crypto/md5" "fmt" "io" "io/ioutil" "log" "net" "net/http" "os" "strings" "sync" "sync/atomic" "testing" "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" . "gopkg.in/check.v1" ) func Test(t *testing.T) { DefaultRetryDelay = 50 * time.Millisecond TestingT(t) } // Gocheck boilerplate var _ = Suite(&ServerRequiredSuite{}) var _ = Suite(&StandaloneSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} // Standalone tests type StandaloneSuite struct { origDefaultRetryDelay time.Duration origMinimumRetryDelay time.Duration } var origHOME = os.Getenv("HOME") func (s *StandaloneSuite) SetUpTest(c *C) { RefreshServiceDiscovery() // Prevent cache state from leaking between test cases os.Setenv("HOME", c.MkDir()) s.origDefaultRetryDelay = DefaultRetryDelay s.origMinimumRetryDelay = MinimumRetryDelay } func (s *StandaloneSuite) TearDownTest(c *C) { os.Setenv("HOME", origHOME) DefaultRetryDelay = s.origDefaultRetryDelay MinimumRetryDelay = s.origMinimumRetryDelay } func pythonDir() string { cwd, _ := os.Getwd() return fmt.Sprintf("%s/../../python/tests", cwd) } func (s *ServerRequiredSuite) SetUpSuite(c *C) { arvadostest.StartKeep(2, false) } func (s *ServerRequiredSuite) TearDownSuite(c *C) { arvadostest.StopKeep(2) os.Setenv("HOME", origHOME) } func (s *ServerRequiredSuite) SetUpTest(c *C) { RefreshServiceDiscovery() // Prevent cache state from leaking between test cases os.Setenv("HOME", c.MkDir()) } func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) kc, err := MakeKeepClient(arv) c.Assert(err, IsNil) c.Check(len(kc.LocalRoots()), Equals, 2) for _, root := range kc.LocalRoots() { c.Check(root, Matches, "http://localhost:\\d+") } } func (s *ServerRequiredSuite) TestDefaultStorageClasses(c *C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) cc, err := arv.ClusterConfig("StorageClasses") c.Assert(err, IsNil) c.Assert(cc, NotNil) c.Assert(cc.(map[string]interface{})["default"], NotNil) kc := New(arv) c.Assert(kc.DefaultStorageClasses, DeepEquals, []string{"default"}) } func (s *ServerRequiredSuite) TestDefaultReplications(c *C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) kc, err := MakeKeepClient(arv) c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 2) arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0 kc, err = MakeKeepClient(arv) c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 3) arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0 kc, err = MakeKeepClient(arv) c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 1) } type StubPutHandler struct { c *C expectPath string expectAPIToken string expectBody string expectStorageClass string returnStorageClasses string handled chan string requests []*http.Request mtx sync.Mutex } func (sph *StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { sph.mtx.Lock() sph.requests = append(sph.requests, req) sph.mtx.Unlock() sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath) sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken)) if sph.expectStorageClass != "*" { sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass) } body, err := ioutil.ReadAll(req.Body) sph.c.Check(err, IsNil) sph.c.Check(body, DeepEquals, []byte(sph.expectBody)) resp.Header().Set("X-Keep-Replicas-Stored", "1") if sph.returnStorageClasses != "" { resp.Header().Set("X-Keep-Storage-Classes-Confirmed", sph.returnStorageClasses) } resp.WriteHeader(200) sph.handled <- fmt.Sprintf("http://%s", req.Host) } func RunFakeKeepServer(st http.Handler) (ks KeepServer) { var err error // If we don't explicitly bind it to localhost, ks.listener.Addr() will // bind to 0.0.0.0 or [::] which is not a valid address for Dial() ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0}) if err != nil { panic("Could not listen on any port") } ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String()) go http.Serve(ks.listener, st) return } func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string, io.ReadCloser, io.WriteCloser, chan uploadStatus)) { ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, _ := arvadosclient.MakeArvadosClient() arv.ApiToken = "abc123" kc, _ := MakeKeepClient(arv) reader, writer := io.Pipe() uploadStatusChan := make(chan uploadStatus) f(kc, ks.url, reader, writer, uploadStatusChan) } func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { log.Printf("TestUploadToStubKeepServer") st := &StubPutHandler{ c: c, expectPath: "acbd18db4cc2f85cedef654fccc4a4d8", expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "", returnStorageClasses: "default=1", handled: make(chan string), } UploadToStubHelper(c, st, func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) { go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""}) }) } func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { st := &StubPutHandler{ c: c, expectPath: "acbd18db4cc2f85cedef654fccc4a4d8", expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "", returnStorageClasses: "default=1", handled: make(chan string), } UploadToStubHelper(c, st, func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) { go kc.uploadToKeepServer(url, st.expectPath, nil, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID()) <-st.handled status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""}) }) } func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) { for _, trial := range []struct { respHeader string expectMap map[string]int }{ {"", nil}, {"foo=1", map[string]int{"foo": 1}}, {" foo=1 , bar=2 ", map[string]int{"foo": 1, "bar": 2}}, {" =foo=1 ", nil}, {"foo", nil}, } { st := &StubPutHandler{ c: c, expectPath: "acbd18db4cc2f85cedef654fccc4a4d8", expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "", returnStorageClasses: trial.respHeader, handled: make(chan string), } UploadToStubHelper(c, st, func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) { go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, trial.expectMap, ""}) }) } } func (s *StandaloneSuite) TestPutWithoutStorageClassesClusterSupport(c *C) { nServers := 5 for _, trial := range []struct { replicas int clientClasses []string putClasses []string minRequests int maxRequests int success bool }{ // Talking to an older cluster (no default storage classes exported // config) and no other additional storage classes requirements. {1, nil, nil, 1, 1, true}, {2, nil, nil, 2, 2, true}, {3, nil, nil, 3, 3, true}, {nServers*2 + 1, nil, nil, nServers, nServers, false}, {1, []string{"class1"}, nil, 1, 1, true}, {2, []string{"class1"}, nil, 2, 2, true}, {3, []string{"class1"}, nil, 3, 3, true}, {1, []string{"class1", "class2"}, nil, 1, 1, true}, {nServers*2 + 1, []string{"class1"}, nil, nServers, nServers, false}, {1, nil, []string{"class1"}, 1, 1, true}, {2, nil, []string{"class1"}, 2, 2, true}, {3, nil, []string{"class1"}, 3, 3, true}, {1, nil, []string{"class1", "class2"}, 1, 1, true}, {nServers*2 + 1, nil, []string{"class1"}, nServers, nServers, false}, } { c.Logf("%+v", trial) st := &StubPutHandler{ c: c, expectPath: "acbd18db4cc2f85cedef654fccc4a4d8", expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "*", returnStorageClasses: "", // Simulate old cluster without SC keep support handled: make(chan string, 100), } ks := RunSomeFakeKeepServers(st, nServers) arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = trial.replicas kc.StorageClasses = trial.clientClasses kc.DefaultStorageClasses = nil // Simulate an old cluster without SC defaults arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{ Data: []byte("foo"), StorageClasses: trial.putClasses, }) if trial.success { c.Check(err, IsNil) } else { c.Check(err, NotNil) } c.Check(len(st.handled) >= trial.minRequests, Equals, true, Commentf("len(st.handled)==%d, trial.minRequests==%d", len(st.handled), trial.minRequests)) c.Check(len(st.handled) <= trial.maxRequests, Equals, true, Commentf("len(st.handled)==%d, trial.maxRequests==%d", len(st.handled), trial.maxRequests)) if trial.clientClasses == nil && trial.putClasses == nil { c.Check(st.requests[0].Header.Get("X-Keep-Storage-Classes"), Equals, "") } } } func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) { nServers := 5 for _, trial := range []struct { replicas int defaultClasses []string clientClasses []string // clientClasses takes precedence over defaultClasses putClasses []string // putClasses takes precedence over clientClasses minRequests int maxRequests int success bool }{ {1, []string{"class1"}, nil, nil, 1, 1, true}, {2, []string{"class1"}, nil, nil, 1, 2, true}, {3, []string{"class1"}, nil, nil, 2, 3, true}, {1, []string{"class1", "class2"}, nil, nil, 1, 1, true}, // defaultClasses doesn't matter when any of the others is specified. {1, []string{"class1"}, []string{"class1"}, nil, 1, 1, true}, {2, []string{"class1"}, []string{"class1"}, nil, 1, 2, true}, {3, []string{"class1"}, []string{"class1"}, nil, 2, 3, true}, {1, []string{"class1"}, []string{"class1", "class2"}, nil, 1, 1, true}, {3, []string{"class1"}, nil, []string{"class1"}, 2, 3, true}, {1, []string{"class1"}, nil, []string{"class1", "class2"}, 1, 1, true}, {1, []string{"class1"}, []string{"class404"}, []string{"class1", "class2"}, 1, 1, true}, {1, []string{"class1"}, []string{"class1"}, []string{"class404", "class2"}, nServers, nServers, false}, {nServers*2 + 1, []string{}, []string{"class1"}, nil, nServers, nServers, false}, {1, []string{"class1"}, []string{"class404"}, nil, nServers, nServers, false}, {1, []string{"class1"}, []string{"class1", "class404"}, nil, nServers, nServers, false}, {1, []string{"class1"}, nil, []string{"class1", "class404"}, nServers, nServers, false}, } { c.Logf("%+v", trial) st := &StubPutHandler{ c: c, expectPath: "acbd18db4cc2f85cedef654fccc4a4d8", expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "*", returnStorageClasses: "class1=2, class2=2", handled: make(chan string, 100), } ks := RunSomeFakeKeepServers(st, nServers) arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = trial.replicas kc.StorageClasses = trial.clientClasses kc.DefaultStorageClasses = trial.defaultClasses arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{ Data: []byte("foo"), StorageClasses: trial.putClasses, }) if trial.success { c.Check(err, IsNil) } else { c.Check(err, NotNil) } c.Check(len(st.handled) >= trial.minRequests, Equals, true, Commentf("len(st.handled)==%d, trial.minRequests==%d", len(st.handled), trial.minRequests)) c.Check(len(st.handled) <= trial.maxRequests, Equals, true, Commentf("len(st.handled)==%d, trial.maxRequests==%d", len(st.handled), trial.maxRequests)) if !trial.success && trial.replicas == 1 && c.Check(len(st.requests) >= 2, Equals, true) { // Max concurrency should be 1. First request // should have succeeded for class1. Second // request should only ask for class404. c.Check(st.requests[1].Header.Get("X-Keep-Storage-Classes"), Equals, "class404") } } } type FailHandler struct { handled chan string } func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.WriteHeader(500) fh.handled <- fmt.Sprintf("http://%s", req.Host) } type FailThenSucceedHandler struct { morefails int // fail 1 + this many times before succeeding handled chan string count atomic.Int64 successhandler http.Handler reqIDs []string } func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id")) if int(fh.count.Add(1)) <= fh.morefails+1 { resp.WriteHeader(500) fh.handled <- fmt.Sprintf("http://%s", req.Host) } else { fh.successhandler.ServeHTTP(resp, req) } } type Error404Handler struct { handled chan string } func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.WriteHeader(404) fh.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { st := FailHandler{ make(chan string)} hash := "acbd18db4cc2f85cedef654fccc4a4d8" UploadToStubHelper(c, st, func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) { go kc.uploadToKeepServer(url, hash, nil, reader, uploadStatusChan, 3, kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled status := <-uploadStatusChan c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash)) c.Check(status.statusCode, Equals, 500) }) } type KeepServer struct { listener net.Listener url string } func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) { ks = make([]KeepServer, n) for i := 0; i < n; i++ { ks[i] = RunFakeKeepServer(st) } return ks } func (s *StandaloneSuite) TestPutB(c *C) { hash := Md5String("foo") st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 5), } arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks := RunSomeFakeKeepServers(st, 5) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) kc.PutB([]byte("foo")) shuff := NewRootSorter( kc.LocalRoots(), Md5String("foo")).GetSortedRoots() s1 := <-st.handled s2 := <-st.handled c.Check((s1 == shuff[0] && s2 == shuff[1]) || (s1 == shuff[1] && s2 == shuff[0]), Equals, true) } func (s *StandaloneSuite) TestPutHR(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 5), } arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks := RunSomeFakeKeepServers(st, 5) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) kc.PutHR(hash, bytes.NewBuffer([]byte("foo")), 3) shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots() s1 := <-st.handled s2 := <-st.handled c.Check((s1 == shuff[0] && s2 == shuff[1]) || (s1 == shuff[1] && s2 == shuff[0]), Equals, true) } func (s *StandaloneSuite) TestPutWithFail(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 4), } fh := FailHandler{ make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks1 := RunSomeFakeKeepServers(st, 4) ks2 := RunSomeFakeKeepServers(fh, 1) for i, k := range ks1 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) shuff := NewRootSorter( kc.LocalRoots(), Md5String("foo")).GetSortedRoots() c.Logf("%+v", shuff) phash, replicas, err := kc.PutB([]byte("foo")) <-fh.handled c.Check(err, IsNil) c.Check(phash, Equals, "") c.Check(replicas, Equals, 2) s1 := <-st.handled s2 := <-st.handled c.Check((s1 == shuff[1] && s2 == shuff[2]) || (s1 == shuff[2] && s2 == shuff[1]), Equals, true) } func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 1), } fh := FailHandler{ make(chan string, 4)} arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 kc.Retries = 0 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks1 := RunSomeFakeKeepServers(st, 1) ks2 := RunSomeFakeKeepServers(fh, 4) for i, k := range ks1 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) c.Check(err, FitsTypeOf, InsufficientReplicasError{}) c.Check(replicas, Equals, 1) c.Check(<-st.handled, Equals, ks1[0].url) } type StubGetHandler struct { c *C expectPath string expectAPIToken string httpStatus int body []byte } func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath) sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken)) resp.WriteHeader(sgh.httpStatus) resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body))) resp.Write(sgh.body) } func (s *StandaloneSuite) TestGet(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := StubGetHandler{ c, hash, "abc123", http.StatusOK, []byte("foo")} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(hash) c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) content, err2 := ioutil.ReadAll(r) c.Check(err2, IsNil) c.Check(content, DeepEquals, []byte("foo")) c.Check(r.Close(), IsNil) } func (s *StandaloneSuite) TestGet404(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := Error404Handler{make(chan string, 1)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(hash) c.Check(err, Equals, BlockNotFound) c.Check(n, Equals, int64(0)) c.Check(r, IsNil) } func (s *StandaloneSuite) TestGetEmptyBlock(c *C) { st := Error404Handler{make(chan string, 1)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0") c.Check(err, IsNil) c.Check(n, Equals, int64(0)) c.Assert(r, NotNil) buf, err := ioutil.ReadAll(r) c.Check(err, IsNil) c.Check(buf, DeepEquals, []byte{}) c.Check(r.Close(), IsNil) } func (s *StandaloneSuite) TestGetFail(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := FailHandler{make(chan string, 1)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) kc.Retries = 0 r, n, _, err := kc.Get(hash) errNotFound, _ := err.(*ErrNotFound) if c.Check(errNotFound, NotNil) { c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true) c.Check(errNotFound.Temporary(), Equals, true) } c.Check(n, Equals, int64(0)) c.Check(r, IsNil) } func (s *StandaloneSuite) TestGetFailRetry(c *C) { defer func(origDefault, origMinimum time.Duration) { DefaultRetryDelay = origDefault MinimumRetryDelay = origMinimum }(DefaultRetryDelay, MinimumRetryDelay) DefaultRetryDelay = time.Second / 8 MinimumRetryDelay = time.Millisecond hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) for _, delay := range []time.Duration{0, time.Nanosecond, time.Second / 8, time.Second / 16} { c.Logf("=== initial delay %v", delay) st := &FailThenSucceedHandler{ morefails: 2, handled: make(chan string, 4), successhandler: StubGetHandler{ c, hash, "abc123", http.StatusOK, []byte("foo")}} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) kc.Retries = 3 kc.RetryDelay = delay kc.DiskCacheSize = DiskCacheDisabled t0 := time.Now() r, n, _, err := kc.Get(hash) c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) elapsed := time.Since(t0) nonsleeptime := time.Second / 10 expect := kc.RetryDelay if expect == 0 { expect = DefaultRetryDelay } min := MinimumRetryDelay * 3 max := expect + expect*2 + expect*2*2 + nonsleeptime c.Check(elapsed >= min, Equals, true, Commentf("elapsed %v / expect min %v", elapsed, min)) c.Check(elapsed <= max, Equals, true, Commentf("elapsed %v / expect max %v", elapsed, max)) content, err := ioutil.ReadAll(r) c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("foo")) c.Check(r.Close(), IsNil) c.Logf("%q", st.reqIDs) if c.Check(st.reqIDs, Not(HasLen), 0) { for _, reqid := range st.reqIDs { c.Check(reqid, Not(Equals), "") c.Check(reqid, Equals, st.reqIDs[0]) } } } } func (s *StandaloneSuite) TestGetNetError(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) r, n, _, err := kc.Get(hash) errNotFound, _ := err.(*ErrNotFound) if c.Check(errNotFound, NotNil) { c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true) c.Check(errNotFound.Temporary(), Equals, true) } c.Check(n, Equals, int64(0)) c.Check(r, IsNil) } func (s *StandaloneSuite) TestGetWithServiceHint(c *C) { uuid := "zzzzz-bi6l4-123451234512345" hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) // This one shouldn't be used: ks0 := RunFakeKeepServer(StubGetHandler{ c, "error if used", "abc123", http.StatusOK, []byte("foo")}) defer ks0.listener.Close() // This one should be used: ks := RunFakeKeepServer(StubGetHandler{ c, hash + "+K@" + uuid, "abc123", http.StatusOK, []byte("foo")}) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( map[string]string{"x": ks0.url}, nil, map[string]string{uuid: ks.url}) r, n, _, err := kc.Get(hash + "+K@" + uuid) c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) content, err := ioutil.ReadAll(r) c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("foo")) c.Check(r.Close(), IsNil) } // Use a service hint to fetch from a local disk service, overriding // rendezvous probe order. func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) { uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz" hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) // This one shouldn't be used, although it appears first in // rendezvous probe order: ks0 := RunFakeKeepServer(StubGetHandler{ c, "error if used", "abc123", http.StatusBadGateway, nil}) defer ks0.listener.Close() // This one should be used: ks := RunFakeKeepServer(StubGetHandler{ c, hash + "+K@" + uuid, "abc123", http.StatusOK, []byte("foo")}) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( map[string]string{ "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, uuid: ks.url}, nil, map[string]string{ "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, uuid: ks.url}, ) r, n, _, err := kc.Get(hash + "+K@" + uuid) c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) content, err := ioutil.ReadAll(r) c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("foo")) c.Check(r.Close(), IsNil) } func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) { uuid := "zzzzz-bi6l4-123451234512345" hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) ksLocal := RunFakeKeepServer(StubGetHandler{ c, hash + "+K@" + uuid, "abc123", http.StatusOK, []byte("foo")}) defer ksLocal.listener.Close() ksGateway := RunFakeKeepServer(StubGetHandler{ c, hash + "+K@" + uuid, "abc123", http.StatusInternalServerError, []byte("Error")}) defer ksGateway.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url}, nil, map[string]string{uuid: ksGateway.url}) r, n, _, err := kc.Get(hash + "+K@" + uuid) c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) content, err := ioutil.ReadAll(r) c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("foo")) c.Check(r.Close(), IsNil) } type BarHandler struct { handled chan string } func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Write([]byte("bar")) h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestChecksum(c *C) { foohash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) barhash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar"))) st := BarHandler{make(chan string, 1)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(barhash) if c.Check(err, IsNil) { _, err = ioutil.ReadAll(r) c.Check(n, Equals, int64(3)) c.Check(err, IsNil) } select { case <-st.handled: case <-time.After(time.Second): c.Fatal("timed out") } r, n, _, err = kc.Get(foohash) if err == nil { buf, readerr := ioutil.ReadAll(r) c.Logf("%q", buf) err = readerr } c.Check(err, Equals, BadChecksum) select { case <-st.handled: case <-time.After(time.Second): c.Fatal("timed out") } } func (s *StandaloneSuite) TestGetWithFailures(c *C) { content := []byte("waz") hash := fmt.Sprintf("%x+3", md5.Sum(content)) fh := Error404Handler{ make(chan string, 4)} st := StubGetHandler{ c, hash, "abc123", http.StatusOK, content} arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks1 := RunSomeFakeKeepServers(st, 1) ks2 := RunSomeFakeKeepServers(fh, 4) for i, k := range ks1 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) kc.Retries = 0 // This test works only if one of the failing services is // attempted before the succeeding service. Otherwise, // <-fh.handled below will just hang! (Probe order depends on // the choice of block content "waz" and the UUIDs of the fake // servers, so we just tried different strings until we found // an example that passes this Assert.) c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url) r, n, _, err := kc.Get(hash) select { case <-fh.handled: case <-time.After(time.Second): c.Fatal("timed out") } c.Assert(err, IsNil) c.Check(n, Equals, int64(3)) readContent, err2 := ioutil.ReadAll(r) c.Check(err2, IsNil) c.Check(readContent, DeepEquals, content) c.Check(r.Close(), IsNil) } func (s *ServerRequiredSuite) TestPutGetHead(c *C) { content := []byte("TestPutGetHead") arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, err := MakeKeepClient(arv) c.Assert(err, IsNil) hash := fmt.Sprintf("%x+%d", md5.Sum(content), len(content)) { n, _, err := kc.Ask(hash) c.Check(err, Equals, BlockNotFound) c.Check(n, Equals, int64(0)) } { hash2, replicas, err := kc.PutB(content) c.Check(err, IsNil) c.Check(hash2, Matches, `\Q`+hash+`\E\b.*`) c.Check(replicas, Equals, 2) } { r, n, _, err := kc.Get(hash) c.Check(err, IsNil) c.Check(n, Equals, int64(len(content))) if c.Check(r, NotNil) { readContent, err := ioutil.ReadAll(r) c.Check(err, IsNil) if c.Check(len(readContent), Equals, len(content)) { c.Check(readContent, DeepEquals, content) } c.Check(r.Close(), IsNil) } } { n, url2, err := kc.Ask(hash) c.Check(err, IsNil) c.Check(n, Equals, int64(len(content))) c.Check(url2, Matches, "http://localhost:\\d+/\\Q"+hash+"\\E") } { loc, err := kc.LocalLocator(hash) c.Check(err, IsNil) c.Assert(len(loc) >= 32, Equals, true) c.Check(loc[:32], Equals, hash[:32]) } { content := []byte("the perth county conspiracy") loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content))) c.Check(loc, Equals, "") c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`) c.Check(err, ErrorMatches, `.*HTTP 400.*`) } } type StubProxyHandler struct { handled chan string } func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Header().Set("X-Keep-Replicas-Stored", "2") h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestPutProxy(c *C) { st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks1 := RunSomeFakeKeepServers(st, 1) for i, k := range ks1 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) <-st.handled c.Check(err, IsNil) c.Check(replicas, Equals, 2) } func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 3 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks1 := RunSomeFakeKeepServers(st, 1) for i, k := range ks1 { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) <-st.handled c.Check(err, FitsTypeOf, InsufficientReplicasError{}) c.Check(replicas, Equals, 2) } func (s *StandaloneSuite) TestMakeLocator(c *C) { l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678") c.Check(err, IsNil) c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") c.Check(l.Size, Equals, 3) c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"}) } func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) { l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce") c.Check(err, IsNil) c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") c.Check(l.Size, Equals, -1) c.Check(l.Hints, DeepEquals, []string{}) } func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) { l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678") c.Check(err, IsNil) c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") c.Check(l.Size, Equals, -1) c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"}) } func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) { str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar" l, err := MakeLocator(str) c.Check(err, IsNil) c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") c.Check(l.Size, Equals, 3) c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"}) c.Check(l.String(), Equals, str) } func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) { _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c") c.Check(err, Equals, InvalidLocatorError) } func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) { hash := Md5String("foo") st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 5), } arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks := RunSomeFakeKeepServers(st, 5) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url if i == 0 { writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url } defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) c.Check(err, FitsTypeOf, InsufficientReplicasError{}) c.Check(replicas, Equals, 1) c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)]) } func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) { hash := Md5String("foo") st := &StubPutHandler{ c: c, expectPath: hash, expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "", returnStorageClasses: "", handled: make(chan string, 5), } arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks := RunSomeFakeKeepServers(st, 5) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) c.Check(err, FitsTypeOf, InsufficientReplicasError{}) c.Check(replicas, Equals, 0) } type StubGetIndexHandler struct { c *C expectPath string expectAPIToken string httpStatus int body []byte } func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { h.c.Check(req.URL.Path, Equals, h.expectPath) h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken)) resp.WriteHeader(h.httpStatus) resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body))) resp.Write(h.body) } func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := StubGetIndexHandler{ c, "/index", "abc123", http.StatusOK, []byte(hash + " 1443559274\n\n")} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) kc, err := MakeKeepClient(arv) c.Assert(err, IsNil) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, err := kc.GetIndex("x", "") c.Check(err, IsNil) content, err2 := ioutil.ReadAll(r) c.Check(err2, IsNil) c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) } func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := StubGetIndexHandler{ c, "/index/" + hash[0:3], "abc123", http.StatusOK, []byte(hash + " 1443559274\n\n")} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, err := kc.GetIndex("x", hash[0:3]) c.Assert(err, IsNil) content, err2 := ioutil.ReadAll(r) c.Check(err2, IsNil) c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) } func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := StubGetIndexHandler{ c, "/index/" + hash[0:3], "abc123", http.StatusOK, []byte(hash)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) _, err = kc.GetIndex("x", hash[0:3]) c.Check(err, Equals, ErrIncompleteIndex) } func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) { hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo"))) st := StubGetIndexHandler{ c, "/index/" + hash[0:3], "abc123", http.StatusOK, []byte(hash)} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) _, err = kc.GetIndex("y", hash[0:3]) c.Check(err, Equals, ErrNoSuchKeepServer) } func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { st := StubGetIndexHandler{ c, "/index/abcd", "abc123", http.StatusOK, []byte("\n")} ks := RunFakeKeepServer(st) defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, err := kc.GetIndex("x", "abcd") c.Check(err, IsNil) content, err2 := ioutil.ReadAll(r) c.Check(err2, IsNil) c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) } func (s *StandaloneSuite) TestPutBRetry(c *C) { DefaultRetryDelay = time.Second / 8 MinimumRetryDelay = time.Millisecond for _, delay := range []time.Duration{0, time.Nanosecond, time.Second / 8, time.Second / 16} { c.Logf("=== initial delay %v", delay) st := &FailThenSucceedHandler{ morefails: 5, // handler will fail 6x in total, 3 for each server handled: make(chan string, 10), successhandler: &StubPutHandler{ c: c, expectPath: Md5String("foo"), expectAPIToken: "abc123", expectBody: "foo", expectStorageClass: "default", returnStorageClasses: "", handled: make(chan string, 5), }, } arv, _ := arvadosclient.MakeArvadosClient() kc, _ := MakeKeepClient(arv) kc.Retries = 3 kc.RetryDelay = delay kc.DiskCacheSize = DiskCacheDisabled kc.Want_replicas = 2 arv.ApiToken = "abc123" localRoots := make(map[string]string) writableLocalRoots := make(map[string]string) ks := RunSomeFakeKeepServers(st, 2) for i, k := range ks { localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } kc.SetServiceRoots(localRoots, writableLocalRoots, nil) t0 := time.Now() hash, replicas, err := kc.PutB([]byte("foo")) c.Check(err, IsNil) c.Check(hash, Equals, "") c.Check(replicas, Equals, 2) elapsed := time.Since(t0) nonsleeptime := time.Second / 10 expect := kc.RetryDelay if expect == 0 { expect = DefaultRetryDelay } min := MinimumRetryDelay * 3 max := expect + expect*2 + expect*2*2 max += nonsleeptime checkInterval(c, elapsed, min, max) } } func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, IsNil) // Add an additional "testblobstore" keepservice blobKeepService := make(arvadosclient.Dict) err = arv.Create("keep_services", arvadosclient.Dict{"keep_service": arvadosclient.Dict{ "service_host": "localhost", "service_port": "21321", "service_type": "testblobstore"}}, &blobKeepService) c.Assert(err, IsNil) defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }() RefreshServiceDiscovery() // Make a keepclient and ensure that the testblobstore is included kc, err := MakeKeepClient(arv) c.Assert(err, IsNil) // verify kc.LocalRoots c.Check(len(kc.LocalRoots()), Equals, 3) for _, root := range kc.LocalRoots() { c.Check(root, Matches, "http://localhost:\\d+") } c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") // verify kc.GatewayRoots c.Check(len(kc.GatewayRoots()), Equals, 3) for _, root := range kc.GatewayRoots() { c.Check(root, Matches, "http://localhost:\\d+") } c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") // verify kc.WritableLocalRoots c.Check(len(kc.WritableLocalRoots()), Equals, 3) for _, root := range kc.WritableLocalRoots() { c.Check(root, Matches, "http://localhost:\\d+") } c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") c.Assert(kc.replicasPerService, Equals, 0) c.Assert(kc.foundNonDiskSvc, Equals, true) c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second) } func (s *StandaloneSuite) TestDelayCalculator_Default(c *C) { MinimumRetryDelay = time.Second / 2 DefaultRetryDelay = time.Second dc := delayCalculator{InitialMaxDelay: 0} checkInterval(c, dc.Next(), time.Second/2, time.Second) checkInterval(c, dc.Next(), time.Second/2, time.Second*2) checkInterval(c, dc.Next(), time.Second/2, time.Second*4) checkInterval(c, dc.Next(), time.Second/2, time.Second*8) checkInterval(c, dc.Next(), time.Second/2, time.Second*10) checkInterval(c, dc.Next(), time.Second/2, time.Second*10) } func (s *StandaloneSuite) TestDelayCalculator_SetInitial(c *C) { MinimumRetryDelay = time.Second / 2 DefaultRetryDelay = time.Second dc := delayCalculator{InitialMaxDelay: time.Second * 2} checkInterval(c, dc.Next(), time.Second/2, time.Second*2) checkInterval(c, dc.Next(), time.Second/2, time.Second*4) checkInterval(c, dc.Next(), time.Second/2, time.Second*8) checkInterval(c, dc.Next(), time.Second/2, time.Second*16) checkInterval(c, dc.Next(), time.Second/2, time.Second*20) checkInterval(c, dc.Next(), time.Second/2, time.Second*20) checkInterval(c, dc.Next(), time.Second/2, time.Second*20) } func (s *StandaloneSuite) TestDelayCalculator_EnsureSomeLongDelays(c *C) { dc := delayCalculator{InitialMaxDelay: time.Second * 5} var d time.Duration n := 4000 for i := 0; i < n; i++ { if i < 20 || i%10 == 0 { c.Logf("i=%d, delay=%v", i, d) } if d = dc.Next(); d > dc.InitialMaxDelay*9 { return } } c.Errorf("after %d trials, never got a delay more than 90%% of expected max %d; last was %v", n, dc.InitialMaxDelay*10, d) } // If InitialMaxDelay is less than MinimumRetryDelay/10, then delay is // always MinimumRetryDelay. func (s *StandaloneSuite) TestDelayCalculator_InitialLessThanMinimum(c *C) { MinimumRetryDelay = time.Second / 2 dc := delayCalculator{InitialMaxDelay: time.Millisecond} for i := 0; i < 20; i++ { c.Check(dc.Next(), Equals, time.Second/2) } } func checkInterval(c *C, t, min, max time.Duration) { c.Check(t >= min, Equals, true, Commentf("got %v which is below expected min %v", t, min)) c.Check(t <= max, Equals, true, Commentf("got %v which is above expected max %v", t, max)) }