X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/22361307cf41f916afd562e7f33fcdaacefe5f9d..da4bc7c758d09c1c02542b54b96eab018f746eae:/sdk/go/keepclient/keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 19a6d4e03b..531db31b25 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -40,7 +40,10 @@ var _ = Suite(&StandaloneSuite{})
 type ServerRequiredSuite struct{}
 
 // Standalone tests
-type StandaloneSuite struct{}
+type StandaloneSuite struct {
+	origDefaultRetryDelay time.Duration
+	origMinimumRetryDelay time.Duration
+}
 
 var origHOME = os.Getenv("HOME")
 
@@ -48,10 +51,14 @@ func (s *StandaloneSuite) SetUpTest(c *C) {
 	RefreshServiceDiscovery()
 	// Prevent cache state from leaking between test cases
 	os.Setenv("HOME", c.MkDir())
+	s.origDefaultRetryDelay = DefaultRetryDelay
+	s.origMinimumRetryDelay = MinimumRetryDelay
 }
 
 func (s *StandaloneSuite) TearDownTest(c *C) {
 	os.Setenv("HOME", origHOME)
+	DefaultRetryDelay = s.origDefaultRetryDelay
+	MinimumRetryDelay = s.origMinimumRetryDelay
 }
 
 func pythonDir() string {
@@ -798,8 +805,12 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
 }
 
 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
-	defer func(orig time.Duration) { DefaultRetryDelay = orig }(DefaultRetryDelay)
+	defer func(origDefault, origMinimum time.Duration) {
+		DefaultRetryDelay = origDefault
+		MinimumRetryDelay = origMinimum
+	}(DefaultRetryDelay, MinimumRetryDelay)
 	DefaultRetryDelay = time.Second / 8
+	MinimumRetryDelay = time.Millisecond
 
 	hash := fmt.Sprintf("%x+3", md5.Sum([]byte("foo")))
 
@@ -839,7 +850,7 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) {
 		if expect == 0 {
 			expect = DefaultRetryDelay
 		}
-		min := expect + expect*7/4 + expect*7/4*7/4
+		min := MinimumRetryDelay * 3
 		max := expect + expect*2 + expect*2*2 + nonsleeptime
 		c.Check(elapsed >= min, Equals, true, Commentf("elapsed %v / expect min %v", elapsed, min))
 		c.Check(elapsed <= max, Equals, true, Commentf("elapsed %v / expect max %v", elapsed, max))
@@ -1502,8 +1513,8 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
 }
 
 func (s *StandaloneSuite) TestPutBRetry(c *C) {
-	defer func(orig time.Duration) { DefaultRetryDelay = orig }(DefaultRetryDelay)
 	DefaultRetryDelay = time.Second / 8
+	MinimumRetryDelay = time.Millisecond
 
 	for _, delay := range []time.Duration{0, time.Nanosecond, time.Second / 8, time.Second / 16} {
 		c.Logf("=== initial delay %v", delay)
@@ -1556,10 +1567,10 @@ func (s *StandaloneSuite) TestPutBRetry(c *C) {
 		if expect == 0 {
 			expect = DefaultRetryDelay
 		}
-		min := expect + expect*7/4 + expect*7/4*7/4
-		max := expect + expect*2 + expect*2*2 + nonsleeptime
-		c.Check(elapsed >= min, Equals, true, Commentf("elapsed %v / expect min %v", elapsed, min))
-		c.Check(elapsed <= max, Equals, true, Commentf("elapsed %v / expect max %v", elapsed, max))
+		min := MinimumRetryDelay * 3
+		max := expect + expect*2 + expect*2*2
+		max += nonsleeptime
+		checkInterval(c, elapsed, min, max)
 	}
 }
 
@@ -1608,3 +1619,60 @@ func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
 	c.Assert(kc.foundNonDiskSvc, Equals, true)
 	c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
 }
+
+func (s *StandaloneSuite) TestDelayCalculator_Default(c *C) {
+	MinimumRetryDelay = time.Second / 2
+	DefaultRetryDelay = time.Second
+
+	dc := delayCalculator{InitialMaxDelay: 0}
+	checkInterval(c, dc.Next(), time.Second/2, time.Second)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*2)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*4)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*8)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*10)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*10)
+}
+
+func (s *StandaloneSuite) TestDelayCalculator_SetInitial(c *C) {
+	MinimumRetryDelay = time.Second / 2
+	DefaultRetryDelay = time.Second
+
+	dc := delayCalculator{InitialMaxDelay: time.Second * 2}
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*2)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*4)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*8)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*16)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*20)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*20)
+	checkInterval(c, dc.Next(), time.Second/2, time.Second*20)
+}
+
+func (s *StandaloneSuite) TestDelayCalculator_EnsureSomeLongDelays(c *C) {
+	dc := delayCalculator{InitialMaxDelay: time.Second * 5}
+	var d time.Duration
+	n := 4000
+	for i := 0; i < n; i++ {
+		if i < 20 || i%10 == 0 {
+			c.Logf("i=%d, delay=%v", i, d)
+		}
+		if d = dc.Next(); d > dc.InitialMaxDelay*9 {
+			return
+		}
+	}
+	c.Errorf("after %d trials, never got a delay more than 90%% of expected max %d; last was %v", n, dc.InitialMaxDelay*10, d)
+}
+
+// If InitialMaxDelay is less than MinimumRetryDelay/10, then delay is
+// always MinimumRetryDelay.
+func (s *StandaloneSuite) TestDelayCalculator_InitialLessThanMinimum(c *C) {
+	MinimumRetryDelay = time.Second / 2
+	dc := delayCalculator{InitialMaxDelay: time.Millisecond}
+	for i := 0; i < 20; i++ {
+		c.Check(dc.Next(), Equals, time.Second/2)
+	}
+}
+
+func checkInterval(c *C, t, min, max time.Duration) {
+	c.Check(t >= min, Equals, true, Commentf("got %v which is below expected min %v", t, min))
+	c.Check(t <= max, Equals, true, Commentf("got %v which is above expected max %v", t, max))
+}