From: Tom Clegg <tom@curii.com>
Date: Tue, 28 Feb 2023 21:04:10 +0000 (-0500)
Subject: 19972: Disable auto-retry in ws and dispatcher.
X-Git-Tag: 2.6.0~15^2~4
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/785469f3ca5d79f64f26a88f7499e95b88d811cd

19972: Disable auto-retry in ws and dispatcher.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index 0254c6526c..81982cdc1a 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -21,6 +21,10 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg
 	if err != nil {
 		return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
 	}
+	// Disable auto-retry.  We have transient failure recovery at
+	// the application level, so we would rather receive/report
+	// upstream errors right away.
+	ac.Timeout = 0
 	d := &dispatcher{
 		Cluster:   cluster,
 		Context:   ctx,
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index a9ed95c7c3..9d7d3736b0 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -101,7 +101,8 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
 
 	arvClient, err := arvados.NewClientFromConfig(s.cluster)
-	c.Check(err, check.IsNil)
+	c.Assert(err, check.IsNil)
+	arvClient.Timeout = 0 // disable auto-retry
 
 	s.error503Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) }))
 	arvClient.Client = &http.Client{
diff --git a/services/ws/permission.go b/services/ws/permission.go
index ac895f80e5..78158b12a1 100644
--- a/services/ws/permission.go
+++ b/services/ws/permission.go
@@ -75,7 +75,9 @@ func (pc *cachingPermChecker) Check(ctx context.Context, uuid string) (bool, err
 	}
 
 	pc.nMisses++
-	err = pc.RequestAndDecode(&buf, "GET", path, nil, url.Values{
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+	defer cancel()
+	err = pc.RequestAndDecodeContext(ctx, &buf, "GET", path, nil, url.Values{
 		"include_trash": {"true"},
 		"select":        {`["uuid"]`},
 	})
diff --git a/services/ws/permission_test.go b/services/ws/permission_test.go
index 023656c01f..2e08cea484 100644
--- a/services/ws/permission_test.go
+++ b/services/ws/permission_test.go
@@ -17,7 +17,9 @@ var _ = check.Suite(&permSuite{})
 type permSuite struct{}
 
 func (s *permSuite) TestCheck(c *check.C) {
-	pc := newPermChecker(*(arvados.NewClientFromEnv())).(*cachingPermChecker)
+	client := arvados.NewClientFromEnv()
+	client.Timeout = 0 // disable auto-retry
+	pc := newPermChecker(*client).(*cachingPermChecker)
 	setToken := func(label, token string) {
 		c.Logf("...%s token %q", label, token)
 		pc.SetToken(token)
diff --git a/services/ws/service.go b/services/ws/service.go
index 761e22e16c..d6501e0771 100644
--- a/services/ws/service.go
+++ b/services/ws/service.go
@@ -7,6 +7,7 @@ package ws
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/lib/service"
@@ -24,6 +25,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg
 	if err != nil {
 		return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
 	}
+	client.Timeout = time.Minute
 	eventSource := &pgEventSource{
 		DataSource:   cluster.PostgreSQL.Connection.String(),
 		MaxOpenConns: cluster.PostgreSQL.ConnectionPool,