14199: Fix deadlock in test suite.

If a KeepClient has been created using an API server address that is
no longer reachable, calling keepclient.RefreshServiceDiscovery() puts
the poll() goroutine into an endless retry loop, and a second call
never returns because ent.clear is never ready to receive.

Work around this in the pull worker tests by only refreshing services
from the API server actually being used, not on additional ones
referenced by previous test cases.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 4377c1951528c4eab88a3525934669c9457f954c..2392fcde7bdc7475068bfac3452665daa2ef5a61 100644 (file)
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -20,9 +20,10 @@ import (
 
 // ClearCache clears the Keep service discovery cache.
 func RefreshServiceDiscovery() {
+       var wg sync.WaitGroup
+       defer wg.Wait()
        svcListCacheMtx.Lock()
        defer svcListCacheMtx.Unlock()
-       var wg sync.WaitGroup
        for _, ent := range svcListCache {
                wg.Add(1)
                go func() {
@@ -30,7 +31,6 @@ func RefreshServiceDiscovery() {
                        wg.Done()
                }()
        }
-       wg.Wait()
 }
 
 // ClearCacheOnSIGHUP installs a signal handler that calls
@@ -153,6 +153,16 @@ func (kc *KeepClient) discoverServices() error {
        return kc.loadKeepServers(<-cacheEnt.latest)
 }
 
+func (kc *KeepClient) RefreshServiceDiscovery() {
+       svcListCacheMtx.Lock()
+       ent, ok := svcListCache[kc.Arvados.ApiServer]
+       svcListCacheMtx.Unlock()
+       if !ok || kc.Arvados.KeepServiceURIs != nil || kc.disableDiscovery {
+               return
+       }
+       ent.clear <- struct{}{}
+}
+
 // LoadKeepServicesFromJSON gets list of available keep services from
 // given JSON and disables automatic service discovery.
 func (kc *KeepClient) LoadKeepServicesFromJSON(services string) error {

diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index c06bbe05c4b2ecde56efa2143f88382b19205235..231a4c0ab28097340f558c6179fd14d9d50b9d3f 100644 (file)
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -33,7 +33,6 @@ func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTe
        // start api and keep servers
        arvadostest.StartAPI()
        arvadostest.StartKeep(2, false)
-       keepclient.RefreshServiceDiscovery()
 
        // make arvadosclient
        arv, err := arvadosclient.MakeArvadosClient()
@@ -47,6 +46,7 @@ func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTe
                t.Fatalf("error creating KeepClient: %s", err)
        }
        keepClient.Want_replicas = 1
+       keepClient.RefreshServiceDiscovery()
 
        // discover keep services
        var servers []string