7710: Remove KeepClient.Using_proxy and all it's references since we are no longer...

Update datamanager to use a configured supported service type instead of hardcoding "disk" type.

diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 1095085edb1077aa65c4357e0cde8c229ddd1bb7..099c56f343b5e65ead54c723271f4a312d65ad2b 100644 (file)
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -84,7 +84,6 @@ func (this *KeepClient) loadKeepServers(list svcList) error {
 
        // replicasPerService is 1 for disks; unknown or unlimited otherwise
        this.replicasPerService = 1
-       this.Using_proxy = false
 
        for _, service := range list.Items {
                scheme := "http"
@@ -100,10 +99,6 @@ func (this *KeepClient) loadKeepServers(list svcList) error {
                listed[url] = true
 
                localRoots[service.Uuid] = url
-               if service.SvcType == "proxy" {
-                       this.Using_proxy = true
-               }
-
                if service.ReadOnly == false {
                        writableLocalRoots[service.Uuid] = url
                        if service.SvcType != "disk" {

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index ddb502c87ded5e7209846863c1a48844b8448399..f15a6b2a2c5cbfeb6320840e5c7346b8bab3bb8a 100644 (file)
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -68,7 +68,6 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
 type KeepClient struct {
        Arvados            *arvadosclient.ArvadosClient
        Want_replicas      int
-       Using_proxy        bool
        localRoots         *map[string]string
        writableLocalRoots *map[string]string
        gatewayRoots       *map[string]string
@@ -104,7 +103,6 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
        kc := &KeepClient{
                Arvados:       arv,
                Want_replicas: defaultReplicationLevel,
-               Using_proxy:   false,
                Client: &http.Client{Transport: &http.Transport{
                        TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
                Retries: 2,

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 2c77e68af006a769fd6aa1dafda046566f00a3bc..87b9b1d1eb5652a63c9253fa34aa57db60ef054f 100644 (file)
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -895,7 +895,6 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
        kc, _ := MakeKeepClient(&arv)
 
        kc.Want_replicas = 2
-       kc.Using_proxy = true
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
        writableLocalRoots := make(map[string]string)
@@ -928,7 +927,6 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
        kc, _ := MakeKeepClient(&arv)
 
        kc.Want_replicas = 3
-       kc.Using_proxy = true
        arv.ApiToken = "abc123"
        localRoots := make(map[string]string)
        writableLocalRoots := make(map[string]string)
@@ -1271,21 +1269,21 @@ func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
        for _, root := range kc.LocalRoots() {
                c.Check(root, Matches, "http://localhost:\\d+")
        }
-       c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], NotNil)
+       c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
 
        // verify kc.GatewayRoots
        c.Check(len(kc.GatewayRoots()), Equals, 3)
        for _, root := range kc.GatewayRoots() {
                c.Check(root, Matches, "http://localhost:\\d+")
        }
-       c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], NotNil)
+       c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
 
        // verify kc.WritableLocalRoots
        c.Check(len(kc.WritableLocalRoots()), Equals, 3)
        for _, root := range kc.WritableLocalRoots() {
                c.Check(root, Matches, "http://localhost:\\d+")
        }
-       c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], NotNil)
+       c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
 
        c.Assert(kc.replicasPerService, Equals, 0)
        c.Assert(kc.foundNonDiskSvc, Equals, true)

diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 28faf989ce956c667e51fd00fc19954e18e91794..42039365236851ebbb4f7b289382c6e39da9176a 100644 (file)
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -41,7 +41,6 @@ func SetupDataManagerTest(t *testing.T) {
        keepClient = &keepclient.KeepClient{
                Arvados:       &arv,
                Want_replicas: 2,
-               Using_proxy:   true,
                Client:        &http.Client{},
        }
 

diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 7da74b168069ac682d03896ee59fd6af2a5c1b04..c16512069ebcce3ea75c5c91398630d260989102 100644 (file)
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -6,6 +6,7 @@ import (
        "bufio"
        "encoding/json"
        "errors"
+       "flag"
        "fmt"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/blockdigest"
@@ -76,6 +77,15 @@ type ServiceList struct {
        KeepServers    []ServerAddress `json:"items"`
 }
 
+var serviceType string
+
+func init() {
+       flag.StringVar(&serviceType,
+               "service-type",
+               "disk",
+               "Supported keepservice type. Default is disk.")
+}
+
 // String
 // TODO(misha): Change this to include the UUID as well.
 func (s ServerAddress) String() string {
@@ -122,12 +132,19 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
        }
 
        // Currently, only "disk" types are supported. Stop if any other service types are found.
+       foundSupportedServieType := false
        for _, server := range sdkResponse.KeepServers {
-               if server.ServiceType != "disk" {
-                       return results, fmt.Errorf("Unsupported service type %q found for: %v", server.ServiceType, server)
+               if server.ServiceType == serviceType {
+                       foundSupportedServieType = true
+               } else {
+                       log.Printf("Ignore unsupported service type: %v", server.ServiceType)
                }
        }
 
+       if !foundSupportedServieType {
+               return results, fmt.Errorf("Found no keepservices with the supported type %v", serviceType)
+       }
+
        if params.Logger != nil {
                params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
                        keepInfo := logger.GetOrCreateMap(p, "keep_info")

diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 0691e647bf9476bdddc312668effb95cbdaadcb0..2286453b2f5d10e699d0b17c9ffddb42095de592 100644 (file)
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -101,7 +101,7 @@ type APITestData struct {
 }
 
 func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) {
-       testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Unsupported service type")
+       testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Found no keepservices with the supported type disk")
 }
 
 func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) {

diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 7c5b3628963ca44e29f3c487e6792ed1f8e33feb..56c9bff91c63d30306ed7b08f3fa08b215e7479f 100644 (file)
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -105,7 +105,6 @@ func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient
        }
        kc.SetServiceRoots(sr, sr, sr)
        kc.Arvados.External = true
-       kc.Using_proxy = true
 
        return kc
 }
@@ -474,7 +473,7 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
        defer closeListener()
 
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
-       for _, f := range []func()error {
+       for _, f := range []func() error{
                func() error {
                        _, _, err := kc.Ask(hash)
                        return err

diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 7525441aaec995d4a524a7ff20f954ee06ae50c9..96a887fecb20b278a2c9a763ebfc094b71bf31ac 100644 (file)
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -300,7 +300,6 @@ func main() {
        keepClient := &keepclient.KeepClient{
                Arvados:       nil,
                Want_replicas: 1,
-               Using_proxy:   true,
                Client:        &http.Client{},
        }
 

diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index 52b59ec1ef797b1e09b25529b20285f5190166df..e0613a2cd12685195d841c2ef2ae31f7b90f60bb 100644 (file)
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -39,7 +39,6 @@ func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTe
        keepClient = &keepclient.KeepClient{
                Arvados:       &arv,
                Want_replicas: 1,
-               Using_proxy:   true,
                Client:        &http.Client{},
        }