7167: Refactor MakeKeepClient and DiscoverKeepServers to allow making KeepClient...

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 53dfb2b5384302b0f113ec608c016760a2ca77ce..05fad6e10e33f1daa3c93d8c3dab2a6ad07b3b78 100644 (file)
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -54,9 +54,20 @@ type KeepClient struct {
        replicasPerService int
 }
 
-// Create a new KeepClient.  This will contact the API server to discover Keep
-// servers.
+// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
 func MakeKeepClient(arv *arvadosclient.ArvadosClient) (*KeepClient, error) {
+       kc := initKeepClient(arv)
+       return kc, kc.DiscoverKeepServers()
+}
+
+// MakeKeepClientFromJSON creates a new KeepClient using the given json to load keep servers.
+func MakeKeepClientFromJSON(arv *arvadosclient.ArvadosClient, svcJSON string) (*KeepClient, error) {
+       kc := initKeepClient(arv)
+       return kc, kc.DiscoverKeepServersFromJSON(svcJSON)
+}
+
+// Make a new KeepClient struct.
+func initKeepClient(arv *arvadosclient.ArvadosClient) *KeepClient {
        var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
        insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
        kc := &KeepClient{
@@ -66,7 +77,7 @@ func MakeKeepClient(arv *arvadosclient.ArvadosClient) (*KeepClient, error) {
                Client: &http.Client{Transport: &http.Transport{
                        TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
        }
-       return kc, kc.DiscoverKeepServers()
+       return kc
 }
 
 // Put a block given the block hash, a reader, and the number of bytes

diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 63800b1e08577edec185cab13bc6c97780b349b1..8be178036ad0cce1a447391cf0d3a4ec8e8a625d 100644 (file)
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -2,6 +2,7 @@ package keepclient
 
 import (
        "crypto/md5"
+       "encoding/json"
        "errors"
        "fmt"
        "git.curoverse.com/arvados.git/sdk/go/streamer"
@@ -76,19 +77,38 @@ func (this *KeepClient) setClientSettingsDisk() {
        }
 }
 
+type svcList struct {
+       Items []keepService `json:"items"`
+}
+
 // DiscoverKeepServers gets list of available keep services from api server
 func (this *KeepClient) DiscoverKeepServers() error {
-       type svcList struct {
-               Items []keepService `json:"items"`
-       }
-       var m svcList
+       var list svcList
 
        // Get keep services from api server
-       err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &m)
+       err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &list)
        if err != nil {
                return err
        }
 
+       return this.loadKeepServers(list)
+}
+
+// DiscoverKeepServersFromJSON gets list of available keep services from given JSON
+func (this *KeepClient) DiscoverKeepServersFromJSON(services string) error {
+       var list svcList
+
+       // Load keep services from given json
+       dec := json.NewDecoder(strings.NewReader(services))
+       if err := dec.Decode(&list); err != nil {
+               return err
+       }
+
+       return this.loadKeepServers(list)
+}
+
+// loadKeepServers
+func (this *KeepClient) loadKeepServers(list svcList) error {
        listed := make(map[string]bool)
        localRoots := make(map[string]string)
        gatewayRoots := make(map[string]string)
@@ -98,7 +118,7 @@ func (this *KeepClient) DiscoverKeepServers() error {
        this.replicasPerService = 1
        this.Using_proxy = false
 
-       for _, service := range m.Items {
+       for _, service := range list.Items {
                scheme := "http"
                if service.SSL {
                        scheme = "https"

diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index c9fd77a34dd14b2e3178df901cf7df5b9258baa5..3761cc7e9c9d154b4a5d061c5e98c85b2b0260e1 100644 (file)
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -120,22 +120,43 @@ var (
 
 // Initializes keep-rsync using the config provided
 func initializeKeepRsync() (err error) {
+       // arvSrc from srcConfig
        arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig)
        if err != nil {
                return
        }
 
+       // arvDst from dstConfig
        arvDst, err = arvadosclient.MakeArvadosClientWithConfig(dstConfig)
        if err != nil {
                return
        }
 
-       kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
-       if err != nil {
-               return
+       // if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+       if srcKeepServicesJSON == "" {
+               kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
+               if err != nil {
+                       return
+               }
+       } else {
+               kcSrc, err = keepclient.MakeKeepClientFromJSON(&arvSrc, srcKeepServicesJSON)
+               if err != nil {
+                       return
+               }
        }
 
-       kcDst, err = keepclient.MakeKeepClient(&arvDst)
+       // if dstKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+       if dstKeepServicesJSON == "" {
+               kcDst, err = keepclient.MakeKeepClient(&arvDst)
+               if err != nil {
+                       return
+               }
+       } else {
+               kcDst, err = keepclient.MakeKeepClientFromJSON(&arvDst, dstKeepServicesJSON)
+               if err != nil {
+                       return
+               }
+       }
 
        return
 }

diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index c45fbf83bbebc9de4feec40b397400f9c174aa15..7636c2e564aed4cab4b3c66b768d9c20da2cc504 100644 (file)
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -29,6 +29,8 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) {
 
 func (s *ServerRequiredSuite) SetUpTest(c *C) {
        arvadostest.ResetEnv()
+       srcKeepServicesJSON = ""
+       dstKeepServicesJSON = ""
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
@@ -115,3 +117,29 @@ func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDstShouldFail(c *C) {
        _, _, _, err = kcDst.Get(hash)
        c.Assert(err.Error(), Equals, "Block not found")
 }
+
+// Test keep-rsync initialization, with srcKeepServicesJSON
+func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
+       srcKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
+
+       setupRsync(c)
+
+       localRoots := kcSrc.LocalRoots()
+       c.Check(localRoots != nil, Equals, true)
+
+       foundIt := false
+       for k, _ := range localRoots {
+               if k == "zzzzz-bi6l4-123456789012340" {
+                       foundIt = true
+               }
+       }
+       c.Check(foundIt, Equals, true)
+
+       foundIt = false
+       for k, _ := range localRoots {
+               if k == "zzzzz-bi6l4-123456789012341" {
+                       foundIt = true
+               }
+       }
+       c.Check(foundIt, Equals, true)
+}