X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bb663993cc851971d9444d834acfc416363931b2..e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98:/sdk/go/keepclient/support.go

diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 592d77e3c5..f5554618fe 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -2,6 +2,7 @@ package keepclient
 
 import (
 	"crypto/md5"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/streamer"
@@ -14,7 +15,7 @@ import (
 	"time"
 )
 
-type keepServices struct {
+type keepService struct {
 	Uuid     string `json:"uuid"`
 	Hostname string `json:"service_host"`
 	Port     int    `json:"service_port"`
@@ -76,29 +77,48 @@ func (this *KeepClient) setClientSettingsDisk() {
 	}
 }
 
+type svcList struct {
+	Items []keepService `json:"items"`
+}
+
 // DiscoverKeepServers gets list of available keep services from api server
 func (this *KeepClient) DiscoverKeepServers() error {
-	type svcList struct {
-		Items []keepServices `json:"items"`
-	}
-	var m svcList
+	var list svcList
 
 	// Get keep services from api server
-	err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &m)
-
-	// If there is error getting keep services, get list of keep disks
+	err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &list)
 	if err != nil {
-		if err := this.Arvados.List("keep_disks", nil, &m); err != nil {
-			return err
-		}
+		return err
+	}
+
+	return this.loadKeepServers(list)
+}
+
+// LoadKeepServicesFromJSON gets list of available keep services from given JSON
+func (this *KeepClient) LoadKeepServicesFromJSON(services string) error {
+	var list svcList
+
+	// Load keep services from given json
+	dec := json.NewDecoder(strings.NewReader(services))
+	if err := dec.Decode(&list); err != nil {
+		return err
 	}
 
+	return this.loadKeepServers(list)
+}
+
+// loadKeepServers
+func (this *KeepClient) loadKeepServers(list svcList) error {
 	listed := make(map[string]bool)
 	localRoots := make(map[string]string)
 	gatewayRoots := make(map[string]string)
 	writableLocalRoots := make(map[string]string)
 
-	for _, service := range m.Items {
+	// replicasPerService is 1 for disks; unknown or unlimited otherwise
+	this.replicasPerService = 1
+	this.Using_proxy = false
+
+	for _, service := range list.Items {
 		scheme := "http"
 		if service.SSL {
 			scheme = "https"
@@ -111,16 +131,16 @@ func (this *KeepClient) DiscoverKeepServers() error {
 		}
 		listed[url] = true
 
-		switch service.SvcType {
-		case "disk":
-			localRoots[service.Uuid] = url
-		default:
-			localRoots[service.Uuid] = url
+		localRoots[service.Uuid] = url
+		if service.SvcType == "proxy" {
 			this.Using_proxy = true
 		}
 
 		if service.ReadOnly == false {
 			writableLocalRoots[service.Uuid] = url
+			if service.SvcType != "disk" {
+				this.replicasPerService = 0
+			}
 		}
 
 		// Gateway services are only used when specified by
@@ -149,7 +169,7 @@ type uploadStatus struct {
 	response        string
 }
 
-func (this KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
+func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
 	upload_status chan<- uploadStatus, expectedLength int64, requestId string) {
 
 	var req *http.Request
@@ -177,10 +197,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
 
 	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
 	req.Header.Add("Content-Type", "application/octet-stream")
-
-	if this.Using_proxy {
-		req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
-	}
+	req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
 
 	var resp *http.Response
 	if resp, err = this.Client.Do(req); err != nil {
@@ -197,7 +214,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
 	defer resp.Body.Close()
 	defer io.Copy(ioutil.Discard, resp.Body)
 
-	respbody, err2 := ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+	respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
 	response := strings.TrimSpace(string(respbody))
 	if err2 != nil && err2 != io.EOF {
 		log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, err2.Error(), response)
@@ -211,7 +228,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
 	}
 }
 
-func (this KeepClient) putReplicas(
+func (this *KeepClient) putReplicas(
 	hash string,
 	tr *streamer.AsyncStream,
 	expectedLength int64) (locator string, replicas int, err error) {
@@ -231,13 +248,29 @@ func (this KeepClient) putReplicas(
 
 	// Used to communicate status from the upload goroutines
 	upload_status := make(chan uploadStatus)
-	defer close(upload_status)
+	defer func() {
+		// Wait for any abandoned uploads (e.g., we started
+		// two uploads and the first replied with replicas=2)
+		// to finish before closing the status channel.
+		go func() {
+			for active > 0 {
+				<-upload_status
+			}
+			close(upload_status)
+		}()
+	}()
 
 	// Desired number of replicas
 	remaining_replicas := this.Want_replicas
 
+	replicasPerThread := this.replicasPerService
+	if replicasPerThread < 1 {
+		// unlimited or unknown
+		replicasPerThread = remaining_replicas
+	}
+
 	for remaining_replicas > 0 {
-		for active < remaining_replicas {
+		for active*replicasPerThread < remaining_replicas {
 			// Start some upload requests
 			if next_server < len(sv) {
 				log.Printf("[%v] Begin upload %s to %s", requestId, hash, sv[next_server])