X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e2197875b3fa58b235268a86170fec582c1a7f59..0d2a561091b777dfb7735d842a88f8529f4f982a:/services/datamanager/keep/keep.go

diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index ea28ffa59d..39d2d5bd5d 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -66,7 +66,7 @@ type ReadServers struct {
 
 // GetKeepServersParams struct
 type GetKeepServersParams struct {
-	Client arvadosclient.ArvadosClient
+	Client *arvadosclient.ArvadosClient
 	Logger *logger.Logger
 	Limit  int
 }
@@ -118,7 +118,7 @@ func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServer
 // GetKeepServers from api server
 func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error) {
 	sdkParams := arvadosclient.Dict{
-		"filters": [][]string{[]string{"service_type", "!=", "proxy"}},
+		"filters": [][]string{{"service_type", "!=", "proxy"}},
 	}
 	if params.Limit > 0 {
 		sdkParams["limit"] = params.Limit
@@ -215,7 +215,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
 // GetServerContents of the keep server
 func GetServerContents(arvLogger *logger.Logger,
 	keepServer ServerAddress,
-	arv arvadosclient.ArvadosClient) (response ServerResponse) {
+	arv *arvadosclient.ArvadosClient) (response ServerResponse) {
 
 	err := GetServerStatus(arvLogger, keepServer, arv)
 	if err != nil {
@@ -247,7 +247,7 @@ func GetServerContents(arvLogger *logger.Logger,
 // GetServerStatus get keep server status by invoking /status.json
 func GetServerStatus(arvLogger *logger.Logger,
 	keepServer ServerAddress,
-	arv arvadosclient.ArvadosClient) error {
+	arv *arvadosclient.ArvadosClient) error {
 	url := fmt.Sprintf("http://%s:%d/status.json",
 		keepServer.Host,
 		keepServer.Port)
@@ -298,7 +298,7 @@ func GetServerStatus(arvLogger *logger.Logger,
 // CreateIndexRequest to the keep server
 func CreateIndexRequest(arvLogger *logger.Logger,
 	keepServer ServerAddress,
-	arv arvadosclient.ArvadosClient) (req *http.Request, err error) {
+	arv *arvadosclient.ArvadosClient) (req *http.Request, err error) {
 	url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
 	log.Println("About to fetch keep server contents from " + url)
 
@@ -430,13 +430,23 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
 		return
 	}
 
-	blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
+	var ns int64
+	ns, err = strconv.ParseInt(tokens[1], 10, 64)
 	if err != nil {
 		return
 	}
-	blockInfo.Digest =
-		blockdigest.DigestWithSize{Digest: locator.Digest,
-			Size: uint32(locator.Size)}
+	if ns < 1e12 {
+		// An old version of keepstore is giving us timestamps
+		// in seconds instead of nanoseconds. (This threshold
+		// correctly handles all times between 1970-01-02 and
+		// 33658-09-27.)
+		ns = ns * 1e9
+	}
+	blockInfo.Mtime = ns
+	blockInfo.Digest = blockdigest.DigestWithSize{
+		Digest: locator.Digest,
+		Size:   uint32(locator.Size),
+	}
 	return
 }
 
@@ -466,13 +476,29 @@ type TrashRequest struct {
 type TrashList []TrashRequest
 
 // SendTrashLists to trash queue
-func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
+func SendTrashLists(arvLogger *logger.Logger, kc *keepclient.KeepClient, spl map[string]TrashList, dryRun bool) (errs []error) {
 	count := 0
 	barrier := make(chan error)
 
 	client := kc.Client
 
 	for url, v := range spl {
+		if arvLogger != nil {
+			// We need a local variable because Update doesn't call our mutator func until later,
+			// when our list variable might have been reused by the next loop iteration.
+			url := url
+			trashLen := len(v)
+			arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+				trashListInfo := logger.GetOrCreateMap(p, "trash_list_len")
+				trashListInfo[url] = trashLen
+			})
+		}
+
+		if dryRun {
+			log.Printf("dry run, not sending trash list to service %s with %d blocks", url, len(v))
+			continue
+		}
+
 		count++
 		log.Printf("Sending trash list to %v", url)
 
@@ -512,7 +538,6 @@ func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs [
 				barrier <- nil
 			}
 		})(url, v)
-
 	}
 
 	for i := 0; i < count; i++ {