Merge branch '9567-keep-web-session-test'

[arvados.git] / services / datamanager / keep / keep.go

diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index ea28ffa59d2a8bce1a99ee047c524c429092fb57..651c869ef0780a91ce10c03ed9756b81f9a5ca6f 100644 (file)
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -118,7 +118,7 @@ func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServer
 // GetKeepServers from api server
 func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error) {
        sdkParams := arvadosclient.Dict{
-               "filters": [][]string{[]string{"service_type", "!=", "proxy"}},
+               "filters": [][]string{{"service_type", "!=", "proxy"}},
        }
        if params.Limit > 0 {
                sdkParams["limit"] = params.Limit
@@ -430,13 +430,23 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
                return
        }
 
-       blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
+       var ns int64
+       ns, err = strconv.ParseInt(tokens[1], 10, 64)
        if err != nil {
                return
        }
-       blockInfo.Digest =
-               blockdigest.DigestWithSize{Digest: locator.Digest,
-                       Size: uint32(locator.Size)}
+       if ns < 1e12 {
+               // An old version of keepstore is giving us timestamps
+               // in seconds instead of nanoseconds. (This threshold
+               // correctly handles all times between 1970-01-02 and
+               // 33658-09-27.)
+               ns = ns * 1e9
+       }
+       blockInfo.Mtime = ns
+       blockInfo.Digest = blockdigest.DigestWithSize{
+               Digest: locator.Digest,
+               Size:   uint32(locator.Size),
+       }
        return
 }
 
@@ -466,13 +476,29 @@ type TrashRequest struct {
 type TrashList []TrashRequest
 
 // SendTrashLists to trash queue
-func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
+func SendTrashLists(arvLogger *logger.Logger, kc *keepclient.KeepClient, spl map[string]TrashList, dryRun bool) (errs []error) {
        count := 0
        barrier := make(chan error)
 
        client := kc.Client
 
        for url, v := range spl {
+               if arvLogger != nil {
+                       // We need a local variable because Update doesn't call our mutator func until later,
+                       // when our list variable might have been reused by the next loop iteration.
+                       url := url
+                       trashLen := len(v)
+                       arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+                               trashListInfo := logger.GetOrCreateMap(p, "trash_list_len")
+                               trashListInfo[url] = trashLen
+                       })
+               }
+
+               if dryRun {
+                       log.Printf("dry run, not sending trash list to service %s with %d blocks", url, len(v))
+                       continue
+               }
+
                count++
                log.Printf("Sending trash list to %v", url)
 
@@ -512,7 +538,6 @@ func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs [
                                barrier <- nil
                        }
                })(url, v)
-
        }
 
        for i := 0; i < count; i++ {