9187: Slurm dispatcher improvements around squeue

[arvados.git] / services / datamanager / summary / trash_list.go

diff --git a/services/datamanager/summary/trash_list.go b/services/datamanager/summary/trash_list.go
index efb40e25a7d20e0ef84563d2367b96c7d7bfdd7c..b6ceacecde2b8e2ffe810deea9e3777aade06625 100644 (file)
--- a/services/datamanager/summary/trash_list.go
+++ b/services/datamanager/summary/trash_list.go
@@ -1,22 +1,19 @@
 // Code for generating trash lists
+
 package summary
 
 import (
-       "encoding/json"
+       "errors"
        "fmt"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
-       "git.curoverse.com/arvados.git/sdk/go/logger"
        "git.curoverse.com/arvados.git/services/datamanager/keep"
-       "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
-       "log"
-       "os"
-       "strings"
        "time"
 )
 
+// BuildTrashLists builds list of blocks to be sent to trash queue
 func BuildTrashLists(kc *keepclient.KeepClient,
        keepServerInfo *keep.ReadServers,
-       keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList) {
+       keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList, err error) {
 
        // Servers that are writeable
        writableServers := map[string]struct{}{}
@@ -24,12 +21,9 @@ func BuildTrashLists(kc *keepclient.KeepClient,
                writableServers[url] = struct{}{}
        }
 
-       m = make(map[string]keep.TrashList)
-
        _ttl, err := kc.Arvados.Discovery("blobSignatureTtl")
        if err != nil {
-               log.Printf("Failed to get blobSignatureTtl: %v", err)
-               return
+               return nil, errors.New(fmt.Sprintf("Failed to get blobSignatureTtl, can't build trash lists: %v", err))
        }
 
        ttl := int64(_ttl.(float64))
@@ -37,46 +31,32 @@ func BuildTrashLists(kc *keepclient.KeepClient,
        // expire unreferenced blocks more than "ttl" seconds old.
        expiry := time.Now().UTC().Unix() - ttl
 
-       for block, _ := range keepBlocksNotInCollections {
-               for _, block_on_server := range keepServerInfo.BlockToServers[block] {
-                       if block_on_server.Mtime < expiry {
-                               // block is older than expire cutoff
-                               srv := keepServerInfo.KeepServerIndexToAddress[block_on_server.ServerIndex].String()
+       return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil
+}
+
+func buildTrashListsInternal(writableServers map[string]struct{},
+       keepServerInfo *keep.ReadServers,
+       expiry int64,
+       keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList) {
+
+       m = make(map[string]keep.TrashList)
+
+       for block := range keepBlocksNotInCollections {
+               for _, blockOnServer := range keepServerInfo.BlockToServers[block] {
+                       if blockOnServer.Mtime >= expiry {
+                               continue
+                       }
 
-                               _, writable := writableServers[srv]
+                       // block is older than expire cutoff
+                       srv := keepServerInfo.KeepServerIndexToAddress[blockOnServer.ServerIndex].String()
 
-                               if writable {
-                                       m[srv] = append(m[srv], keep.TrashRequest{Locator: block.Digest.String(), BlockMtime: block_on_server.Mtime})
-                               }
+                       if _, writable := writableServers[srv]; !writable {
+                               continue
                        }
+
+                       m[srv] = append(m[srv], keep.TrashRequest{Locator: block.Digest.String(), BlockMtime: blockOnServer.Mtime})
                }
        }
        return
-}
 
-// Writes each pull list to a file.
-// The filename is based on the hostname.
-//
-// This is just a hack for prototyping, it is not expected to be used
-// in production.
-func WriteTrashLists(arvLogger *logger.Logger,
-       trashLists map[string]keep.TrashList) {
-       r := strings.NewReplacer(":", ".")
-       for host, list := range trashLists {
-               filename := fmt.Sprintf("trash_list.%s", r.Replace(RemoveProtocolPrefix(host)))
-               trashListFile, err := os.Create(filename)
-               if err != nil {
-                       loggerutil.FatalWithMessage(arvLogger,
-                               fmt.Sprintf("Failed to open %s: %v", filename, err))
-               }
-               defer trashListFile.Close()
-
-               enc := json.NewEncoder(trashListFile)
-               err = enc.Encode(list)
-               if err != nil {
-                       loggerutil.FatalWithMessage(arvLogger,
-                               fmt.Sprintf("Failed to write trash list to %s: %v", filename, err))
-               }
-               log.Printf("Wrote trash list to %s.", filename)
-       }
 }