services/datamanager/summary/trash_list.go

   1 // Code for generating trash lists
   2 package summary
   3
   4 import (
   5         "encoding/json"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   8         "git.curoverse.com/arvados.git/sdk/go/logger"
   9         "git.curoverse.com/arvados.git/services/datamanager/keep"
  10         "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
  11         "log"
  12         "os"
  13         "strings"
  14         "time"
  15 )
  16
  17 func BuildTrashLists(kc *keepclient.KeepClient,
  18         keepServerInfo *keep.ReadServers,
  19         keepBlocksNotInCollections BlockSet) (m map[string]keep.TrashList) {
  20
  21         // Servers that are writeable
  22         writableServers := map[string]struct{}{}
  23         for _, url := range kc.WritableLocalRoots() {
  24                 writableServers[url] = struct{}{}
  25         }
  26
  27         m = make(map[string]keep.TrashList)
  28
  29         _ttl, err := kc.Arvados.Discovery("blobSignatureTtl")
  30         if err != nil {
  31                 log.Printf("Failed to get blobSignatureTtl: %v", err)
  32                 return
  33         }
  34
  35         ttl := int64(_ttl.(float64))
  36
  37         // expire unreferenced blocks more than "ttl" seconds old.
  38         expiry := time.Now().UTC().Unix() - ttl
  39
  40         for block, _ := range keepBlocksNotInCollections {
  41                 for _, block_on_server := range keepServerInfo.BlockToServers[block] {
  42                         if block_on_server.Mtime < expiry {
  43                                 // block is older than expire cutoff
  44                                 srv := keepServerInfo.KeepServerIndexToAddress[block_on_server.ServerIndex].String()
  45
  46                                 _, writable := writableServers[srv]
  47
  48                                 if writable {
  49                                         m[srv] = append(m[srv], keep.TrashRequest{Locator: block.Digest.String(), BlockMtime: block_on_server.Mtime})
  50                                 }
  51                         }
  52                 }
  53         }
  54         return
  55 }
  56
  57 // Writes each pull list to a file.
  58 // The filename is based on the hostname.
  59 //
  60 // This is just a hack for prototyping, it is not expected to be used
  61 // in production.
  62 func WriteTrashLists(arvLogger *logger.Logger,
  63         trashLists map[string]keep.TrashList) {
  64         r := strings.NewReplacer(":", ".")
  65         for host, list := range trashLists {
  66                 filename := fmt.Sprintf("trash_list.%s", r.Replace(RemoveProtocolPrefix(host)))
  67                 trashListFile, err := os.Create(filename)
  68                 if err != nil {
  69                         loggerutil.FatalWithMessage(arvLogger,
  70                                 fmt.Sprintf("Failed to open %s: %v", filename, err))
  71                 }
  72                 defer trashListFile.Close()
  73
  74                 enc := json.NewEncoder(trashListFile)
  75                 err = enc.Encode(list)
  76                 if err != nil {
  77                         loggerutil.FatalWithMessage(arvLogger,
  78                                 fmt.Sprintf("Failed to write trash list to %s: %v", filename, err))
  79                 }
  80                 log.Printf("Wrote trash list to %s.", filename)
  81         }
  82 }