services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
  13         "log"
  14         "os"
  15 )
  16
  17 // Used to locally cache data read from servers to reduce execution
  18 // time when developing. Not for use in production.
  19 type serializedData struct {
  20         ReadCollections collection.ReadCollections
  21         KeepServerInfo  keep.ReadServers
  22 }
  23
  24 var (
  25         writeDataTo  string
  26         readDataFrom string
  27 )
  28
  29 type DataFetcher func(arvLogger *logger.Logger,
  30         readCollections *collection.ReadCollections,
  31         keepServerInfo *keep.ReadServers)
  32
  33 func init() {
  34         flag.StringVar(&writeDataTo,
  35                 "write-data-to",
  36                 "",
  37                 "Write summary of data received to this file. Used for development only.")
  38         flag.StringVar(&readDataFrom,
  39                 "read-data-from",
  40                 "",
  41                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  42 }
  43
  44 // Writes data we've read to a file.
  45 //
  46 // This is useful for development, so that we don't need to read all
  47 // our data from the network every time we tweak something.
  48 //
  49 // This should not be used outside of development, since you'll be
  50 // working with stale data.
  51 func MaybeWriteData(arvLogger *logger.Logger,
  52         readCollections collection.ReadCollections,
  53         keepServerInfo keep.ReadServers) bool {
  54         if writeDataTo == "" {
  55                 return false
  56         } else {
  57                 summaryFile, err := os.Create(writeDataTo)
  58                 if err != nil {
  59                         loggerutil.FatalWithMessage(arvLogger,
  60                                 fmt.Sprintf("Failed to open %s: %v", writeDataTo, err))
  61                 }
  62                 defer summaryFile.Close()
  63
  64                 enc := gob.NewEncoder(summaryFile)
  65                 data := serializedData{
  66                         ReadCollections: readCollections,
  67                         KeepServerInfo:  keepServerInfo}
  68                 err = enc.Encode(data)
  69                 if err != nil {
  70                         loggerutil.FatalWithMessage(arvLogger,
  71                                 fmt.Sprintf("Failed to write summary data: %v", err))
  72                 }
  73                 log.Printf("Wrote summary data to: %s", writeDataTo)
  74                 return true
  75         }
  76 }
  77
  78 func ShouldReadData() bool {
  79         return readDataFrom != ""
  80 }
  81
  82 // Reads data that we've written to a file.
  83 //
  84 // This is useful for development, so that we don't need to read all
  85 // our data from the network every time we tweak something.
  86 //
  87 // This should not be used outside of development, since you'll be
  88 // working with stale data.
  89 func ReadData(arvLogger *logger.Logger,
  90         readCollections *collection.ReadCollections,
  91         keepServerInfo *keep.ReadServers) {
  92         if readDataFrom == "" {
  93                 loggerutil.FatalWithMessage(arvLogger,
  94                         "ReadData() called with empty filename.")
  95         } else {
  96                 summaryFile, err := os.Open(readDataFrom)
  97                 if err != nil {
  98                         loggerutil.FatalWithMessage(arvLogger,
  99                                 fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
 100                 }
 101                 defer summaryFile.Close()
 102
 103                 dec := gob.NewDecoder(summaryFile)
 104                 data := serializedData{}
 105                 err = dec.Decode(&data)
 106                 if err != nil {
 107                         loggerutil.FatalWithMessage(arvLogger,
 108                                 fmt.Sprintf("Failed to read summary data: %v", err))
 109                 }
 110
 111                 // re-summarize data, so that we can update our summarizing
 112                 // functions without needing to do all our network i/o
 113                 data.ReadCollections.Summarize(arvLogger)
 114                 data.KeepServerInfo.Summarize(arvLogger)
 115
 116                 *readCollections = data.ReadCollections
 117                 *keepServerInfo = data.KeepServerInfo
 118                 log.Printf("Read summary data from: %s", readDataFrom)
 119         }
 120 }