services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
  13         "log"
  14         "os"
  15 )
  16
  17 // Used to locally cache data read from servers to reduce execution
  18 // time when developing. Not for use in production.
  19 type serializedData struct {
  20         ReadCollections collection.ReadCollections
  21         KeepServerInfo  keep.ReadServers
  22 }
  23
  24 var (
  25         writeDataTo  string
  26         readDataFrom string
  27 )
  28
  29 // DataFetcher to fetch data from keep servers
  30 type DataFetcher func(arvLogger *logger.Logger,
  31         readCollections *collection.ReadCollections,
  32         keepServerInfo *keep.ReadServers)
  33
  34 func init() {
  35         flag.StringVar(&writeDataTo,
  36                 "write-data-to",
  37                 "",
  38                 "Write summary of data received to this file. Used for development only.")
  39         flag.StringVar(&readDataFrom,
  40                 "read-data-from",
  41                 "",
  42                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  43 }
  44
  45 // MaybeWriteData writes data we've read to a file.
  46 //
  47 // This is useful for development, so that we don't need to read all
  48 // our data from the network every time we tweak something.
  49 //
  50 // This should not be used outside of development, since you'll be
  51 // working with stale data.
  52 func MaybeWriteData(arvLogger *logger.Logger,
  53         readCollections collection.ReadCollections,
  54         keepServerInfo keep.ReadServers) bool {
  55         if writeDataTo == "" {
  56                 return false
  57         }
  58         summaryFile, err := os.Create(writeDataTo)
  59         if err != nil {
  60                 loggerutil.FatalWithMessage(arvLogger,
  61                         fmt.Sprintf("Failed to open %s: %v", writeDataTo, err))
  62         }
  63         defer summaryFile.Close()
  64
  65         enc := gob.NewEncoder(summaryFile)
  66         data := serializedData{
  67                 ReadCollections: readCollections,
  68                 KeepServerInfo:  keepServerInfo}
  69         err = enc.Encode(data)
  70         if err != nil {
  71                 loggerutil.FatalWithMessage(arvLogger,
  72                         fmt.Sprintf("Failed to write summary data: %v", err))
  73         }
  74         log.Printf("Wrote summary data to: %s", writeDataTo)
  75         return true
  76 }
  77
  78 // ShouldReadData should not be used outside of development
  79 func ShouldReadData() bool {
  80         return readDataFrom != ""
  81 }
  82
  83 // ReadData reads data that we've written to a file.
  84 //
  85 // This is useful for development, so that we don't need to read all
  86 // our data from the network every time we tweak something.
  87 //
  88 // This should not be used outside of development, since you'll be
  89 // working with stale data.
  90 func ReadData(arvLogger *logger.Logger,
  91         readCollections *collection.ReadCollections,
  92         keepServerInfo *keep.ReadServers) {
  93         if readDataFrom == "" {
  94                 loggerutil.FatalWithMessage(arvLogger,
  95                         "ReadData() called with empty filename.")
  96         } else {
  97                 summaryFile, err := os.Open(readDataFrom)
  98                 if err != nil {
  99                         loggerutil.FatalWithMessage(arvLogger,
 100                                 fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
 101                 }
 102                 defer summaryFile.Close()
 103
 104                 dec := gob.NewDecoder(summaryFile)
 105                 data := serializedData{}
 106                 err = dec.Decode(&data)
 107                 if err != nil {
 108                         loggerutil.FatalWithMessage(arvLogger,
 109                                 fmt.Sprintf("Failed to read summary data: %v", err))
 110                 }
 111
 112                 // re-summarize data, so that we can update our summarizing
 113                 // functions without needing to do all our network i/o
 114                 data.ReadCollections.Summarize(arvLogger)
 115                 data.KeepServerInfo.Summarize(arvLogger)
 116
 117                 *readCollections = data.ReadCollections
 118                 *keepServerInfo = data.KeepServerInfo
 119                 log.Printf("Read summary data from: %s", readDataFrom)
 120         }
 121 }