services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
  13         "log"
  14         "os"
  15 )
  16
  17 // Used to locally cache data read from servers to reduce execution
  18 // time when developing. Not for use in production.
  19 type serializedData struct {
  20         ReadCollections collection.ReadCollections
  21         KeepServerInfo  keep.ReadServers
  22 }
  23
  24 var (
  25         writeDataTo  string
  26         readDataFrom string
  27 )
  28
  29 func init() {
  30         flag.StringVar(&writeDataTo,
  31                 "write-data-to",
  32                 "",
  33                 "Write summary of data received to this file. Used for development only.")
  34         flag.StringVar(&readDataFrom,
  35                 "read-data-from",
  36                 "",
  37                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  38 }
  39
  40 // Writes data we've read to a file.
  41 //
  42 // This is useful for development, so that we don't need to read all
  43 // our data from the network every time we tweak something.
  44 //
  45 // This should not be used outside of development, since you'll be
  46 // working with stale data.
  47 func MaybeWriteData(arvLogger *logger.Logger,
  48         readCollections collection.ReadCollections,
  49         keepServerInfo keep.ReadServers) bool {
  50         if writeDataTo == "" {
  51                 return false
  52         } else {
  53                 summaryFile, err := os.Create(writeDataTo)
  54                 if err != nil {
  55                         loggerutil.FatalWithMessage(arvLogger,
  56                                 fmt.Sprintf("Failed to open %s: %v", writeDataTo, err))
  57                 }
  58                 defer summaryFile.Close()
  59
  60                 enc := gob.NewEncoder(summaryFile)
  61                 data := serializedData{
  62                         ReadCollections: readCollections,
  63                         KeepServerInfo:  keepServerInfo}
  64                 err = enc.Encode(data)
  65                 if err != nil {
  66                         loggerutil.FatalWithMessage(arvLogger,
  67                                 fmt.Sprintf("Failed to write summary data: %v", err))
  68                 }
  69                 log.Printf("Wrote summary data to: %s", writeDataTo)
  70                 return true
  71         }
  72 }
  73
  74 // Reads data that we've written to a file.
  75 //
  76 // This is useful for development, so that we don't need to read all
  77 // our data from the network every time we tweak something.
  78 //
  79 // This should not be used outside of development, since you'll be
  80 // working with stale data.
  81 func MaybeReadData(arvLogger *logger.Logger,
  82         readCollections *collection.ReadCollections,
  83         keepServerInfo *keep.ReadServers) bool {
  84         if readDataFrom == "" {
  85                 return false
  86         } else {
  87                 summaryFile, err := os.Open(readDataFrom)
  88                 if err != nil {
  89                         loggerutil.FatalWithMessage(arvLogger,
  90                                 fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
  91                 }
  92                 defer summaryFile.Close()
  93
  94                 dec := gob.NewDecoder(summaryFile)
  95                 data := serializedData{}
  96                 err = dec.Decode(&data)
  97                 if err != nil {
  98                         loggerutil.FatalWithMessage(arvLogger,
  99                                 fmt.Sprintf("Failed to read summary data: %v", err))
 100                 }
 101
 102                 // re-summarize data, so that we can update our summarizing
 103                 // functions without needing to do all our network i/o
 104                 data.ReadCollections.Summarize(arvLogger)
 105                 data.KeepServerInfo.Summarize(arvLogger)
 106
 107                 *readCollections = data.ReadCollections
 108                 *keepServerInfo = data.KeepServerInfo
 109                 log.Printf("Read summary data from: %s", readDataFrom)
 110                 return true
 111         }
 112 }