services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
  13         "log"
  14         "os"
  15 )
  16
  17 // Used to locally cache data read from servers to reduce execution
  18 // time when developing. Not for use in production.
  19 type serializedData struct {
  20         ReadCollections collection.ReadCollections
  21         KeepServerInfo  keep.ReadServers
  22 }
  23
  24 var (
  25         WriteDataTo  string
  26         readDataFrom string
  27 )
  28
  29 // DataFetcher to fetch data from keep servers
  30 type DataFetcher func(arvLogger *logger.Logger,
  31         readCollections *collection.ReadCollections,
  32         keepServerInfo *keep.ReadServers)
  33
  34 func init() {
  35         flag.StringVar(&WriteDataTo,
  36                 "write-data-to",
  37                 "",
  38                 "Write summary of data received to this file. Used for development only.")
  39         flag.StringVar(&readDataFrom,
  40                 "read-data-from",
  41                 "",
  42                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  43 }
  44
  45 // MaybeWriteData writes data we've read to a file.
  46 //
  47 // This is useful for development, so that we don't need to read all
  48 // our data from the network every time we tweak something.
  49 //
  50 // This should not be used outside of development, since you'll be
  51 // working with stale data.
  52 func MaybeWriteData(arvLogger *logger.Logger,
  53         readCollections collection.ReadCollections,
  54         keepServerInfo keep.ReadServers) error {
  55         if WriteDataTo == "" {
  56                 return nil
  57         }
  58         summaryFile, err := os.Create(WriteDataTo)
  59         if err != nil {
  60                 return err
  61         }
  62         defer summaryFile.Close()
  63
  64         enc := gob.NewEncoder(summaryFile)
  65         data := serializedData{
  66                 ReadCollections: readCollections,
  67                 KeepServerInfo:  keepServerInfo}
  68         err = enc.Encode(data)
  69         if err != nil {
  70                 return err
  71         }
  72         log.Printf("Wrote summary data to: %s", WriteDataTo)
  73         return nil
  74 }
  75
  76 // ShouldReadData should not be used outside of development
  77 func ShouldReadData() bool {
  78         return readDataFrom != ""
  79 }
  80
  81 // ReadData reads data that we've written to a file.
  82 //
  83 // This is useful for development, so that we don't need to read all
  84 // our data from the network every time we tweak something.
  85 //
  86 // This should not be used outside of development, since you'll be
  87 // working with stale data.
  88 func ReadData(arvLogger *logger.Logger,
  89         readCollections *collection.ReadCollections,
  90         keepServerInfo *keep.ReadServers) {
  91         if readDataFrom == "" {
  92                 loggerutil.FatalWithMessage(arvLogger,
  93                         "ReadData() called with empty filename.")
  94         } else {
  95                 summaryFile, err := os.Open(readDataFrom)
  96                 if err != nil {
  97                         loggerutil.FatalWithMessage(arvLogger,
  98                                 fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
  99                 }
 100                 defer summaryFile.Close()
 101
 102                 dec := gob.NewDecoder(summaryFile)
 103                 data := serializedData{}
 104                 err = dec.Decode(&data)
 105                 if err != nil {
 106                         loggerutil.FatalWithMessage(arvLogger,
 107                                 fmt.Sprintf("Failed to read summary data: %v", err))
 108                 }
 109
 110                 // re-summarize data, so that we can update our summarizing
 111                 // functions without needing to do all our network i/o
 112                 data.ReadCollections.Summarize(arvLogger)
 113                 data.KeepServerInfo.Summarize(arvLogger)
 114
 115                 *readCollections = data.ReadCollections
 116                 *keepServerInfo = data.KeepServerInfo
 117                 log.Printf("Read summary data from: %s", readDataFrom)
 118         }
 119 }