services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "log"
  13         "os"
  14 )
  15
  16 // Used to locally cache data read from servers to reduce execution
  17 // time when developing. Not for use in production.
  18 type serializedData struct {
  19         ReadCollections collection.ReadCollections
  20         KeepServerInfo  keep.ReadServers
  21 }
  22
  23 var (
  24         WriteDataTo  string
  25         readDataFrom string
  26 )
  27
  28 // DataFetcher to fetch data from keep servers
  29 type DataFetcher func(arvLogger *logger.Logger,
  30         readCollections *collection.ReadCollections,
  31         keepServerInfo *keep.ReadServers)
  32
  33 func init() {
  34         flag.StringVar(&WriteDataTo,
  35                 "write-data-to",
  36                 "",
  37                 "Write summary of data received to this file. Used for development only.")
  38         flag.StringVar(&readDataFrom,
  39                 "read-data-from",
  40                 "",
  41                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  42 }
  43
  44 // MaybeWriteData writes data we've read to a file.
  45 //
  46 // This is useful for development, so that we don't need to read all
  47 // our data from the network every time we tweak something.
  48 //
  49 // This should not be used outside of development, since you'll be
  50 // working with stale data.
  51 func MaybeWriteData(arvLogger *logger.Logger,
  52         readCollections collection.ReadCollections,
  53         keepServerInfo keep.ReadServers) error {
  54         if WriteDataTo == "" {
  55                 return nil
  56         }
  57         summaryFile, err := os.Create(WriteDataTo)
  58         if err != nil {
  59                 return err
  60         }
  61         defer summaryFile.Close()
  62
  63         enc := gob.NewEncoder(summaryFile)
  64         data := serializedData{
  65                 ReadCollections: readCollections,
  66                 KeepServerInfo:  keepServerInfo}
  67         err = enc.Encode(data)
  68         if err != nil {
  69                 return err
  70         }
  71         log.Printf("Wrote summary data to: %s", WriteDataTo)
  72         return nil
  73 }
  74
  75 // ShouldReadData should not be used outside of development
  76 func ShouldReadData() bool {
  77         return readDataFrom != ""
  78 }
  79
  80 // ReadData reads data that we've written to a file.
  81 //
  82 // This is useful for development, so that we don't need to read all
  83 // our data from the network every time we tweak something.
  84 //
  85 // This should not be used outside of development, since you'll be
  86 // working with stale data.
  87 func ReadData(arvLogger *logger.Logger,
  88         readCollections *collection.ReadCollections,
  89         keepServerInfo *keep.ReadServers) {
  90         if readDataFrom == "" {
  91                 readCollections.Err = fmt.Errorf("ReadData() called with empty filename.")
  92                 return
  93         } else {
  94                 summaryFile, err := os.Open(readDataFrom)
  95                 if err != nil {
  96                         readCollections.Err = err
  97                         return
  98                 }
  99                 defer summaryFile.Close()
 100
 101                 dec := gob.NewDecoder(summaryFile)
 102                 data := serializedData{}
 103                 err = dec.Decode(&data)
 104                 if err != nil {
 105                         readCollections.Err = err
 106                         return
 107                 }
 108
 109                 // re-summarize data, so that we can update our summarizing
 110                 // functions without needing to do all our network i/o
 111                 data.ReadCollections.Summarize(arvLogger)
 112                 data.KeepServerInfo.Summarize(arvLogger)
 113
 114                 *readCollections = data.ReadCollections
 115                 *keepServerInfo = data.KeepServerInfo
 116                 log.Printf("Read summary data from: %s", readDataFrom)
 117         }
 118 }