services/datamanager/summary/file.go

   1 // Handles writing data to and reading data from disk to speed up development.
   2
   3 package summary
   4
   5 import (
   6         "encoding/gob"
   7         "flag"
   8         "fmt"
   9         "git.curoverse.com/arvados.git/sdk/go/logger"
  10         "git.curoverse.com/arvados.git/services/datamanager/collection"
  11         "git.curoverse.com/arvados.git/services/datamanager/keep"
  12         "log"
  13         "os"
  14 )
  15
  16 // Used to locally cache data read from servers to reduce execution
  17 // time when developing. Not for use in production.
  18 type serializedData struct {
  19         ReadCollections collection.ReadCollections
  20         KeepServerInfo  keep.ReadServers
  21 }
  22
  23 var (
  24         WriteDataTo  string
  25         readDataFrom string
  26 )
  27
  28 // DataFetcher to fetch data from keep servers
  29 type DataFetcher func(arvLogger *logger.Logger,
  30         readCollections *collection.ReadCollections,
  31         keepServerInfo *keep.ReadServers) error
  32
  33 func init() {
  34         flag.StringVar(&WriteDataTo,
  35                 "write-data-to",
  36                 "",
  37                 "Write summary of data received to this file. Used for development only.")
  38         flag.StringVar(&readDataFrom,
  39                 "read-data-from",
  40                 "",
  41                 "Avoid network i/o and read summary data from this file instead. Used for development only.")
  42 }
  43
  44 // MaybeWriteData writes data we've read to a file.
  45 //
  46 // This is useful for development, so that we don't need to read all
  47 // our data from the network every time we tweak something.
  48 //
  49 // This should not be used outside of development, since you'll be
  50 // working with stale data.
  51 func MaybeWriteData(arvLogger *logger.Logger,
  52         readCollections collection.ReadCollections,
  53         keepServerInfo keep.ReadServers) error {
  54         if WriteDataTo == "" {
  55                 return nil
  56         }
  57         summaryFile, err := os.Create(WriteDataTo)
  58         if err != nil {
  59                 return err
  60         }
  61         defer summaryFile.Close()
  62
  63         enc := gob.NewEncoder(summaryFile)
  64         data := serializedData{
  65                 ReadCollections: readCollections,
  66                 KeepServerInfo:  keepServerInfo}
  67         err = enc.Encode(data)
  68         if err != nil {
  69                 return err
  70         }
  71         log.Printf("Wrote summary data to: %s", WriteDataTo)
  72         return nil
  73 }
  74
  75 // ShouldReadData should not be used outside of development
  76 func ShouldReadData() bool {
  77         return readDataFrom != ""
  78 }
  79
  80 // ReadData reads data that we've written to a file.
  81 //
  82 // This is useful for development, so that we don't need to read all
  83 // our data from the network every time we tweak something.
  84 //
  85 // This should not be used outside of development, since you'll be
  86 // working with stale data.
  87 func ReadData(arvLogger *logger.Logger,
  88         readCollections *collection.ReadCollections,
  89         keepServerInfo *keep.ReadServers) error {
  90         if readDataFrom == "" {
  91                 return fmt.Errorf("ReadData() called with empty filename.")
  92         }
  93         summaryFile, err := os.Open(readDataFrom)
  94         if err != nil {
  95                 return err
  96         }
  97         defer summaryFile.Close()
  98
  99         dec := gob.NewDecoder(summaryFile)
 100         data := serializedData{}
 101         err = dec.Decode(&data)
 102         if err != nil {
 103                 return err
 104         }
 105
 106         // re-summarize data, so that we can update our summarizing
 107         // functions without needing to do all our network i/o
 108         data.ReadCollections.Summarize(arvLogger)
 109         data.KeepServerInfo.Summarize(arvLogger)
 110
 111         *readCollections = data.ReadCollections
 112         *keepServerInfo = data.KeepServerInfo
 113         log.Printf("Read summary data from: %s", readDataFrom)
 114         return nil
 115 }