1 // Handles writing data to and reading data from disk to speed up development.
9 "git.curoverse.com/arvados.git/sdk/go/logger"
10 "git.curoverse.com/arvados.git/services/datamanager/collection"
11 "git.curoverse.com/arvados.git/services/datamanager/keep"
12 "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
17 // Used to locally cache data read from servers to reduce execution
18 // time when developing. Not for use in production.
19 type serializedData struct {
20 ReadCollections collection.ReadCollections
21 KeepServerInfo keep.ReadServers
29 type DataFetcher func(arvLogger *logger.Logger,
30 readCollections *collection.ReadCollections,
31 keepServerInfo *keep.ReadServers)
34 flag.StringVar(&writeDataTo,
37 "Write summary of data received to this file. Used for development only.")
38 flag.StringVar(&readDataFrom,
41 "Avoid network i/o and read summary data from this file instead. Used for development only.")
44 // Writes data we've read to a file.
46 // This is useful for development, so that we don't need to read all
47 // our data from the network every time we tweak something.
49 // This should not be used outside of development, since you'll be
50 // working with stale data.
51 func MaybeWriteData(arvLogger *logger.Logger,
52 readCollections collection.ReadCollections,
53 keepServerInfo keep.ReadServers) bool {
54 if writeDataTo == "" {
57 summaryFile, err := os.Create(writeDataTo)
59 loggerutil.FatalWithMessage(arvLogger,
60 fmt.Sprintf("Failed to open %s: %v", writeDataTo, err))
62 defer summaryFile.Close()
64 enc := gob.NewEncoder(summaryFile)
65 data := serializedData{
66 ReadCollections: readCollections,
67 KeepServerInfo: keepServerInfo}
68 err = enc.Encode(data)
70 loggerutil.FatalWithMessage(arvLogger,
71 fmt.Sprintf("Failed to write summary data: %v", err))
73 log.Printf("Wrote summary data to: %s", writeDataTo)
78 func ShouldReadData() bool {
79 return readDataFrom != ""
82 // Reads data that we've written to a file.
84 // This is useful for development, so that we don't need to read all
85 // our data from the network every time we tweak something.
87 // This should not be used outside of development, since you'll be
88 // working with stale data.
89 func ReadData(arvLogger *logger.Logger,
90 readCollections *collection.ReadCollections,
91 keepServerInfo *keep.ReadServers) {
92 if readDataFrom == "" {
93 loggerutil.FatalWithMessage(arvLogger,
94 "ReadData() called with empty filename.")
96 summaryFile, err := os.Open(readDataFrom)
98 loggerutil.FatalWithMessage(arvLogger,
99 fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
101 defer summaryFile.Close()
103 dec := gob.NewDecoder(summaryFile)
104 data := serializedData{}
105 err = dec.Decode(&data)
107 loggerutil.FatalWithMessage(arvLogger,
108 fmt.Sprintf("Failed to read summary data: %v", err))
111 // re-summarize data, so that we can update our summarizing
112 // functions without needing to do all our network i/o
113 data.ReadCollections.Summarize(arvLogger)
114 data.KeepServerInfo.Summarize(arvLogger)
116 *readCollections = data.ReadCollections
117 *keepServerInfo = data.KeepServerInfo
118 log.Printf("Read summary data from: %s", readDataFrom)