1 // Handles writing data to and reading data from disk to speed up development.
9 "git.curoverse.com/arvados.git/sdk/go/logger"
10 "git.curoverse.com/arvados.git/services/datamanager/collection"
11 "git.curoverse.com/arvados.git/services/datamanager/keep"
16 // Used to locally cache data read from servers to reduce execution
17 // time when developing. Not for use in production.
18 type serializedData struct {
19 ReadCollections collection.ReadCollections
20 KeepServerInfo keep.ReadServers
28 // DataFetcher to fetch data from keep servers
29 type DataFetcher func(arvLogger *logger.Logger,
30 readCollections *collection.ReadCollections,
31 keepServerInfo *keep.ReadServers) error
34 flag.StringVar(&WriteDataTo,
37 "Write summary of data received to this file. Used for development only.")
38 flag.StringVar(&readDataFrom,
41 "Avoid network i/o and read summary data from this file instead. Used for development only.")
44 // MaybeWriteData writes data we've read to a file.
46 // This is useful for development, so that we don't need to read all
47 // our data from the network every time we tweak something.
49 // This should not be used outside of development, since you'll be
50 // working with stale data.
51 func MaybeWriteData(arvLogger *logger.Logger,
52 readCollections collection.ReadCollections,
53 keepServerInfo keep.ReadServers) error {
54 if WriteDataTo == "" {
57 summaryFile, err := os.Create(WriteDataTo)
61 defer summaryFile.Close()
63 enc := gob.NewEncoder(summaryFile)
64 data := serializedData{
65 ReadCollections: readCollections,
66 KeepServerInfo: keepServerInfo}
67 err = enc.Encode(data)
71 log.Printf("Wrote summary data to: %s", WriteDataTo)
75 // ShouldReadData should not be used outside of development
76 func ShouldReadData() bool {
77 return readDataFrom != ""
80 // ReadData reads data that we've written to a file.
82 // This is useful for development, so that we don't need to read all
83 // our data from the network every time we tweak something.
85 // This should not be used outside of development, since you'll be
86 // working with stale data.
87 func ReadData(arvLogger *logger.Logger,
88 readCollections *collection.ReadCollections,
89 keepServerInfo *keep.ReadServers) error {
90 if readDataFrom == "" {
91 return fmt.Errorf("ReadData() called with empty filename.")
93 summaryFile, err := os.Open(readDataFrom)
97 defer summaryFile.Close()
99 dec := gob.NewDecoder(summaryFile)
100 data := serializedData{}
101 err = dec.Decode(&data)
106 // re-summarize data, so that we can update our summarizing
107 // functions without needing to do all our network i/o
108 data.ReadCollections.Summarize(arvLogger)
109 data.KeepServerInfo.Summarize(arvLogger)
111 *readCollections = data.ReadCollections
112 *keepServerInfo = data.KeepServerInfo
113 log.Printf("Read summary data from: %s", readDataFrom)