CollectionUUIDToIndex map[string]int
CollectionIndexToUUID []string
BlockToCollectionIndices map[blockdigest.DigestWithSize][]int
- Err error
}
// GetCollectionsParams params
}
// GetCollectionsAndSummarize gets collections from api and summarizes
-func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
- results, err := GetCollections(params)
+func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections, err error) {
+ results, err = GetCollections(params)
if err != nil {
- results.Err = err
return
}
dataFetcher = BuildDataFetcher(arv)
}
- dataFetcher(arvLogger, &readCollections, &keepServerInfo)
-
- if readCollections.Err != nil {
- return readCollections.Err
+ err = dataFetcher(arvLogger, &readCollections, &keepServerInfo)
+ if err != nil {
+ return err
}
err = summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
// BuildDataFetcher returns a data fetcher that fetches data from remote servers.
func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
- return func(arvLogger *logger.Logger,
+ return func(
+ arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
- keepServerInfo *keep.ReadServers) {
- collectionChannel := make(chan collection.ReadCollections)
-
+ keepServerInfo *keep.ReadServers,
+ ) error {
+ collDone := make(chan struct{})
+ var collErr error
go func() {
- collectionChannel <- collection.GetCollectionsAndSummarize(
+ *readCollections, collErr = collection.GetCollectionsAndSummarize(
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
BatchSize: 50})
+ collDone <- struct{}{}
}()
- var err error
- *keepServerInfo, err = keep.GetKeepServersAndSummarize(
+ var keepErr error
+ *keepServerInfo, keepErr = keep.GetKeepServersAndSummarize(
keep.GetKeepServersParams{
Client: arv,
Logger: arvLogger,
Limit: 1000})
- if err != nil {
- return
- }
+ <- collDone
- *readCollections = <-collectionChannel
+ // Return a nil error only if both parts succeeded.
+ if collErr != nil {
+ return collErr
+ }
+ return keepErr
}
}
// DataFetcher to fetch data from keep servers
type DataFetcher func(arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
- keepServerInfo *keep.ReadServers)
+ keepServerInfo *keep.ReadServers) error
func init() {
flag.StringVar(&WriteDataTo,
// working with stale data.
func ReadData(arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
- keepServerInfo *keep.ReadServers) {
+ keepServerInfo *keep.ReadServers) error {
if readDataFrom == "" {
- readCollections.Err = fmt.Errorf("ReadData() called with empty filename.")
- return
- } else {
- summaryFile, err := os.Open(readDataFrom)
- if err != nil {
- readCollections.Err = err
- return
- }
- defer summaryFile.Close()
+ return fmt.Errorf("ReadData() called with empty filename.")
+ }
+ summaryFile, err := os.Open(readDataFrom)
+ if err != nil {
+ return err
+ }
+ defer summaryFile.Close()
- dec := gob.NewDecoder(summaryFile)
- data := serializedData{}
- err = dec.Decode(&data)
- if err != nil {
- readCollections.Err = err
- return
- }
+ dec := gob.NewDecoder(summaryFile)
+ data := serializedData{}
+ err = dec.Decode(&data)
+ if err != nil {
+ return err
+ }
- // re-summarize data, so that we can update our summarizing
- // functions without needing to do all our network i/o
- data.ReadCollections.Summarize(arvLogger)
- data.KeepServerInfo.Summarize(arvLogger)
+ // re-summarize data, so that we can update our summarizing
+ // functions without needing to do all our network i/o
+ data.ReadCollections.Summarize(arvLogger)
+ data.KeepServerInfo.Summarize(arvLogger)
- *readCollections = data.ReadCollections
- *keepServerInfo = data.KeepServerInfo
- log.Printf("Read summary data from: %s", readDataFrom)
- }
+ *readCollections = data.ReadCollections
+ *keepServerInfo = data.KeepServerInfo
+ log.Printf("Read summary data from: %s", readDataFrom)
+ return nil
}