+ if len(keepServers) == 0 {
+ return results, fmt.Errorf("Found no keepservices with the service type %v", serviceType)
+ }
+
+ if params.Logger != nil {
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := logger.GetOrCreateMap(p, "keep_info")
+ keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
+ keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
+ keepInfo["keep_servers"] = sdkResponse.KeepServers
+ keepInfo["indexable_keep_servers"] = keepServers
+ })
+ }
+
+ log.Printf("Received keep services list: %+v", sdkResponse)
+
+ if len(sdkResponse.KeepServers) < sdkResponse.ItemsAvailable {
+ return results, fmt.Errorf("Did not receive all available keep servers: %+v", sdkResponse)
+ }
+
+ results.KeepServerIndexToAddress = keepServers
+ results.KeepServerAddressToIndex = make(map[ServerAddress]int)
+ for i, address := range results.KeepServerIndexToAddress {
+ results.KeepServerAddressToIndex[address] = i
+ }
+
+ log.Printf("Got Server Addresses: %v", results)
+
+ // Send off all the index requests concurrently
+ responseChan := make(chan ServerResponse)
+ for _, keepServer := range results.KeepServerIndexToAddress {
+ // The above keepsServer variable is reused for each iteration, so
+ // it would be shared across all goroutines. This would result in
+ // us querying one server n times instead of n different servers
+ // as we intended. To avoid this we add it as an explicit
+ // parameter which gets copied. This bug and solution is described
+ // in https://golang.org/doc/effective_go.html#channels
+ go func(keepServer ServerAddress) {
+ responseChan <- GetServerContents(params.Logger,
+ keepServer,
+ params.Client)
+ }(keepServer)
+ }
+
+ results.ServerToContents = make(map[ServerAddress]ServerContents)
+ results.BlockToServers = make(map[blockdigest.DigestWithSize][]BlockServerInfo)
+
+ // Read all the responses
+ for i := range results.KeepServerIndexToAddress {
+ _ = i // Here to prevent go from complaining.
+ response := <-responseChan
+
+ // Check if there were any errors during GetServerContents
+ if response.Err != nil {
+ return results, response.Err
+ }
+
+ log.Printf("Received channel response from %v containing %d files",
+ response.Address,
+ len(response.Contents.BlockDigestToInfo))
+ results.ServerToContents[response.Address] = response.Contents
+ serverIndex := results.KeepServerAddressToIndex[response.Address]
+ for _, blockInfo := range response.Contents.BlockDigestToInfo {
+ results.BlockToServers[blockInfo.Digest] = append(
+ results.BlockToServers[blockInfo.Digest],
+ BlockServerInfo{ServerIndex: serverIndex,
+ Mtime: blockInfo.Mtime})
+ }
+ }
+ return