import (
"bufio"
+ "encoding/json"
"flag"
"fmt"
- //"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- "git.curoverse.com/arvados.git/sdk/go/util"
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"io/ioutil"
"log"
"strconv"
"strings"
"sync"
+ "time"
)
type ServerAddress struct {
Host string `json:"service_host"`
Port int `json:"service_port"`
+ Uuid string `json:"uuid"`
}
// Info about a particular block returned by the server
type BlockInfo struct {
Digest blockdigest.BlockDigest
Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Mtime int64 // TODO(misha): Replace this with a timestamp.
}
// Info about a specified block given by a server
type BlockServerInfo struct {
ServerIndex int
Size int
- Mtime int // TODO(misha): Replace this with a timestamp.
+ Mtime int64 // TODO(misha): Replace this with a timestamp.
}
type ServerContents struct {
KeepServers []ServerAddress `json:"items"`
}
-// Methods to implement util.SdkListResponse Interface
-func (k KeepServiceList) NumItemsAvailable() (numAvailable int, err error) {
- return k.ItemsAvailable, nil
-}
-
-func (k KeepServiceList) NumItemsContained() (numContained int, err error) {
- return len(k.KeepServers), nil
-}
-
var (
// Don't access the token directly, use getDataManagerToken() to
// make sure it's been read.
"File with the API token we should use to contact keep servers.")
}
+// TODO(misha): Change this to include the UUID as well.
func (s ServerAddress) String() string {
return fmt.Sprintf("%s:%d", s.Host, s.Port)
}
"contain a valid ArvadosClient, but instead it is nil.")
}
- sdkParams := arvadosclient.Dict{}
+ sdkParams := arvadosclient.Dict{
+ "filters": [][]string{[]string{"service_type", "=", "disk"}},
+ }
if params.Limit > 0 {
sdkParams["limit"] = params.Limit
}
var sdkResponse KeepServiceList
- err := params.Client.Call("GET", "keep_services", "", "accessible", sdkParams, &sdkResponse)
+ err := params.Client.List("keep_services", sdkParams, &sdkResponse)
if err != nil {
loggerutil.FatalWithMessage(params.Logger,
fmt.Sprintf("Error requesting keep disks from API server: %v", err))
}
- // TODO(misha): Rewrite this block, stop using ContainsAllAvailableItems()
- {
- var numReceived, numAvailable int
- results.ReadAllServers, numReceived, numAvailable =
- util.ContainsAllAvailableItems(sdkResponse)
-
- if !results.ReadAllServers {
- log.Printf("ERROR: Did not receive all keep server addresses.")
- }
- log.Printf("Received %d of %d available keep server addresses.",
- numReceived,
- numAvailable)
- }
-
if params.Logger != nil {
- properties, _ := params.Logger.Edit()
- keepInfo := make(map[string]interface{})
-
- keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
- keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
- keepInfo["keep_servers"] = sdkResponse.KeepServers
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := make(map[string]interface{})
- properties["keep_info"] = keepInfo
+ keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
+ keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
+ keepInfo["keep_servers"] = sdkResponse.KeepServers
- params.Logger.Record()
+ p["keep_info"] = keepInfo
+ })
}
- log.Printf("Received keep services list: %v", sdkResponse)
+ log.Printf("Received keep services list: %+v", sdkResponse)
+
+ if len(sdkResponse.KeepServers) < sdkResponse.ItemsAvailable {
+ loggerutil.FatalWithMessage(params.Logger,
+ fmt.Sprintf("Did not receive all available keep servers: %+v", sdkResponse))
+ }
results.KeepServerIndexToAddress = sdkResponse.KeepServers
results.KeepServerAddressToIndex = make(map[ServerAddress]int)
keepServer ServerAddress,
client http.Client) (response ServerResponse) {
+ GetServerStatus(arvLogger, keepServer, client)
+
req := CreateIndexRequest(arvLogger, keepServer)
resp, err := client.Do(req)
if err != nil {
- log.Fatalf("Error fetching %s: %v", req.URL.String(), err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error fetching %s: %v", req.URL.String(), err))
}
return ReadServerResponse(arvLogger, keepServer, resp)
}
+func GetServerStatus(arvLogger *logger.Logger,
+ keepServer ServerAddress,
+ client http.Client) {
+ url := fmt.Sprintf("http://%s:%d/status.json",
+ keepServer.Host,
+ keepServer.Port)
+
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := make(map[string]interface{})
+ serverInfo["status_request_sent_at"] = now
+ serverInfo["host"] = keepServer.Host
+ serverInfo["port"] = keepServer.Port
+
+ keepInfo[keepServer.Uuid] = serverInfo
+ })
+ }
+
+ resp, err := client.Get(url)
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error getting keep status from %s: %v", url, err))
+ } else if resp.StatusCode != 200 {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error code %d in response to request "+
+ "for %s status: %s",
+ resp.StatusCode, url, resp.Status))
+ }
+
+ var keepStatus map[string]interface{}
+ decoder := json.NewDecoder(resp.Body)
+ decoder.UseNumber()
+ err = decoder.Decode(&keepStatus)
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error decoding keep status from %s: %v", url, err))
+ }
+
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+ serverInfo["status_response_processed_at"] = now
+ serverInfo["status"] = keepStatus
+ })
+ }
+}
+
func CreateIndexRequest(arvLogger *logger.Logger,
keepServer ServerAddress) (req *http.Request) {
- // Create and send request.
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+ serverInfo["index_request_sent_at"] = now
+ })
+ }
+
req, err := http.NewRequest("GET", url, nil)
if err != nil {
- log.Fatalf("Error building http request for %s: %v", url, err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error building http request for %s: %v", url, err))
}
req.Header.Add("Authorization",
resp *http.Response) (response ServerResponse) {
if resp.StatusCode != 200 {
- log.Fatalf("Received error code %d in response to request for %s index: %s",
- resp.StatusCode, keepServer.String(), resp.Status)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error code %d in response to request "+
+ "for %s index: %s",
+ resp.StatusCode, keepServer.String(), resp.Status))
+ }
+
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+ serverInfo["index_response_received_at"] = now
+ })
}
response.Address = keepServer
numLines++
blockInfo, err := parseBlockInfoFromIndexLine(scanner.Text())
if err != nil {
- log.Fatalf("Error parsing BlockInfo from index line received from %s: %v",
- keepServer.String(),
- err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error parsing BlockInfo from index line "+
+ "received from %s: %v",
+ keepServer.String(),
+ err))
}
if storedBlock, ok := response.Contents.BlockDigestToInfo[blockInfo.Digest]; ok {
if storedBlock.Size != blockInfo.Size {
numSizeDisagreements += 1
// TODO(misha): Consider failing here.
- log.Printf("Saw different sizes for the same block on %s: %v %v",
+ message := fmt.Sprintf("Saw different sizes for the same block "+
+ "on %s: %+v %+v",
keepServer.String(),
storedBlock,
blockInfo)
+ log.Println(message)
+ if arvLogger != nil {
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+ var error_list []string
+ read_error_list, has_list := serverInfo["error_list"]
+ if has_list {
+ error_list = read_error_list.([]string)
+ } // If we didn't have the list, error_list is already an empty list
+ serverInfo["error_list"] = append(error_list, message)
+ })
+ }
}
// Keep the block that is bigger, or the block that's newer in
// the case of a size tie.
}
}
if err := scanner.Err(); err != nil {
- log.Fatalf("Received error scanning index response from %s: %v",
- keepServer.String(),
- err)
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Received error scanning index response from %s: %v",
+ keepServer.String(),
+ err))
} else {
log.Printf("%s index contained %d lines with %d duplicates with "+
"%d size disagreements",
numLines,
numDuplicates,
numSizeDisagreements)
+
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ keepInfo := p["keep_info"].(map[string]interface{})
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+
+ serverInfo["processing_finished_at"] = now
+ serverInfo["lines_received"] = numLines
+ serverInfo["duplicates_seen"] = numDuplicates
+ serverInfo["size_disagreements_seen"] = numSizeDisagreements
+ })
+ }
}
resp.Body.Close()
return
return
}
- blockInfo.Mtime, err = strconv.Atoi(tokens[1])
+ blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
if err != nil {
return
}
projects / arvados.git / blobdiff