X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b185fc94a543b5b1361497c8502e876d6fdc2838..8934b505b0cdd6af699d910148046ede7c44b7da:/services/keepstore/handlers.go diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go index 9d4c617d88..289b0c8c74 100644 --- a/services/keepstore/handlers.go +++ b/services/keepstore/handlers.go @@ -10,10 +10,10 @@ package main import ( "bufio" "bytes" + "container/list" "crypto/md5" "encoding/json" "fmt" - "git.curoverse.com/arvados.git/services/keepstore/replicator" "github.com/gorilla/mux" "io" "log" @@ -59,10 +59,17 @@ func MakeRESTRouter() *mux.Router { `/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler).Methods("GET", "HEAD") rest.HandleFunc(`/status.json`, StatusHandler).Methods("GET", "HEAD") - // The PullHandler processes "PUT /pull" commands from Data Manager. - // It parses the JSON list of pull requests in the request body, and - // delivers them to the PullBlocks goroutine for replication. + // The PullHandler and TrashHandler process "PUT /pull" and "PUT + // /trash" requests from Data Manager. These requests instruct + // Keep to replicate or delete blocks; see + // https://arvados.org/projects/orvos-private/wiki/Keep_Design_Doc + // for more details. + // + // Each handler parses the JSON list of block management requests + // in the message body, and delivers them to the pull queue or + // trash queue, respectively. rest.HandleFunc(`/pull`, PullHandler).Methods("PUT") + rest.HandleFunc(`/trash`, TrashHandler).Methods("PUT") // Any request which does not match any of these routes gets // 400 Bad Request. @@ -436,33 +443,88 @@ func DeleteHandler(resp http.ResponseWriter, req *http.Request) { If the JSON unmarshalling fails, return 400 Bad Request. */ +type PullRequest struct { + Locator string `json:"locator"` + Servers []string `json:"servers"` +} + func PullHandler(resp http.ResponseWriter, req *http.Request) { // Reject unauthorized requests. api_token := GetApiToken(req) if !IsDataManagerToken(api_token) { http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode) + log.Printf("%s %s: %s\n", req.Method, req.URL, UnauthorizedError.Error()) return } // Parse the request body. - var pull_list []replicator.PullRequest + var pr []PullRequest r := json.NewDecoder(req.Body) - if err := r.Decode(&pull_list); err != nil { + if err := r.Decode(&pr); err != nil { http.Error(resp, BadRequestError.Error(), BadRequestError.HTTPCode) + log.Printf("%s %s: %s\n", req.Method, req.URL, err.Error()) return } // We have a properly formatted pull list sent from the data - // manager. Report success and send the list to the keep - // replicator for further handling. + // manager. Report success and send the list to the pull list + // manager for further handling. + log.Printf("%s %s: received %v\n", req.Method, req.URL, pr) + resp.WriteHeader(http.StatusOK) + resp.Write([]byte( + fmt.Sprintf("Received %d pull requests\n", len(pr)))) + + plist := list.New() + for _, p := range pr { + plist.PushBack(p) + } + + if pullq == nil { + pullq = NewWorkQueue() + } + pullq.ReplaceQueue(plist) +} + +type TrashRequest struct { + Locator string `json:"locator"` + BlockMtime int64 `json:"block_mtime"` +} + +func TrashHandler(resp http.ResponseWriter, req *http.Request) { + // Reject unauthorized requests. + api_token := GetApiToken(req) + if !IsDataManagerToken(api_token) { + http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode) + log.Printf("%s %s: %s\n", req.Method, req.URL, UnauthorizedError.Error()) + return + } + + // Parse the request body. + var trash []TrashRequest + r := json.NewDecoder(req.Body) + if err := r.Decode(&trash); err != nil { + http.Error(resp, BadRequestError.Error(), BadRequestError.HTTPCode) + log.Printf("%s %s: %s\n", req.Method, req.URL, err.Error()) + return + } + + // We have a properly formatted trash list sent from the data + // manager. Report success and send the list to the trash work + // queue for further handling. + log.Printf("%s %s: received %v\n", req.Method, req.URL, trash) resp.WriteHeader(http.StatusOK) resp.Write([]byte( - fmt.Sprintf("Received %d pull requests\n", len(pull_list)))) + fmt.Sprintf("Received %d trash requests\n", len(trash)))) + + tlist := list.New() + for _, t := range trash { + tlist.PushBack(t) + } - if replica == nil { - replica = replicator.New() + if trashq == nil { + trashq = NewWorkQueue() } - replica.Pull(pull_list) + trashq.ReplaceQueue(tlist) } // ==============================