X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/577bd728e410d6a9b57b11578264b61220a50d76..8805632994d42c0e3b31fd5ac010d916cac12de1:/services/keepstore/keepstore.go

diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 06054f5205..a363bac255 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"io/ioutil"
 	"log"
 	"net"
@@ -91,12 +92,16 @@ func (e *KeepError) Error() string {
 // Initialized by the --volumes flag (or by FindKeepVolumes).
 var KeepVM VolumeManager
 
-// The pull list manager is a singleton pull list (a list of blocks
-// that the current keepstore process should be pulling from remote
-// keepstore servers in order to increase data replication) with
-// atomic update methods that are safe to use from multiple
-// goroutines.
-var pullmgr *BlockWorkList
+// The pull list manager and trash queue are threadsafe queues which
+// support atomic update operations. The PullHandler and TrashHandler
+// store results from Data Manager /pull and /trash requests here.
+//
+// See the Keep and Data Manager design documents for more details:
+// https://arvados.org/projects/arvados/wiki/Keep_Design_Doc
+// https://arvados.org/projects/arvados/wiki/Data_Manager_Design_Doc
+//
+var pullq *WorkQueue
+var trashq *WorkQueue
 
 // TODO(twp): continue moving as much code as possible out of main
 // so it can be effectively tested. Esp. handling and postprocessing
@@ -258,9 +263,11 @@ func main() {
 	// Start a round-robin VolumeManager with the volumes we have found.
 	KeepVM = MakeRRVolumeManager(goodvols)
 
-	// Tell the built-in HTTP server to direct all requests to the REST
-	// router.
-	http.Handle("/", MakeRESTRouter())
+	// Tell the built-in HTTP server to direct all requests to the REST router.
+	loggingRouter := MakeLoggingRESTRouter()
+	http.HandleFunc("/", func(resp http.ResponseWriter, req *http.Request) {
+		loggingRouter.ServeHTTP(resp, req)
+	})
 
 	// Set up a TCP listener.
 	listener, err := net.Listen("tcp", listen)
@@ -268,6 +275,22 @@ func main() {
 		log.Fatal(err)
 	}
 
+	// Initialize Pull queue and worker
+	keepClient := keepclient.KeepClient{
+		Arvados:       nil,
+		Want_replicas: 1,
+		Using_proxy:   true,
+		Client:        &http.Client{},
+	}
+
+	// Initialize the pullq and worker
+	pullq = NewWorkQueue()
+	go RunPullWorker(pullq, keepClient)
+
+	// Initialize the trashq and worker
+	trashq = NewWorkQueue()
+	go RunTrashWorker(trashq)
+
 	// Shut down the server gracefully (by closing the listener)
 	// if SIGTERM is received.
 	term := make(chan os.Signal, 1)