X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e1902b8a0490aa6b7ffc544c1609d4d57a5110ce..472d5d0daa87835ab04c02a6298fdc338d1a0446:/services/keepstore/keepstore.go

diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 69ff5659c1..6ae414bf93 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -1,24 +1,27 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
 	"flag"
 	"fmt"
-	"log"
 	"net"
-	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/config"
-	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"github.com/coreos/go-systemd/daemon"
-	"github.com/ghodss/yaml"
 )
 
+var version = "dev"
+
 // A Keep "block" is 64MB.
 const BlockSize = 64 * 1024 * 1024
 
@@ -82,15 +85,11 @@ var KeepVM VolumeManager
 var pullq *WorkQueue
 var trashq *WorkQueue
 
-// TODO(twp): continue moving as much code as possible out of main
-// so it can be effectively tested. Esp. handling and postprocessing
-// of command line flags (identifying Keep volumes and initializing
-// permission arguments).
-
 func main() {
 	deprecated.beforeFlagParse(theConfig)
 
 	dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
+	getVersion := flag.Bool("version", false, "Print version information and exit.")
 
 	defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
 	var configPath string
@@ -102,6 +101,12 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	// Print version information if requested
+	if *getVersion {
+		fmt.Printf("keepstore %s\n", version)
+		return
+	}
+
 	deprecated.afterFlagParse(theConfig)
 
 	err := config.LoadFile(theConfig, configPath)
@@ -110,15 +115,15 @@ func main() {
 	}
 
 	if *dumpConfig {
-		y, err := yaml.Marshal(theConfig)
-		if err != nil {
-			log.Fatal(err)
-		}
-		os.Stdout.Write(y)
-		os.Exit(0)
+		log.Fatal(config.DumpAndExit(theConfig))
 	}
 
+	log.Printf("keepstore %s started", version)
+
 	err = theConfig.Start()
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	if pidfile := theConfig.PIDFile; pidfile != "" {
 		f, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0777)
@@ -145,17 +150,30 @@ func main() {
 		}
 	}
 
+	var cluster *arvados.Cluster
+	cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
+	if err != nil && os.IsNotExist(err) {
+		log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
+		cluster = &arvados.Cluster{
+			ClusterID: "xxxxx",
+		}
+	} else if err != nil {
+		log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
+	} else {
+		cluster, err = cfg.GetCluster("")
+		if err != nil {
+			log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
+		}
+	}
+
 	log.Println("keepstore starting, pid", os.Getpid())
 	defer log.Println("keepstore exiting, pid", os.Getpid())
 
 	// Start a round-robin VolumeManager with the volumes we have found.
 	KeepVM = MakeRRVolumeManager(theConfig.Volumes)
 
-	// Middleware stack: logger, MaxRequests limiter, method handlers
-	http.Handle("/", &LoggingRESTRouter{
-		httpserver.NewRequestLimiter(theConfig.MaxRequests,
-			MakeRESTRouter()),
-	})
+	// Middleware/handler stack
+	router := MakeRESTRouter(cluster)
 
 	// Set up a TCP listener.
 	listener, err := net.Listen("tcp", theConfig.Listen)
@@ -163,20 +181,23 @@ func main() {
 		log.Fatal(err)
 	}
 
-	// Initialize Pull queue and worker
+	// Initialize keepclient for pull workers
 	keepClient := &keepclient.KeepClient{
 		Arvados:       &arvadosclient.ArvadosClient{},
 		Want_replicas: 1,
-		Client:        &http.Client{},
 	}
 
-	// Initialize the pullq and worker
+	// Initialize the pullq and workers
 	pullq = NewWorkQueue()
-	go RunPullWorker(pullq, keepClient)
+	for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
+		go RunPullWorker(pullq, keepClient)
+	}
 
-	// Initialize the trashq and worker
+	// Initialize the trashq and workers
 	trashq = NewWorkQueue()
-	go RunTrashWorker(trashq)
+	for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
+		go RunTrashWorker(trashq)
+	}
 
 	// Start emptyTrash goroutine
 	doneEmptyingTrash := make(chan bool)
@@ -194,11 +215,12 @@ func main() {
 	signal.Notify(term, syscall.SIGTERM)
 	signal.Notify(term, syscall.SIGINT)
 
-	if _, err := daemon.SdNotify("READY=1"); err != nil {
+	if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
 		log.Printf("Error notifying init daemon: %v", err)
 	}
-	log.Println("listening at", listener.Addr)
-	srv := &http.Server{}
+	log.Println("listening at", listener.Addr())
+	srv := &server{}
+	srv.Handler = router
 	srv.Serve(listener)
 }