From: Lucas Di Pentima Date: Wed, 24 Jul 2019 12:48:41 +0000 (-0300) Subject: 14716: Migrates to new config. X-Git-Tag: 2.0.0~223^2~25 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/1fa33cf0c06e07c7f3668f994f8d0def93d5ecbd 14716: Migrates to new config. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index b9a1f3069f..8d1062825e 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -17,13 +17,7 @@ import ( const metricsUpdateInterval = time.Second / 10 type cache struct { - TTL arvados.Duration - UUIDTTL arvados.Duration - MaxCollectionEntries int - MaxCollectionBytes int64 - MaxPermissionEntries int - MaxUUIDEntries int - + config *arvados.WebDAVCacheConfig registry *prometheus.Registry metrics cacheMetrics pdhs *lru.TwoQueueCache @@ -110,15 +104,15 @@ type cachedPermission struct { func (c *cache) setup() { var err error - c.pdhs, err = lru.New2Q(c.MaxUUIDEntries) + c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries) if err != nil { panic(err) } - c.collections, err = lru.New2Q(c.MaxCollectionEntries) + c.collections, err = lru.New2Q(c.config.MaxCollectionEntries) if err != nil { panic(err) } - c.permissions, err = lru.New2Q(c.MaxPermissionEntries) + c.permissions, err = lru.New2Q(c.config.MaxPermissionEntries) if err != nil { panic(err) } @@ -164,7 +158,7 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad }) if err == nil { c.collections.Add(client.AuthToken+"\000"+coll.PortableDataHash, &cachedCollection{ - expire: time.Now().Add(time.Duration(c.TTL)), + expire: time.Now().Add(time.Duration(c.config.TTL)), collection: &updated, }) } @@ -221,11 +215,11 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo } if current.PortableDataHash == pdh { c.permissions.Add(permKey, &cachedPermission{ - expire: time.Now().Add(time.Duration(c.TTL)), + expire: time.Now().Add(time.Duration(c.config.TTL)), }) if pdh != targetID { c.pdhs.Add(targetID, &cachedPDH{ - expire: time.Now().Add(time.Duration(c.UUIDTTL)), + expire: time.Now().Add(time.Duration(c.config.UUIDTTL)), pdh: pdh, }) } @@ -246,19 +240,19 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo if err != nil { return nil, err } - exp := time.Now().Add(time.Duration(c.TTL)) + exp := time.Now().Add(time.Duration(c.config.TTL)) c.permissions.Add(permKey, &cachedPermission{ expire: exp, }) c.pdhs.Add(targetID, &cachedPDH{ - expire: time.Now().Add(time.Duration(c.UUIDTTL)), + expire: time.Now().Add(time.Duration(c.config.UUIDTTL)), pdh: collection.PortableDataHash, }) c.collections.Add(arv.ApiToken+"\000"+collection.PortableDataHash, &cachedCollection{ expire: exp, collection: collection, }) - if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { + if int64(len(collection.ManifestText)) > c.config.MaxCollectionBytes/int64(c.config.MaxCollectionEntries) { go c.pruneCollections() } return collection, nil @@ -295,7 +289,7 @@ func (c *cache) pruneCollections() { } } for i, k := range keys { - if size <= c.MaxCollectionBytes { + if size <= c.config.MaxCollectionBytes { break } if expired[i] { diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go index b5c11e553c..083040b219 100644 --- a/services/keep-web/handler.go +++ b/services/keep-web/handler.go @@ -81,7 +81,7 @@ func (h *handler) setup() { keepclient.RefreshServiceDiscoveryOnSIGHUP() h.healthHandler = &health.Handler{ - Token: h.Config.ManagementToken, + Token: h.Config.cluster.ManagementToken, Prefix: "/_health/", } @@ -249,9 +249,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { var pathToken bool var attachment bool var useSiteFS bool - credentialsOK := h.Config.TrustAllContent + credentialsOK := h.Config.cluster.Collections.TrustAllContent - if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost { + if r.Host != "" && r.Host == h.Config.cluster.Services.WebDAVDownload.ExternalURL.Host { credentialsOK = true attachment = true } else if r.FormValue("disposition") == "attachment" { @@ -283,7 +283,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } else { // /collections/ID/PATH... collectionID = parseCollectionIDFromURL(pathParts[1]) - tokens = h.Config.AnonymousTokens + tokens = []string{h.Config.cluster.Users.AnonymousUserToken} stripParts = 2 } } @@ -350,7 +350,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { if credentialsOK { reqTokens = auth.CredentialsFromRequest(r).Tokens } - tokens = append(reqTokens, h.Config.AnonymousTokens...) + tokens = append(reqTokens, h.Config.cluster.Users.AnonymousUserToken) } if len(targetPath) > 0 && targetPath[0] == "_" { diff --git a/services/keep-web/main.go b/services/keep-web/main.go index 018b5a2e81..b8b2725096 100644 --- a/services/keep-web/main.go +++ b/services/keep-web/main.go @@ -8,52 +8,36 @@ import ( "flag" "fmt" "os" - "time" + "git.curoverse.com/arvados.git/lib/config" "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/config" + sdkConfig "git.curoverse.com/arvados.git/sdk/go/config" "github.com/coreos/go-systemd/daemon" log "github.com/sirupsen/logrus" ) var ( - defaultConfigPath = "/etc/arvados/keep-web/keep-web.yml" - version = "dev" + version = "dev" ) // Config specifies server configuration. type Config struct { - Client arvados.Client - - Listen string - - AnonymousTokens []string - AttachmentOnlyHost string - TrustAllContent bool - - Cache cache - - // Hack to support old command line flag, which is a bool - // meaning "get actual token from environment". - deprecatedAllowAnonymous bool - - //Authorization token to be included in all health check requests. - ManagementToken string + Client arvados.Client + Cache cache + cluster *arvados.Cluster } // DefaultConfig returns the default configuration. -func DefaultConfig() *Config { - return &Config{ - Listen: ":80", - Cache: cache{ - TTL: arvados.Duration(5 * time.Minute), - UUIDTTL: arvados.Duration(5 * time.Second), - MaxCollectionEntries: 1000, - MaxCollectionBytes: 100000000, - MaxPermissionEntries: 1000, - MaxUUIDEntries: 1000, - }, +func DefaultConfig(arvCfg *arvados.Config) *Config { + cfg := Config{} + var cls *arvados.Cluster + var err error + if cls, err = arvCfg.GetCluster(""); err != nil { + log.Fatal(err) } + cfg.cluster = cls + cfg.Cache.config = &cfg.cluster.Collections.WebDAVCache + return &cfg } func init() { @@ -72,29 +56,23 @@ func init() { } func main() { - cfg := DefaultConfig() - - var configPath string - deprecated := " (DEPRECATED -- use config file instead)" - flag.StringVar(&configPath, "config", defaultConfigPath, - "`path` to JSON or YAML configuration file") - flag.StringVar(&cfg.Listen, "listen", "", - "address:port or :port to listen on"+deprecated) - flag.BoolVar(&cfg.deprecatedAllowAnonymous, "allow-anonymous", false, - "Load an anonymous token from the ARVADOS_API_TOKEN environment variable"+deprecated) - flag.StringVar(&cfg.AttachmentOnlyHost, "attachment-only-host", "", - "Only serve attachments at the given `host:port`"+deprecated) - flag.BoolVar(&cfg.TrustAllContent, "trust-all-content", false, - "Serve non-public content from a single origin. Dangerous: read docs before using!"+deprecated) - flag.StringVar(&cfg.ManagementToken, "management-token", "", - "Authorization token to be included in all health check requests.") - - dumpConfig := flag.Bool("dump-config", false, + prog := os.Args[0] + args := os.Args[1:] + logger := log.New() + + flags := flag.NewFlagSet(prog, flag.ExitOnError) + flags.Usage = usage + + loader := config.NewLoader(os.Stdin, logger) + loader.SetupFlags(flags) + + dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit") - getVersion := flag.Bool("version", false, + getVersion := flags.Bool("version", false, "print version information and exit.") - flag.Usage = usage - flag.Parse() + + args = loader.MungeLegacyConfigArgs(logger, args, "-legacy-keepweb-config") + flags.Parse(args) // Print version information if requested if *getVersion { @@ -102,26 +80,19 @@ func main() { return } - if err := config.LoadFile(cfg, configPath); err != nil { - if h := os.Getenv("ARVADOS_API_HOST"); h != "" && configPath == defaultConfigPath { - log.Printf("DEPRECATED: Using ARVADOS_API_HOST environment variable. Use config file instead.") - cfg.Client.APIHost = h - } else { - log.Fatal(err) - } - } - if cfg.deprecatedAllowAnonymous { - log.Printf("DEPRECATED: Using -allow-anonymous command line flag with ARVADOS_API_TOKEN environment variable. Use config file instead.") - cfg.AnonymousTokens = []string{os.Getenv("ARVADOS_API_TOKEN")} + arvCfg, err := loader.Load() + if err != nil { + log.Fatal(err) } + cfg := DefaultConfig(arvCfg) if *dumpConfig { - log.Fatal(config.DumpAndExit(cfg)) + log.Fatal(sdkConfig.DumpAndExit(cfg.cluster)) } log.Printf("keep-web %s started", version) - os.Setenv("ARVADOS_API_HOST", cfg.Client.APIHost) + os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host) srv := &server{Config: cfg} if err := srv.Start(); err != nil { log.Fatal(err) diff --git a/services/keep-web/server.go b/services/keep-web/server.go index 167fbbe5b8..28a9dde247 100644 --- a/services/keep-web/server.go +++ b/services/keep-web/server.go @@ -8,6 +8,7 @@ import ( "context" "net/http" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/ctxlog" "git.curoverse.com/arvados.git/sdk/go/httpserver" "github.com/prometheus/client_golang/prometheus" @@ -25,8 +26,12 @@ func (srv *server) Start() error { h.Config.Cache.registry = reg ctx := ctxlog.Context(context.Background(), logrus.StandardLogger()) mh := httpserver.Instrument(reg, nil, httpserver.HandlerWithContext(ctx, httpserver.AddRequestIDs(httpserver.LogRequests(h)))) - h.MetricsAPI = mh.ServeAPI(h.Config.ManagementToken, http.NotFoundHandler()) + h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler()) srv.Handler = mh - srv.Addr = srv.Config.Listen + var listen arvados.URL + for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs { + break + } + srv.Addr = listen.Host return srv.Server.Start() } diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go index 705955ba0a..fda030dbad 100644 --- a/services/keep-web/usage.go +++ b/services/keep-web/usage.go @@ -12,8 +12,8 @@ import ( ) func usage() { - c := DefaultConfig() - c.AnonymousTokens = []string{"xxxxxxxxxxxxxxxxxxxxxxx"} + c := DefaultConfig(nil) + c.cluster.Users.AnonymousUserToken = "xxxxxxxxxxxxxxxxxxxxxxx" c.Client.APIHost = "zzzzz.arvadosapi.com:443" exampleConfigFile, err := json.MarshalIndent(c, " ", " ") if err != nil {