14712: arv-git-httpd uses cluster config
authorEric Biagiotti <ebiagiotti@veritasgenetics.com>
Fri, 16 Aug 2019 18:54:30 +0000 (14:54 -0400)
committerEric Biagiotti <ebiagiotti@veritasgenetics.com>
Mon, 26 Aug 2019 15:58:34 +0000 (11:58 -0400)
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

services/arv-git-httpd/auth_handler.go
services/arv-git-httpd/git_handler.go
services/arv-git-httpd/main.go
services/arv-git-httpd/server.go
services/arv-git-httpd/usage.go [deleted file]

index 3b3032afda5d9707616ce474c431e10d2e629e37..6c618189474912bab9fa45062739acff6862ddad 100644 (file)
@@ -14,6 +14,7 @@ import (
        "sync"
        "time"
 
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/auth"
        "git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -22,14 +23,21 @@ import (
 type authHandler struct {
        handler    http.Handler
        clientPool *arvadosclient.ClientPool
+       cluster    *arvados.Cluster
        setupOnce  sync.Once
 }
 
 func (h *authHandler) setup() {
-       ac, err := arvadosclient.New(&theConfig.Client)
+       client, err := arvados.NewClientFromConfig(h.cluster)
        if err != nil {
                log.Fatal(err)
        }
+
+       ac, err := arvadosclient.New(client)
+       if err != nil {
+               log.Fatalf("Error setting up arvados client prototype %v", err)
+       }
+
        h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
 }
 
@@ -161,7 +169,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                "/" + repoName + "/.git",
        }
        for _, dir := range tryDirs {
-               if fileInfo, err := os.Stat(theConfig.RepoRoot + dir); err != nil {
+               if fileInfo, err := os.Stat(h.cluster.Git.Repositories + dir); err != nil {
                        if !os.IsNotExist(err) {
                                statusCode, statusText = http.StatusInternalServerError, err.Error()
                                return
@@ -173,7 +181,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        }
        if rewrittenPath == "" {
                log.Println("WARNING:", repoUUID,
-                       "git directory not found in", theConfig.RepoRoot, tryDirs)
+                       "git directory not found in", h.cluster.Git.Repositories, tryDirs)
                // We say "content not found" to disambiguate from the
                // earlier "API says that repo does not exist" error.
                statusCode, statusText = http.StatusNotFound, "content not found"
index d9b08a995b3ab1baee6b3a5b33a3f3f9d084e2b6..0b14e81d54107fb97ff68747bffa58a4ff63de5c 100644 (file)
@@ -10,6 +10,8 @@ import (
        "net/http"
        "net/http/cgi"
        "os"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
 )
 
 // gitHandler is an http.Handler that invokes git-http-backend (or
@@ -20,29 +22,34 @@ type gitHandler struct {
        cgi.Handler
 }
 
-func newGitHandler() http.Handler {
+func newGitHandler(cluster *arvados.Cluster) http.Handler {
        const glBypass = "GL_BYPASS_ACCESS_CHECKS"
        const glHome = "GITOLITE_HTTP_HOME"
        var env []string
        path := os.Getenv("PATH")
-       if theConfig.GitoliteHome != "" {
+       if cluster.Git.GitoliteHome != "" {
                env = append(env,
-                       glHome+"="+theConfig.GitoliteHome,
+                       glHome+"="+cluster.Git.GitoliteHome,
                        glBypass+"=1")
-               path = path + ":" + theConfig.GitoliteHome + "/bin"
+               path = path + ":" + cluster.Git.GitoliteHome + "/bin"
        } else if home, bypass := os.Getenv(glHome), os.Getenv(glBypass); home != "" || bypass != "" {
                env = append(env, glHome+"="+home, glBypass+"="+bypass)
                log.Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
        }
+
+       var listen arvados.URL
+       for listen = range cluster.Services.GitHTTP.InternalURLs {
+               break
+       }
        env = append(env,
-               "GIT_PROJECT_ROOT="+theConfig.RepoRoot,
+               "GIT_PROJECT_ROOT="+cluster.Git.Repositories,
                "GIT_HTTP_EXPORT_ALL=",
-               "SERVER_ADDR="+theConfig.Listen,
+               "SERVER_ADDR="+listen.Host,
                "PATH="+path)
        return &gitHandler{
                Handler: cgi.Handler{
-                       Path: theConfig.GitCommand,
-                       Dir:  theConfig.RepoRoot,
+                       Path: cluster.Git.GitCommand,
+                       Dir:  cluster.Git.Repositories,
                        Env:  env,
                        Args: []string{"http-backend"},
                },
index 74ac7ae55eea05aa396f9ff337a8dbed74505079..3edfcf4ca68454418e62dd29739a16c07f68c242 100644 (file)
@@ -5,89 +5,62 @@
 package main
 
 import (
-       "encoding/json"
        "flag"
        "fmt"
-       "log"
        "os"
-       "regexp"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/config"
+       "git.curoverse.com/arvados.git/lib/config"
        "github.com/coreos/go-systemd/daemon"
+       "github.com/ghodss/yaml"
+       log "github.com/sirupsen/logrus"
 )
 
 var version = "dev"
 
-// Server configuration
-type Config struct {
-       Client          arvados.Client
-       Listen          string
-       GitCommand      string
-       RepoRoot        string
-       GitoliteHome    string
-       ManagementToken string
-}
-
-var theConfig = defaultConfig()
-
-func defaultConfig() *Config {
-       return &Config{
-               Listen:     ":80",
-               GitCommand: "/usr/bin/git",
-               RepoRoot:   "/var/lib/arvados/git/repositories",
-       }
-}
-
 func main() {
-       const defaultCfgPath = "/etc/arvados/git-httpd/git-httpd.yml"
-       const deprecated = " (DEPRECATED -- use config file instead)"
-       flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
-               "Address to listen on, \"host:port\" or \":port\"."+deprecated)
-       flag.StringVar(&theConfig.GitCommand, "git-command", theConfig.GitCommand,
-               "Path to git or gitolite-shell executable. Each authenticated request will execute this program with a single argument, \"http-backend\"."+deprecated)
-       flag.StringVar(&theConfig.RepoRoot, "repo-root", theConfig.RepoRoot,
-               "Path to git repositories."+deprecated)
-       flag.StringVar(&theConfig.GitoliteHome, "gitolite-home", theConfig.GitoliteHome,
-               "Value for GITOLITE_HTTP_HOME environment variable. If not empty, GL_BYPASS_ACCESS_CHECKS=1 will also be set."+deprecated)
+       logger := log.New()
+       log.SetFormatter(&log.JSONFormatter{
+               TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+       })
 
-       cfgPath := flag.String("config", defaultCfgPath, "Configuration file `path`.")
-       dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
-       getVersion := flag.Bool("version", false, "print version information and exit.")
+       flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+       loader := config.NewLoader(os.Stdin, logger)
+       loader.SetupFlags(flags)
 
-       flag.StringVar(&theConfig.ManagementToken, "management-token", theConfig.ManagementToken,
-               "Authorization token to be included in all health check requests.")
+       dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
+       getVersion := flags.Bool("version", false, "print version information and exit.")
 
-       flag.Usage = usage
-       flag.Parse()
+       args := loader.MungeLegacyConfigArgs(logger, os.Args[1:], "-legacy-git-httpd-config")
+       flags.Parse(args)
 
-       // Print version information if requested
        if *getVersion {
                fmt.Printf("arv-git-httpd %s\n", version)
                return
        }
 
-       err := config.LoadFile(theConfig, *cfgPath)
+       cfg, err := loader.Load()
        if err != nil {
-               h := os.Getenv("ARVADOS_API_HOST")
-               if h == "" || !os.IsNotExist(err) || *cfgPath != defaultCfgPath {
-                       log.Fatal(err)
-               }
-               log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST environment variable is set. Please use a config file instead.")
-               theConfig.Client.APIHost = h
-               if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
-                       theConfig.Client.Insecure = true
-               }
-               if j, err := json.MarshalIndent(theConfig, "", "    "); err == nil {
-                       log.Print("Current configuration:\n", string(j))
-               }
+               log.Fatal(err)
+       }
+
+       cluster, err := cfg.GetCluster("")
+       if err != nil {
+               log.Fatal(err)
        }
 
        if *dumpConfig {
-               log.Fatal(config.DumpAndExit(theConfig))
+               out, err := yaml.Marshal(cfg)
+               if err != nil {
+                       log.Fatal(err)
+               }
+               _, err = os.Stdout.Write(out)
+               if err != nil {
+                       log.Fatal(err)
+               }
+               return
        }
 
-       srv := &server{}
+       srv := &server{cluster: cluster}
        if err := srv.Start(); err != nil {
                log.Fatal(err)
        }
@@ -96,7 +69,7 @@ func main() {
        }
        log.Printf("arv-git-httpd %s started", version)
        log.Println("Listening at", srv.Addr)
-       log.Println("Repository root", theConfig.RepoRoot)
+       log.Println("Repository root", cluster.Git.Repositories)
        if err := srv.Wait(); err != nil {
                log.Fatal(err)
        }
index 8f0d90f89e29cad0a0d8d590b66e412b02195ad2..56c9765b56947959792b1ac3e046cfdf5afa2652 100644 (file)
@@ -7,22 +7,30 @@ package main
 import (
        "net/http"
 
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/health"
        "git.curoverse.com/arvados.git/sdk/go/httpserver"
 )
 
 type server struct {
        httpserver.Server
+       cluster *arvados.Cluster
 }
 
 func (srv *server) Start() error {
        mux := http.NewServeMux()
-       mux.Handle("/", &authHandler{handler: newGitHandler()})
+       mux.Handle("/", &authHandler{handler: newGitHandler(srv.cluster), cluster: srv.cluster})
        mux.Handle("/_health/", &health.Handler{
-               Token:  theConfig.ManagementToken,
+               Token:  srv.cluster.ManagementToken,
                Prefix: "/_health/",
        })
+
+       var listen arvados.URL
+       for listen = range srv.cluster.Services.GitHTTP.InternalURLs {
+               break
+       }
+
        srv.Handler = mux
-       srv.Addr = theConfig.Listen
+       srv.Addr = listen.Host
        return srv.Server.Start()
 }
diff --git a/services/arv-git-httpd/usage.go b/services/arv-git-httpd/usage.go
deleted file mode 100644 (file)
index 8863da6..0000000
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-// arvados-git-httpd provides authenticated access to Arvados-hosted
-// git repositories.
-//
-// See http://doc.arvados.org/install/install-arv-git-httpd.html.
-package main
-
-import (
-       "flag"
-       "fmt"
-       "os"
-
-       "github.com/ghodss/yaml"
-)
-
-func usage() {
-       c := defaultConfig()
-       c.Client.APIHost = "zzzzz.arvadosapi.com:443"
-       exampleConfigFile, err := yaml.Marshal(c)
-       if err != nil {
-               panic(err)
-       }
-       fmt.Fprintf(os.Stderr, `
-
-arvados-git-httpd provides authenticated access to Arvados-hosted git
-repositories.
-
-See http://doc.arvados.org/install/install-arv-git-httpd.html.
-
-Usage: arvados-git-httpd [-config path/to/arvados/git-httpd.yml]
-
-Options:
-`)
-       flag.PrintDefaults()
-       fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Client.APIHost:
-
-    Address (or address:port) of the Arvados API endpoint.
-
-Client.AuthToken:
-
-    Unused. Normally empty, or omitted entirely.
-
-Client.Insecure:
-
-    True if your Arvados API endpoint uses an unverifiable SSL/TLS
-    certificate.
-
-GitCommand:
-
-    Path to git or gitolite-shell executable. Each authenticated
-    request will execute this program with the single argument
-    "http-backend".
-
-GitoliteHome:
-
-    Path to Gitolite's home directory. If a non-empty path is given,
-    the CGI environment will be set up to support the use of
-    gitolite-shell as a GitCommand: for example, if GitoliteHome is
-    "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
-    PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
-
-Listen:
-
-    Local port to listen on. Can be "address:port" or ":port", where
-    "address" is a host IP address or name and "port" is a port number
-    or name.
-
-RepoRoot:
-
-    Path to git repositories.
-
-`, exampleConfigFile)
-}