X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f81d411d2becc23fe640366620d9568149d89bd6..9dfd25989a0b93f20936812d99bf453dd9c079f2:/lib/install/init.go diff --git a/lib/install/init.go b/lib/install/init.go index 8debf59be5..11a62f18e9 100644 --- a/lib/install/init.go +++ b/lib/install/init.go @@ -13,16 +13,20 @@ import ( "flag" "fmt" "io" + "net/url" "os" "os/exec" "os/user" "regexp" "strconv" + "strings" "text/template" "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/lib/pq" ) @@ -33,6 +37,16 @@ type initCommand struct { ClusterID string Domain string PostgreSQLPassword string + Login string + TLS string + AdminEmail string + Start bool + + LoginPAM bool + LoginTest bool + LoginGoogle bool + LoginGoogleClientID string + LoginGoogleClientSecret string } func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { @@ -59,22 +73,61 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster") flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com") - err = flags.Parse(args) - if err == flag.ErrHelp { - err = nil - return 0 - } else if err != nil { - return 2 + flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''") + flags.StringVar(&initcmd.AdminEmail, "admin-email", "", "give admin privileges to user with given `email`") + flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none") + flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config") + if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok { + return code } else if *versionFlag { return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr) - } else if len(flags.Args()) > 0 { - err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) - return 2 } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) { err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID) return 1 } + if fields := strings.Fields(initcmd.Login); len(fields) == 3 && fields[0] == "google" { + initcmd.LoginGoogle = true + initcmd.LoginGoogleClientID = fields[1] + initcmd.LoginGoogleClientSecret = fields[2] + } else if initcmd.Login == "test" { + initcmd.LoginTest = true + if initcmd.AdminEmail == "" { + initcmd.AdminEmail = "admin@example.com" + } + } else if initcmd.Login == "pam" { + initcmd.LoginPAM = true + } else if initcmd.Login == "" { + // none; login will show an error page + } else { + err = fmt.Errorf("invalid argument to -login: %q: should be 'test', 'pam', 'google {client-id} {client-secret}', or empty", initcmd.Login) + return 1 + } + + confdir := "/etc/arvados" + conffile := confdir + "/config.yml" + if _, err = os.Stat(conffile); err == nil { + err = fmt.Errorf("config file %s already exists; delete it first if you really want to start over", conffile) + return 1 + } + + // Do the "create extension" thing early. This way, if there's + // no local postgresql server (a likely failure mode), we can + // bail out without any side effects, and the user can start + // over easily. + fmt.Fprintln(stderr, "installing pg_trgm postgresql extension...") + cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet", + "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`) + cmd.Dir = "/" + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + err = fmt.Errorf("error preparing postgresql server: %w", err) + return 1 + } + fmt.Fprintln(stderr, "...done") + wwwuser, err := user.Lookup("www-data") if err != nil { err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err) @@ -86,22 +139,28 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read } initcmd.PostgreSQLPassword = initcmd.RandomHex(32) + fmt.Fprintln(stderr, "creating data storage directory /var/lib/arvados/keep ...") err = os.Mkdir("/var/lib/arvados/keep", 0600) if err != nil && !os.IsExist(err) { err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err) return 1 } - fmt.Fprintln(stderr, "created /var/lib/arvados/keep") + fmt.Fprintln(stderr, "...done") - err = os.Mkdir("/etc/arvados", 0750) + fmt.Fprintln(stderr, "creating config file", conffile, "...") + err = os.Mkdir(confdir, 0750) if err != nil && !os.IsExist(err) { - err = fmt.Errorf("mkdir /etc/arvados: %w", err) + err = fmt.Errorf("mkdir %s: %w", confdir, err) + return 1 + } + err = os.Chown(confdir, 0, wwwgid) + if err != nil { + err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err) return 1 } - err = os.Chown("/etc/arvados", 0, wwwgid) - f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + f, err := os.OpenFile(conffile+".tmp", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) if err != nil { - err = fmt.Errorf("open /etc/arvados/config.yml: %w", err) + err = fmt.Errorf("open %s: %w", conffile+".tmp", err) return 1 } tmpl, err := template.New("config").Parse(`Clusters: @@ -109,69 +168,82 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read Services: Controller: InternalURLs: - "http://0.0.0.0:8003/": {} - ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }} + "http://0.0.0.0:9000/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4440/" ) }} RailsAPI: InternalURLs: - "http://0.0.0.0:8004/": {} + "http://0.0.0.0:9001/": {} Websocket: InternalURLs: "http://0.0.0.0:8005/": {} - ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }} + ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4446/" ) }} Keepbalance: InternalURLs: - "http://0.0.0.0:9005/": {} + "http://0.0.0.0:9019/": {} GitHTTP: InternalURLs: - "http://0.0.0.0:9001/": {} - ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }} + "http://0.0.0.0:9005/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4445/" ) }} DispatchCloud: InternalURLs: "http://0.0.0.0:9006/": {} Keepproxy: InternalURLs: - "http://0.0.0.0:25108/": {} - ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }} + "http://0.0.0.0:9007/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4447/" ) }} WebDAV: InternalURLs: - "http://0.0.0.0:9002/": {} - ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }} + "http://0.0.0.0:9008/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4448/" ) }} WebDAVDownload: InternalURLs: - "http://0.0.0.0:8004/": {} - ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }} + "http://0.0.0.0:9009/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4449/" ) }} Keepstore: InternalURLs: - "http://0.0.0.0:25107/": {} + "http://0.0.0.0:9010/": {} Composer: - ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4459/composer" ) }} Workbench1: InternalURLs: - "http://0.0.0.0:8001/": {} - ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }} + "http://0.0.0.0:9002/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain ":4442/" ) }} Workbench2: InternalURLs: - "http://0.0.0.0:8002/": {} - ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }} + "http://0.0.0.0:9003/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }} Health: InternalURLs: - "http://0.0.0.0:9007/": {} - API: - RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}} + "http://0.0.0.0:9011/": {} Collections: BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}} + {{if eq .TLS "insecure"}} + TrustAllContent: true + {{end}} Containers: DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}} + CloudVMs: + Enable: true + Driver: loopback ManagementToken: {{printf "%q" ( .RandomHex 50 )}} PostgreSQL: Connection: - dbname: arvados_production + dbname: arvados host: localhost user: arvados password: {{printf "%q" .PostgreSQLPassword}} SystemRootToken: {{printf "%q" ( .RandomHex 50 )}} TLS: + {{if eq .TLS "insecure"}} Insecure: true + {{else if eq .TLS "auto"}} + Automatic: true + {{else if eq .TLS "acme"}} + Certificate: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/cert")}} + Key: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/privkey")}} + {{else}} + {} + {{end}} Volumes: {{.ClusterID}}-nyw5e-000000000000000: Driver: Directory @@ -180,27 +252,54 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read Replication: 2 Workbench: SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}} + {{if .LoginPAM}} + Login: + PAM: + Enable: true + {{else if .LoginTest}} + Login: + Test: + Enable: true + Users: + admin: + Email: {{printf "%q" .AdminEmail}} + Password: admin + {{else if .LoginGoogle}} + Login: + Google: + Enable: true + ClientID: {{printf "%q" .LoginGoogleClientID}} + ClientSecret: {{printf "%q" .LoginGoogleClientSecret}} + {{end}} + Users: + AutoAdminUserWithEmail: {{printf "%q" .AdminEmail}} `) if err != nil { return 1 } err = tmpl.Execute(f, initcmd) if err != nil { - err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err) + err = fmt.Errorf("%s: tmpl.Execute: %w", conffile+".tmp", err) return 1 } err = f.Close() if err != nil { - err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err) + err = fmt.Errorf("%s: close: %w", conffile+".tmp", err) + return 1 + } + err = os.Rename(conffile+".tmp", conffile) + if err != nil { + err = fmt.Errorf("rename %s -> %s: %w", conffile+".tmp", conffile, err) return 1 } - fmt.Fprintln(stderr, "created /etc/arvados/config.yml") + fmt.Fprintln(stderr, "...done") ldr := config.NewLoader(nil, logger) ldr.SkipLegacy = true + ldr.Path = conffile // load the file we just wrote, even if $ARVADOS_CONFIG is set cfg, err := ldr.Load() if err != nil { - err = fmt.Errorf("/etc/arvados/config.yml: %w", err) + err = fmt.Errorf("%s: %w", conffile, err) return 1 } cluster, err := cfg.GetCluster("") @@ -208,21 +307,51 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read return 1 } + fmt.Fprintln(stderr, "creating postresql user and database...") err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr) if err != nil { return 1 } + fmt.Fprintln(stderr, "...done") - cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup") + fmt.Fprintln(stderr, "initializing database...") + cmd = exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup") cmd.Dir = "/var/lib/arvados/railsapi" cmd.Stdout = stderr cmd.Stderr = stderr err = cmd.Run() if err != nil { - err = fmt.Errorf("rake db:setup: %w", err) + err = fmt.Errorf("rake db:setup failed: %w", err) return 1 } - fmt.Fprintln(stderr, "initialized database") + fmt.Fprintln(stderr, "...done") + + if initcmd.Start { + fmt.Fprintln(stderr, "starting systemd service...") + cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados") + cmd.Dir = "/" + cmd.Stdout = stderr + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + err = fmt.Errorf("%v: %w", cmd.Args, err) + return 1 + } + fmt.Fprintln(stderr, "...done") + + fmt.Fprintln(stderr, "checking controller API endpoint...") + u := url.URL(cluster.Services.Controller.ExternalURL) + conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider) + ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken)) + _, err = conn.UserGetCurrent(ctx, arvados.GetOptions{}) + if err != nil { + err = fmt.Errorf("API request failed: %w", err) + return 1 + } + fmt.Fprintln(stderr, "...looks good") + } + + fmt.Fprintln(stderr, "Setup complete. You should now be able to log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String()) return 0 } @@ -252,18 +381,16 @@ func (initcmd *initCommand) RandomHex(chars int) string { } func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error { - for _, sql := range []string{ - `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]), - `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`, - `CREATE EXTENSION IF NOT EXISTS pg_trgm`, - } { - cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql) - cmd.Stdout = stderr - cmd.Stderr = stderr - err := cmd.Run() - if err != nil { - return fmt.Errorf("error setting up arvados user/database: %w", err) - } + cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet", + "-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]), + "-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`, + ) + cmd.Dir = "/" + cmd.Stdout = stderr + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("error setting up arvados user/database: %w", err) } return nil }