X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/91d40ffe727119f7661e734f9535fd9005880e98..9dfd25989a0b93f20936812d99bf453dd9c079f2:/lib/install/init.go

diff --git a/lib/install/init.go b/lib/install/init.go
index 6a52441d71..11a62f18e9 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -13,16 +13,20 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"os/exec"
 	"os/user"
 	"regexp"
 	"strconv"
+	"strings"
 	"text/template"
 
 	"git.arvados.org/arvados.git/lib/cmd"
 	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/lib/controller/rpc"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"github.com/lib/pq"
 )
@@ -34,7 +38,15 @@ type initCommand struct {
 	Domain             string
 	PostgreSQLPassword string
 	Login              string
-	Insecure           bool
+	TLS                string
+	AdminEmail         string
+	Start              bool
+
+	LoginPAM                bool
+	LoginTest               bool
+	LoginGoogle             bool
+	LoginGoogleClientID     string
+	LoginGoogleClientSecret string
 }
 
 func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -61,8 +73,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
 	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
 	flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
 	flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
-	flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''")
-	flags.BoolVar(&initcmd.Insecure, "insecure", false, "accept invalid TLS certificates and configure TrustAllContent (do not use in production!)")
+	flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''")
+	flags.StringVar(&initcmd.AdminEmail, "admin-email", "", "give admin privileges to user with given `email`")
+	flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none")
+	flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
 	if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
 		return code
 	} else if *versionFlag {
@@ -72,6 +86,48 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
 		return 1
 	}
 
+	if fields := strings.Fields(initcmd.Login); len(fields) == 3 && fields[0] == "google" {
+		initcmd.LoginGoogle = true
+		initcmd.LoginGoogleClientID = fields[1]
+		initcmd.LoginGoogleClientSecret = fields[2]
+	} else if initcmd.Login == "test" {
+		initcmd.LoginTest = true
+		if initcmd.AdminEmail == "" {
+			initcmd.AdminEmail = "admin@example.com"
+		}
+	} else if initcmd.Login == "pam" {
+		initcmd.LoginPAM = true
+	} else if initcmd.Login == "" {
+		// none; login will show an error page
+	} else {
+		err = fmt.Errorf("invalid argument to -login: %q: should be 'test', 'pam', 'google {client-id} {client-secret}', or empty", initcmd.Login)
+		return 1
+	}
+
+	confdir := "/etc/arvados"
+	conffile := confdir + "/config.yml"
+	if _, err = os.Stat(conffile); err == nil {
+		err = fmt.Errorf("config file %s already exists; delete it first if you really want to start over", conffile)
+		return 1
+	}
+
+	// Do the "create extension" thing early. This way, if there's
+	// no local postgresql server (a likely failure mode), we can
+	// bail out without any side effects, and the user can start
+	// over easily.
+	fmt.Fprintln(stderr, "installing pg_trgm postgresql extension...")
+	cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+		"-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`)
+	cmd.Dir = "/"
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		err = fmt.Errorf("error preparing postgresql server: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "...done")
+
 	wwwuser, err := user.Lookup("www-data")
 	if err != nil {
 		err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
@@ -83,22 +139,28 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
 	}
 	initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
 
+	fmt.Fprintln(stderr, "creating data storage directory /var/lib/arvados/keep ...")
 	err = os.Mkdir("/var/lib/arvados/keep", 0600)
 	if err != nil && !os.IsExist(err) {
 		err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
 		return 1
 	}
-	fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+	fmt.Fprintln(stderr, "...done")
 
-	err = os.Mkdir("/etc/arvados", 0750)
+	fmt.Fprintln(stderr, "creating config file", conffile, "...")
+	err = os.Mkdir(confdir, 0750)
 	if err != nil && !os.IsExist(err) {
-		err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+		err = fmt.Errorf("mkdir %s: %w", confdir, err)
+		return 1
+	}
+	err = os.Chown(confdir, 0, wwwgid)
+	if err != nil {
+		err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err)
 		return 1
 	}
-	err = os.Chown("/etc/arvados", 0, wwwgid)
-	f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	f, err := os.OpenFile(conffile+".tmp", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
 	if err != nil {
-		err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+		err = fmt.Errorf("open %s: %w", conffile+".tmp", err)
 		return 1
 	}
 	tmpl, err := template.New("config").Parse(`Clusters:
@@ -113,8 +175,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
           "http://0.0.0.0:9001/": {}
       Websocket:
         InternalURLs:
-          "http://0.0.0.0:9004/": {}
-        ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4444/websocket" ) }}
+          "http://0.0.0.0:8005/": {}
+        ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4446/" ) }}
       Keepbalance:
         InternalURLs:
           "http://0.0.0.0:9019/": {}
@@ -155,23 +217,33 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
           "http://0.0.0.0:9011/": {}
     Collections:
       BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
-      {{if .Insecure}}
+      {{if eq .TLS "insecure"}}
       TrustAllContent: true
       {{end}}
     Containers:
       DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+      CloudVMs:
+        Enable: true
+        Driver: loopback
     ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
     PostgreSQL:
       Connection:
-        dbname: arvados_production
+        dbname: arvados
         host: localhost
         user: arvados
         password: {{printf "%q" .PostgreSQLPassword}}
     SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
-    {{if .Insecure}}
     TLS:
+      {{if eq .TLS "insecure"}}
       Insecure: true
-    {{end}}
+      {{else if eq .TLS "auto"}}
+      Automatic: true
+      {{else if eq .TLS "acme"}}
+      Certificate: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/cert")}}
+      Key: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/privkey")}}
+      {{else}}
+      {}
+      {{end}}
     Volumes:
       {{.ClusterID}}-nyw5e-000000000000000:
         Driver: Directory
@@ -180,47 +252,54 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
         Replication: 2
     Workbench:
       SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+    {{if .LoginPAM}}
     Login:
-      {{if eq .Login "pam"}}
       PAM:
         Enable: true
-      {{else if eq .Login "test"}}
+    {{else if .LoginTest}}
+    Login:
       Test:
         Enable: true
         Users:
           admin:
-            Email: admin@example.com
+            Email: {{printf "%q" .AdminEmail}}
             Password: admin
-      {{else}}
-      {}
-      {{end}}
+    {{else if .LoginGoogle}}
+    Login:
+      Google:
+        Enable: true
+        ClientID: {{printf "%q" .LoginGoogleClientID}}
+        ClientSecret: {{printf "%q" .LoginGoogleClientSecret}}
+    {{end}}
     Users:
-      {{if eq .Login "test"}}
-      AutoAdminUserWithEmail: admin@example.com
-      {{else}}
-      {}
-      {{end}}
+      AutoAdminUserWithEmail: {{printf "%q" .AdminEmail}}
 `)
 	if err != nil {
 		return 1
 	}
 	err = tmpl.Execute(f, initcmd)
 	if err != nil {
-		err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+		err = fmt.Errorf("%s: tmpl.Execute: %w", conffile+".tmp", err)
 		return 1
 	}
 	err = f.Close()
 	if err != nil {
-		err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+		err = fmt.Errorf("%s: close: %w", conffile+".tmp", err)
 		return 1
 	}
-	fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+	err = os.Rename(conffile+".tmp", conffile)
+	if err != nil {
+		err = fmt.Errorf("rename %s -> %s: %w", conffile+".tmp", conffile, err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "...done")
 
 	ldr := config.NewLoader(nil, logger)
 	ldr.SkipLegacy = true
+	ldr.Path = conffile // load the file we just wrote, even if $ARVADOS_CONFIG is set
 	cfg, err := ldr.Load()
 	if err != nil {
-		err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+		err = fmt.Errorf("%s: %w", conffile, err)
 		return 1
 	}
 	cluster, err := cfg.GetCluster("")
@@ -228,21 +307,51 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
 		return 1
 	}
 
+	fmt.Fprintln(stderr, "creating postresql user and database...")
 	err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
 	if err != nil {
 		return 1
 	}
+	fmt.Fprintln(stderr, "...done")
 
-	cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+	fmt.Fprintln(stderr, "initializing database...")
+	cmd = exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
 	cmd.Dir = "/var/lib/arvados/railsapi"
 	cmd.Stdout = stderr
 	cmd.Stderr = stderr
 	err = cmd.Run()
 	if err != nil {
-		err = fmt.Errorf("rake db:setup: %w", err)
+		err = fmt.Errorf("rake db:setup failed: %w", err)
 		return 1
 	}
-	fmt.Fprintln(stderr, "initialized database")
+	fmt.Fprintln(stderr, "...done")
+
+	if initcmd.Start {
+		fmt.Fprintln(stderr, "starting systemd service...")
+		cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados")
+		cmd.Dir = "/"
+		cmd.Stdout = stderr
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			err = fmt.Errorf("%v: %w", cmd.Args, err)
+			return 1
+		}
+		fmt.Fprintln(stderr, "...done")
+
+		fmt.Fprintln(stderr, "checking controller API endpoint...")
+		u := url.URL(cluster.Services.Controller.ExternalURL)
+		conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider)
+		ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken))
+		_, err = conn.UserGetCurrent(ctx, arvados.GetOptions{})
+		if err != nil {
+			err = fmt.Errorf("API request failed: %w", err)
+			return 1
+		}
+		fmt.Fprintln(stderr, "...looks good")
+	}
+
+	fmt.Fprintln(stderr, "Setup complete. You should now be able to log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
 
 	return 0
 }
@@ -272,18 +381,16 @@ func (initcmd *initCommand) RandomHex(chars int) string {
 }
 
 func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
-	for _, sql := range []string{
-		`CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
-		`CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
-		`CREATE EXTENSION IF NOT EXISTS pg_trgm`,
-	} {
-		cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
-		cmd.Stdout = stderr
-		cmd.Stderr = stderr
-		err := cmd.Run()
-		if err != nil {
-			return fmt.Errorf("error setting up arvados user/database: %w", err)
-		}
+	cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+		"-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]),
+		"-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`,
+	)
+	cmd.Dir = "/"
+	cmd.Stdout = stderr
+	cmd.Stderr = stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error setting up arvados user/database: %w", err)
 	}
 	return nil
 }