"flag"
"fmt"
"io"
+ "net/url"
"os"
"os/exec"
"os/user"
"regexp"
"strconv"
+ "strings"
"text/template"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/lib/pq"
)
ClusterID string
Domain string
PostgreSQLPassword string
+ Login string
+ TLS string
+ AdminEmail string
+ Start bool
+
+ LoginPAM bool
+ LoginTest bool
+ LoginGoogle bool
+ LoginGoogleClientID string
+ LoginGoogleClientSecret string
}
func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''")
+ flags.StringVar(&initcmd.AdminEmail, "admin-email", "", "give admin privileges to user with given `email`")
+ flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none")
+ flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
return code
} else if *versionFlag {
return 1
}
+ if fields := strings.Fields(initcmd.Login); len(fields) == 3 && fields[0] == "google" {
+ initcmd.LoginGoogle = true
+ initcmd.LoginGoogleClientID = fields[1]
+ initcmd.LoginGoogleClientSecret = fields[2]
+ } else if initcmd.Login == "test" {
+ initcmd.LoginTest = true
+ if initcmd.AdminEmail == "" {
+ initcmd.AdminEmail = "admin@example.com"
+ }
+ } else if initcmd.Login == "pam" {
+ initcmd.LoginPAM = true
+ } else if initcmd.Login == "" {
+ // none; login will show an error page
+ } else {
+ err = fmt.Errorf("invalid argument to -login: %q: should be 'test', 'pam', 'google {client-id} {client-secret}', or empty", initcmd.Login)
+ return 1
+ }
+
+ confdir := "/etc/arvados"
+ conffile := confdir + "/config.yml"
+ if _, err = os.Stat(conffile); err == nil {
+ err = fmt.Errorf("config file %s already exists; delete it first if you really want to start over", conffile)
+ return 1
+ }
+
+ // Do the "create extension" thing early. This way, if there's
+ // no local postgresql server (a likely failure mode), we can
+ // bail out without any side effects, and the user can start
+ // over easily.
+ fmt.Fprintln(stderr, "installing pg_trgm postgresql extension...")
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`)
+ cmd.Dir = "/"
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("error preparing postgresql server: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+
wwwuser, err := user.Lookup("www-data")
if err != nil {
err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
}
initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+ fmt.Fprintln(stderr, "creating data storage directory /var/lib/arvados/keep ...")
err = os.Mkdir("/var/lib/arvados/keep", 0600)
if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
return 1
}
- fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+ fmt.Fprintln(stderr, "...done")
- err = os.Mkdir("/etc/arvados", 0750)
+ fmt.Fprintln(stderr, "creating config file", conffile, "...")
+ err = os.Mkdir(confdir, 0750)
if err != nil && !os.IsExist(err) {
- err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ err = fmt.Errorf("mkdir %s: %w", confdir, err)
return 1
}
- err = os.Chown("/etc/arvados", 0, wwwgid)
- f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ err = os.Chown(confdir, 0, wwwgid)
if err != nil {
- err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err)
+ return 1
+ }
+ f, err := os.OpenFile(conffile+".tmp", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open %s: %w", conffile+".tmp", err)
return 1
}
tmpl, err := template.New("config").Parse(`Clusters:
Services:
Controller:
InternalURLs:
- "http://0.0.0.0:8003/": {}
- ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ "http://0.0.0.0:9000/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4440/" ) }}
RailsAPI:
InternalURLs:
- "http://0.0.0.0:8004/": {}
+ "http://0.0.0.0:9001/": {}
Websocket:
InternalURLs:
"http://0.0.0.0:8005/": {}
- ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "wss://" .Domain ":4446/" ) }}
Keepbalance:
InternalURLs:
- "http://0.0.0.0:9005/": {}
+ "http://0.0.0.0:9019/": {}
GitHTTP:
InternalURLs:
- "http://0.0.0.0:9001/": {}
- ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ "http://0.0.0.0:9005/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4445/" ) }}
DispatchCloud:
InternalURLs:
"http://0.0.0.0:9006/": {}
Keepproxy:
InternalURLs:
- "http://0.0.0.0:25108/": {}
- ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ "http://0.0.0.0:9007/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4447/" ) }}
WebDAV:
InternalURLs:
- "http://0.0.0.0:9002/": {}
- ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ "http://0.0.0.0:9008/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4448/" ) }}
WebDAVDownload:
InternalURLs:
- "http://0.0.0.0:8004/": {}
- ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ "http://0.0.0.0:9009/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4449/" ) }}
Keepstore:
InternalURLs:
- "http://0.0.0.0:25107/": {}
+ "http://0.0.0.0:9010/": {}
Composer:
- ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4459/composer" ) }}
Workbench1:
InternalURLs:
- "http://0.0.0.0:8001/": {}
- ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
- #Workbench2:
- # InternalURLs:
- # "http://0.0.0.0:8002/": {}
- # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ "http://0.0.0.0:9002/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4442/" ) }}
+ Workbench2:
+ InternalURLs:
+ "http://0.0.0.0:9003/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
Health:
InternalURLs:
- "http://0.0.0.0:9007/": {}
+ "http://0.0.0.0:9011/": {}
Collections:
BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+ {{if eq .TLS "insecure"}}
+ TrustAllContent: true
+ {{end}}
Containers:
DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+ CloudVMs:
+ Enable: true
+ Driver: loopback
ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
PostgreSQL:
Connection:
- dbname: arvados_production
+ dbname: arvados
host: localhost
user: arvados
password: {{printf "%q" .PostgreSQLPassword}}
SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
TLS:
+ {{if eq .TLS "insecure"}}
Insecure: true
+ {{else if eq .TLS "auto"}}
+ Automatic: true
+ {{else if eq .TLS "acme"}}
+ Certificate: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/cert")}}
+ Key: {{printf "%q" (print "/var/lib/acme/live/" .Domain "/privkey")}}
+ {{else}}
+ {}
+ {{end}}
Volumes:
{{.ClusterID}}-nyw5e-000000000000000:
Driver: Directory
Replication: 2
Workbench:
SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+ {{if .LoginPAM}}
+ Login:
+ PAM:
+ Enable: true
+ {{else if .LoginTest}}
+ Login:
+ Test:
+ Enable: true
+ Users:
+ admin:
+ Email: {{printf "%q" .AdminEmail}}
+ Password: admin
+ {{else if .LoginGoogle}}
+ Login:
+ Google:
+ Enable: true
+ ClientID: {{printf "%q" .LoginGoogleClientID}}
+ ClientSecret: {{printf "%q" .LoginGoogleClientSecret}}
+ {{end}}
+ Users:
+ AutoAdminUserWithEmail: {{printf "%q" .AdminEmail}}
`)
if err != nil {
return 1
}
err = tmpl.Execute(f, initcmd)
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ err = fmt.Errorf("%s: tmpl.Execute: %w", conffile+".tmp", err)
return 1
}
err = f.Close()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ err = fmt.Errorf("%s: close: %w", conffile+".tmp", err)
+ return 1
+ }
+ err = os.Rename(conffile+".tmp", conffile)
+ if err != nil {
+ err = fmt.Errorf("rename %s -> %s: %w", conffile+".tmp", conffile, err)
return 1
}
- fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+ fmt.Fprintln(stderr, "...done")
ldr := config.NewLoader(nil, logger)
ldr.SkipLegacy = true
+ ldr.Path = conffile // load the file we just wrote, even if $ARVADOS_CONFIG is set
cfg, err := ldr.Load()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("%s: %w", conffile, err)
return 1
}
cluster, err := cfg.GetCluster("")
return 1
}
+ fmt.Fprintln(stderr, "creating postresql user and database...")
err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
if err != nil {
return 1
}
+ fmt.Fprintln(stderr, "...done")
- cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ fmt.Fprintln(stderr, "initializing database...")
+ cmd = exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
cmd.Dir = "/var/lib/arvados/railsapi"
cmd.Stdout = stderr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- err = fmt.Errorf("rake db:setup: %w", err)
+ err = fmt.Errorf("rake db:setup failed: %w", err)
return 1
}
- fmt.Fprintln(stderr, "initialized database")
+ fmt.Fprintln(stderr, "...done")
+
+ if initcmd.Start {
+ fmt.Fprintln(stderr, "starting systemd service...")
+ cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados")
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+
+ fmt.Fprintln(stderr, "checking controller API endpoint...")
+ u := url.URL(cluster.Services.Controller.ExternalURL)
+ conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider)
+ ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken))
+ _, err = conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ err = fmt.Errorf("API request failed: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...looks good")
+ }
+
+ if out, err := exec.CommandContext(ctx, "docker", "version").CombinedOutput(); err == nil && strings.Contains(string(out), "\nServer:\n") {
+ fmt.Fprintln(stderr, "loading alpine docker image for diagnostics...")
+ cmd := exec.CommandContext(ctx, "docker", "pull", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ cmd = exec.CommandContext(ctx, "arv", "root", "keep", "docker", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+ } else {
+ fmt.Fprintln(stderr, "docker is not installed -- skipping step of downloading 'alpine' image")
+ }
+
+ fmt.Fprintln(stderr, "Setup complete. You should now be able to log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
return 0
}
}
func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
- for _, sql := range []string{
- `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
- `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
- `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
- } {
- cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
- cmd.Stdout = stderr
- cmd.Stderr = stderr
- err := cmd.Run()
- if err != nil {
- return fmt.Errorf("error setting up arvados user/database: %w", err)
- }
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]),
+ "-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ )
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
}
return nil
}
projects / arvados.git / blobdiff