From 7abc7ca38954acd4eaa53c9280504e06a76b8d71 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Wed, 12 Feb 2020 09:12:02 -0500 Subject: [PATCH] 15954: Start own postgresql server. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- doc/examples/config/zzzzz.yml | 7 -- go.mod | 2 +- go.sum | 2 + lib/boot/cert.go | 55 ++++++++++++++ lib/boot/cmd.go | 130 +++++++++++++++++++++++++--------- lib/boot/nginx.go | 3 +- lib/boot/postgresql.go | 100 ++++++++++++++++++++++++++ 7 files changed, 257 insertions(+), 42 deletions(-) create mode 100644 lib/boot/cert.go create mode 100644 lib/boot/postgresql.go diff --git a/doc/examples/config/zzzzz.yml b/doc/examples/config/zzzzz.yml index 9e3d718ed5..c63550edf7 100644 --- a/doc/examples/config/zzzzz.yml +++ b/doc/examples/config/zzzzz.yml @@ -1,12 +1,5 @@ Clusters: zzzzz: - PostgreSQL: - Connection: - client_encoding: utf8 - host: localhost - dbname: arvados_test - user: arvados - password: insecure_arvados_test ManagementToken: e687950a23c3a9bceec28c6223a06c79 SystemRootToken: systemusertesttoken1234567890aoeuidhtnsqjkxbmwvzpy API: diff --git a/go.mod b/go.mod index 2e16e5a0fb..85e5552f63 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff github.com/julienschmidt/httprouter v1.2.0 github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect - github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd + github.com/lib/pq v1.3.0 github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c // indirect github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index d7a022dda9..6c2323a316 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd h1:2RDaVc4/izhWyAvYxNm8c9saSyCDIxefNwOcqaH7pcU= github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c h1:ouxemItv3B/Zh008HJkEXDYCN3BIRyNHxtUN7ThJ5Js= github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= diff --git a/lib/boot/cert.go b/lib/boot/cert.go new file mode 100644 index 0000000000..011f418e91 --- /dev/null +++ b/lib/boot/cert.go @@ -0,0 +1,55 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package boot + +import ( + "context" + "io/ioutil" + "path/filepath" +) + +func createCertificates(ctx context.Context, boot *Booter, ready chan<- bool) error { + // Generate root key + err := boot.RunProgram(ctx, boot.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096") + if err != nil { + return err + } + // Generate a self-signed root certificate + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost") + if err != nil { + return err + } + // Generate server key + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048") + if err != nil { + return err + } + // Build config file for signing request + defaultconf, err := ioutil.ReadFile("/etc/ssl/openssl.cnf") + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(boot.tempdir, "server.cfg"), append(defaultconf, []byte(` +[SAN] +subjectAltName=DNS:localhost,DNS:localhost.localdomain +`)...), 0777) + if err != nil { + return err + } + // Generate signing request + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr") + if err != nil { + return err + } + // Sign certificate + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-days", "3650", "-sha256") + if err != nil { + return err + } + + close(ready) + <-ctx.Done() + return nil +} diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go index 4d2c01f2c5..93f6ee0a87 100644 --- a/lib/boot/cmd.go +++ b/lib/boot/cmd.go @@ -18,6 +18,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -71,6 +72,7 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou flags.StringVar(&boot.SourcePath, "source", ".", "arvados source tree `directory`") flags.StringVar(&boot.LibPath, "lib", "/var/lib/arvados", "`directory` to install dependencies and library files") flags.StringVar(&boot.ClusterType, "type", "production", "cluster `type`: development, test, or production") + flags.BoolVar(&boot.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database") err = flags.Parse(args) if err == flag.ErrHelp { err = nil @@ -96,10 +98,11 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou } type Booter struct { - SourcePath string // e.g., /home/username/src/arvados - LibPath string // e.g., /var/lib/arvados - ClusterType string // e.g., production - Stderr io.Writer + SourcePath string // e.g., /home/username/src/arvados + LibPath string // e.g., /var/lib/arvados + ClusterType string // e.g., production + OwnTemporaryDatabase bool + Stderr io.Writer logger logrus.FieldLogger cluster *arvados.Cluster @@ -212,29 +215,43 @@ func (boot *Booter) run(loader *config.Loader) error { } var wg sync.WaitGroup - for _, cmpt := range []component{ - {name: "nginx", runFunc: runNginx}, - {name: "controller", cmdHandler: controller.Command}, - {name: "dispatchcloud", cmdHandler: dispatchcloud.Command, notIfTest: true}, - {name: "git-httpd", goProg: "services/arv-git-httpd"}, - {name: "health", goProg: "services/health"}, - {name: "keep-balance", goProg: "services/keep-balance", notIfTest: true}, - {name: "keepproxy", goProg: "services/keepproxy"}, - {name: "keepstore", goProg: "services/keepstore", svc: boot.cluster.Services.Keepstore}, - {name: "keep-web", goProg: "services/keep-web"}, - {name: "railsAPI", svc: boot.cluster.Services.RailsAPI, railsApp: "services/api"}, - {name: "workbench1", svc: boot.cluster.Services.Workbench1, railsApp: "apps/workbench"}, - {name: "ws", goProg: "services/ws"}, - } { - cmpt := cmpt + components := map[string]*component{ + "certificates": &component{runFunc: createCertificates}, + "database": &component{runFunc: runPostgres, depends: []string{"certificates"}}, + "nginx": &component{runFunc: runNginx}, + "controller": &component{cmdHandler: controller.Command, depends: []string{"database"}}, + "dispatchcloud": &component{cmdHandler: dispatchcloud.Command, notIfTest: true}, + "git-httpd": &component{goProg: "services/arv-git-httpd"}, + "health": &component{goProg: "services/health"}, + "keep-balance": &component{goProg: "services/keep-balance", notIfTest: true}, + "keepproxy": &component{goProg: "services/keepproxy"}, + "keepstore": &component{goProg: "services/keepstore", svc: boot.cluster.Services.Keepstore}, + "keep-web": &component{goProg: "services/keep-web"}, + "railsAPI": &component{svc: boot.cluster.Services.RailsAPI, railsApp: "services/api", depends: []string{"database"}}, + "workbench1": &component{svc: boot.cluster.Services.Workbench1, railsApp: "apps/workbench"}, + "ws": &component{goProg: "services/ws", depends: []string{"database"}}, + } + for _, cmpt := range components { + cmpt.ready = make(chan bool) + } + for name, cmpt := range components { + name, cmpt := name, cmpt wg.Add(1) go func() { defer wg.Done() defer boot.cancel() - boot.logger.WithField("component", cmpt.name).Info("starting") - err := cmpt.Run(boot.ctx, boot) + for _, dep := range cmpt.depends { + boot.logger.WithField("component", name).WithField("dependency", dep).Info("waiting") + select { + case <-components[dep].ready: + case <-boot.ctx.Done(): + return + } + } + boot.logger.WithField("component", name).Info("starting") + err := cmpt.Run(boot.ctx, name, boot) if err != nil && err != context.Canceled { - boot.logger.WithError(err).WithField("component", cmpt.name).Error("exited") + boot.logger.WithError(err).WithField("component", name).Error("exited") } }() } @@ -382,24 +399,27 @@ type component struct { name string svc arvados.Service cmdHandler cmd.Handler - runFunc func(ctx context.Context, boot *Booter) error - railsApp string // source dir in arvados tree, e.g., "services/api" - goProg string // source dir in arvados tree, e.g., "services/keepstore" - notIfTest bool // don't run this component on a test cluster + runFunc func(ctx context.Context, boot *Booter, ready chan<- bool) error + railsApp string // source dir in arvados tree, e.g., "services/api" + goProg string // source dir in arvados tree, e.g., "services/keepstore" + notIfTest bool // don't run this component on a test cluster + depends []string // don't start until all of these components are ready + + ready chan bool } -func (cmpt *component) Run(ctx context.Context, boot *Booter) error { +func (cmpt *component) Run(ctx context.Context, name string, boot *Booter) error { if cmpt.notIfTest && boot.ClusterType == "test" { - fmt.Fprintf(boot.Stderr, "skipping component %q in %s mode\n", cmpt.name, boot.ClusterType) + fmt.Fprintf(boot.Stderr, "skipping component %q in %s mode\n", name, boot.ClusterType) <-ctx.Done() return nil } - fmt.Fprintf(boot.Stderr, "starting component %q\n", cmpt.name) + fmt.Fprintf(boot.Stderr, "starting component %q\n", name) if cmpt.cmdHandler != nil { errs := make(chan error, 1) go func() { defer close(errs) - exitcode := cmpt.cmdHandler.RunCommand(cmpt.name, []string{"-config", boot.configfile}, bytes.NewBuffer(nil), boot.Stderr, boot.Stderr) + exitcode := cmpt.cmdHandler.RunCommand(name, []string{"-config", boot.configfile}, bytes.NewBuffer(nil), boot.Stderr, boot.Stderr) if exitcode != 0 { errs <- fmt.Errorf("exit code %d", exitcode) } @@ -439,12 +459,12 @@ func (cmpt *component) Run(ctx context.Context, boot *Booter) error { return nil } if cmpt.runFunc != nil { - return cmpt.runFunc(ctx, boot) + return cmpt.runFunc(ctx, boot, cmpt.ready) } if cmpt.railsApp != "" { port, err := internalPort(cmpt.svc) if err != nil { - return fmt.Errorf("bug: no InternalURLs for component %q: %v", cmpt.name, cmpt.svc.InternalURLs) + return fmt.Errorf("bug: no InternalURLs for component %q: %v", name, cmpt.svc.InternalURLs) } var buf bytes.Buffer err = boot.RunProgram(ctx, cmpt.railsApp, &buf, nil, "gem", "list", "--details", "bundler") @@ -482,7 +502,7 @@ func (cmpt *component) Run(ctx context.Context, boot *Booter) error { } return nil } - return fmt.Errorf("bug: component %q has nothing to run", cmpt.name) + return fmt.Errorf("bug: component %q has nothing to run", name) } func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger) error { @@ -572,6 +592,21 @@ func (boot *Booter) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger) } } } + if boot.OwnTemporaryDatabase { + p, err := availablePort() + if err != nil { + return err + } + cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{ + "client_encoding": "utf8", + "host": "localhost", + "port": strconv.Itoa(p), + "dbname": "arvados_test", + "user": "arvados", + "password": "insecure_arvados_test", + } + } + cfg.Clusters[cluster.ClusterID] = *cluster return nil } @@ -611,3 +646,32 @@ func externalPort(svc arvados.Service) (string, error) { return "80", nil } } + +func availablePort() (int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return 0, err + } + defer ln.Close() + _, p, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + return 0, err + } + return strconv.Atoi(p) +} + +// Try to connect to addr until it works, then close ch. Give up if +// ctx cancels. +func connectAndClose(ctx context.Context, addr string, ch chan<- bool) { + dialer := net.Dialer{Timeout: time.Second} + for ctx.Err() == nil { + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + time.Sleep(time.Second / 10) + continue + } + conn.Close() + close(ch) + return + } +} diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go index 1b361dd9c4..2df5e90b3d 100644 --- a/lib/boot/nginx.go +++ b/lib/boot/nginx.go @@ -16,7 +16,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvados" ) -func runNginx(ctx context.Context, boot *Booter) error { +func runNginx(ctx context.Context, boot *Booter, ready chan<- bool) error { vars := map[string]string{ "SSLCERT": filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca "SSLKEY": filepath.Join(boot.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca @@ -69,6 +69,7 @@ func runNginx(ctx context.Context, boot *Booter) error { } } } + go connectAndClose(ctx, boot.cluster.Services.Controller.ExternalURL.Host, ready) return boot.RunProgram(ctx, ".", nil, nil, nginx, "-g", "error_log stderr info;", "-g", "pid "+filepath.Join(boot.tempdir, "nginx.pid")+";", diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go new file mode 100644 index 0000000000..86328e110c --- /dev/null +++ b/lib/boot/postgresql.go @@ -0,0 +1,100 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package boot + +import ( + "bytes" + "context" + "database/sql" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "github.com/lib/pq" +) + +func runPostgres(ctx context.Context, boot *Booter, ready chan<- bool) error { + buf := bytes.NewBuffer(nil) + err := boot.RunProgram(ctx, boot.tempdir, buf, nil, "pg_config", "--bindir") + if err != nil { + return err + } + datadir := filepath.Join(boot.tempdir, "pgdata") + + err = os.Mkdir(datadir, 0755) + if err != nil { + return err + } + bindir := strings.TrimSpace(buf.String()) + + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, filepath.Join(bindir, "initdb"), "-D", datadir) + if err != nil { + return err + } + + err = boot.RunProgram(ctx, boot.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir) + if err != nil { + return err + } + + port := boot.cluster.PostgreSQL.Connection["port"] + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + for { + if ctx.Err() != nil { + return + } + if exec.CommandContext(ctx, "pg_isready", "--timeout=10", "--host="+boot.cluster.PostgreSQL.Connection["host"], "--port="+port).Run() == nil { + break + } + time.Sleep(time.Second / 2) + } + db, err := sql.Open("postgres", arvados.PostgreSQLConnection{ + "host": datadir, + "port": port, + "dbname": "postgres", + }.String()) + if err != nil { + boot.logger.WithError(err).Error("db open failed") + cancel() + return + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + boot.logger.WithError(err).Error("db conn failed") + cancel() + return + } + defer conn.Close() + _, err = conn.ExecContext(ctx, `CREATE USER `+pq.QuoteIdentifier(boot.cluster.PostgreSQL.Connection["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(boot.cluster.PostgreSQL.Connection["password"])) + if err != nil { + boot.logger.WithError(err).Error("createuser failed") + cancel() + return + } + _, err = conn.ExecContext(ctx, `CREATE DATABASE `+pq.QuoteIdentifier(boot.cluster.PostgreSQL.Connection["dbname"])) + if err != nil { + boot.logger.WithError(err).Error("createdb failed") + cancel() + return + } + close(ready) + return + }() + + return boot.RunProgram(ctx, boot.tempdir, nil, nil, filepath.Join(bindir, "postgres"), + "-l", // enable ssl + "-D", datadir, // data dir + "-k", datadir, // socket dir + "-p", boot.cluster.PostgreSQL.Connection["port"], + ) +} -- 2.39.5