X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0b6013ac5520b7508474705d654f1384994d10e5..060dc61639b8a5ac4458bc21d1120b3ad508b0a3:/lib/boot/supervisor.go diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index 417f09d23f..8746183e6c 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -21,6 +21,7 @@ import ( "os/user" "path/filepath" "reflect" + "strconv" "strings" "sync" "syscall" @@ -41,6 +42,8 @@ type Supervisor struct { ClusterType string // e.g., production ListenHost string // e.g., localhost ControllerAddr string // e.g., 127.0.0.1:8000 + Workbench2Source string // e.g., /home/username/src/arvados-workbench2 + NoWorkbench1 bool OwnTemporaryDatabase bool Stderr io.Writer @@ -62,6 +65,8 @@ type Supervisor struct { environ []string // for child processes } +func (super *Supervisor) Cluster() *arvados.Cluster { return super.cluster } + func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config, cfgPath string) { super.ctx, super.cancel = context.WithCancel(ctx) super.done = make(chan struct{}) @@ -143,7 +148,6 @@ func (super *Supervisor) run(cfg *arvados.Config) error { super.tempdir = "/var/lib/arvados/tmp" super.wwwtempdir = "/var/lib/arvados/wwwtmp" super.bindir = "/var/lib/arvados/bin" - super.configfile = "/etc/arvados/config.yml" } else { super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") if err != nil { @@ -155,28 +159,28 @@ func (super *Supervisor) run(cfg *arvados.Config) error { if err := os.Mkdir(super.bindir, 0755); err != nil { return err } + } - // Fill in any missing config keys, and write the resulting - // config in the temp dir for child services to use. - err = super.autofillConfig(cfg) - if err != nil { - return err - } - conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer conffile.Close() - err = json.NewEncoder(conffile).Encode(cfg) - if err != nil { - return err - } - err = conffile.Close() - if err != nil { - return err - } - super.configfile = conffile.Name() + // Fill in any missing config keys, and write the resulting + // config in the temp dir for child services to use. + err = super.autofillConfig(cfg) + if err != nil { + return err } + conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer conffile.Close() + err = json.NewEncoder(conffile).Encode(cfg) + if err != nil { + return err + } + err = conffile.Close() + if err != nil { + return err + } + super.configfile = conffile.Name() super.environ = os.Environ() super.cleanEnv([]string{"ARVADOS_"}) @@ -207,13 +211,13 @@ func (super *Supervisor) run(cfg *arvados.Config) error { } else if super.SourceVersion == "" { // Find current source tree version. var buf bytes.Buffer - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat") if err != nil { return err } dirty := buf.Len() > 0 buf.Reset() - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H") if err != nil { return err } @@ -238,19 +242,24 @@ func (super *Supervisor) run(cfg *arvados.Config) error { createCertificates{}, runPostgreSQL{}, runNginx{}, - runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}}, + runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}}, runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP}, runGoProgram{src: "services/health", svc: super.cluster.Services.Health}, runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}}, - runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore}, + runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore}, runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, - runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}}, + runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}}, installPassenger{src: "services/api"}, - runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, - installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup - runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, + runWorkbench2{svc: super.cluster.Services.Workbench2}, seedDatabase{}, } + if !super.NoWorkbench1 { + tasks = append(tasks, + installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + ) + } if super.ClusterType != "test" { tasks = append(tasks, runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, @@ -408,7 +417,7 @@ func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) ( if super.ClusterType == "production" { return binfile, nil } - err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) + err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) return binfile, err } @@ -441,7 +450,7 @@ func (super *Supervisor) setupRubyEnv() error { cmd.Env = super.environ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:... if err != nil || len(buf) == 0 { - return fmt.Errorf("gem env gempath: %v", err) + return fmt.Errorf("gem env gempath: %w", err) } gempath := string(bytes.Split(buf, []byte{':'})[0]) super.prependEnv("PATH", gempath+"/bin:") @@ -471,23 +480,27 @@ func (super *Supervisor) lookPath(prog string) string { return prog } -// Run prog with args, using dir as working directory. If ctx is -// cancelled while the child is running, RunProgram terminates the -// child, waits for it to exit, then returns. +type runOptions struct { + output io.Writer // attach stdout + env []string // add/replace environment variables + user string // run as specified user + stdin io.Reader +} + +// RunProgram runs prog with args, using dir as working directory. If ctx is +// cancelled while the child is running, RunProgram terminates the child, waits +// for it to exit, then returns. // // Child's environment will have our env vars, plus any given in env. // // Child's stdout will be written to output if non-nil, otherwise the // boot command's stderr. -func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error { +func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error { cmdline := fmt.Sprintf("%s", append([]string{prog}, args...)) super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing") logprefix := prog { - if logprefix == "setuidgid" && len(args) >= 3 { - logprefix = args[2] - } innerargs := args if logprefix == "sudo" { for i := 0; i < len(args); i++ { @@ -515,6 +528,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W } cmd := exec.Command(super.lookPath(prog), args...) + cmd.Stdin = opts.stdin stdout, err := cmd.StdoutPipe() if err != nil { return err @@ -532,10 +546,10 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W }() copiers.Add(1) go func() { - if output == nil { + if opts.output == nil { io.Copy(logwriter, stdout) } else { - io.Copy(output, stdout) + io.Copy(opts.output, stdout) } copiers.Done() }() @@ -545,10 +559,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W } else { cmd.Dir = filepath.Join(super.SourcePath, dir) } - env = append([]string(nil), env...) + env := append([]string(nil), opts.env...) env = append(env, super.environ...) cmd.Env = dedupEnv(env) + if opts.user != "" { + // Note: We use this approach instead of "sudo" + // because in certain circumstances (we are pid 1 in a + // docker container, and our passenger child process + // changes to pgid 1) the intermediate sudo process + // notices we have the same pgid as our child and + // refuses to propagate signals from us to our child, + // so we can't signal/shutdown our passenger/rails + // apps. "chpst" or "setuidgid" would work, but these + // few lines avoid depending on runit/daemontools. + u, err := user.Lookup(opts.user) + if err != nil { + return fmt.Errorf("user.Lookup(%q): %w", opts.user, err) + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(uid), + Gid: uint32(gid), + }, + } + } + exited := false defer func() { exited = true }() go func() { @@ -594,32 +632,42 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { return err } usedPort := map[string]bool{} - nextPort := func(host string) string { + nextPort := func(host string) (string, error) { for { port, err := availablePort(host) if err != nil { - panic(err) + port, err = availablePort(super.ListenHost) + } + if err != nil { + return "", err } if usedPort[port] { continue } usedPort[port] = true - return port + return port, nil } } if cluster.Services.Controller.ExternalURL.Host == "" { h, p, err := net.SplitHostPort(super.ControllerAddr) if err != nil { - return err + return fmt.Errorf("SplitHostPort(ControllerAddr): %w", err) } if h == "" { h = super.ListenHost } if p == "0" { - p = nextPort(h) + p, err = nextPort(h) + if err != nil { + return err + } } cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"} } + defaultExtHost, _, err := net.SplitHostPort(cluster.Services.Controller.ExternalURL.Host) + if err != nil { + return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err) + } for _, svc := range []*arvados.Service{ &cluster.Services.Controller, &cluster.Services.DispatchCloud, @@ -632,54 +680,79 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { &cluster.Services.WebDAVDownload, &cluster.Services.Websocket, &cluster.Services.Workbench1, + &cluster.Services.Workbench2, } { if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" { continue } if svc.ExternalURL.Host == "" { + port, err := nextPort(defaultExtHost) + if err != nil { + return err + } + host := net.JoinHostPort(defaultExtHost, port) if svc == &cluster.Services.Controller || svc == &cluster.Services.GitHTTP || svc == &cluster.Services.Health || svc == &cluster.Services.Keepproxy || svc == &cluster.Services.WebDAV || svc == &cluster.Services.WebDAVDownload || - svc == &cluster.Services.Workbench1 { - svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"} + svc == &cluster.Services.Workbench1 || + svc == &cluster.Services.Workbench2 { + svc.ExternalURL = arvados.URL{Scheme: "https", Host: host, Path: "/"} } else if svc == &cluster.Services.Websocket { - svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"} + svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"} } } + if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 { + // When workbench1 is disabled, it gets an + // ExternalURL (so we have a valid listening + // port to write in our Nginx config) but no + // InternalURLs (so health checker doesn't + // complain). + continue + } if len(svc.InternalURLs) == 0 { + port, err := nextPort(super.ListenHost) + if err != nil { + return err + } + host := net.JoinHostPort(super.ListenHost, port) svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{ - arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{}, + {Scheme: "http", Host: host, Path: "/"}: {}, } } } - if cluster.SystemRootToken == "" { - cluster.SystemRootToken = randomHexString(64) - } - if cluster.ManagementToken == "" { - cluster.ManagementToken = randomHexString(64) - } - if cluster.API.RailsSessionSecretToken == "" { - cluster.API.RailsSessionSecretToken = randomHexString(64) - } - if cluster.Collections.BlobSigningKey == "" { - cluster.Collections.BlobSigningKey = randomHexString(64) - } - if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" { - buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) - if err != nil { - return err - } - cluster.Containers.DispatchPrivateKey = string(buf) - } if super.ClusterType != "production" { + if cluster.SystemRootToken == "" { + cluster.SystemRootToken = randomHexString(64) + } + if cluster.ManagementToken == "" { + cluster.ManagementToken = randomHexString(64) + } + if cluster.Collections.BlobSigningKey == "" { + cluster.Collections.BlobSigningKey = randomHexString(64) + } + if cluster.Users.AnonymousUserToken == "" { + cluster.Users.AnonymousUserToken = randomHexString(64) + } + if cluster.Containers.DispatchPrivateKey == "" { + buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) + if err != nil { + return err + } + cluster.Containers.DispatchPrivateKey = string(buf) + } cluster.TLS.Insecure = true } if super.ClusterType == "test" { // Add a second keepstore process. - cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{} + port, err := nextPort(super.ListenHost) + if err != nil { + return err + } + host := net.JoinHostPort(super.ListenHost, port) + cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: host, Path: "/"}] = arvados.ServiceInstance{} // Create a directory-backed volume for each keepstore // process. @@ -699,14 +772,28 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{ url: {}, }, + StorageClasses: map[string]bool{ + "default": true, + "foo": true, + "bar": true, + }, } } + cluster.StorageClasses = map[string]arvados.StorageClassConfig{ + "default": {Default: true}, + "foo": {}, + "bar": {}, + } } if super.OwnTemporaryDatabase { + port, err := nextPort("localhost") + if err != nil { + return err + } cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{ "client_encoding": "utf8", "host": "localhost", - "port": nextPort(super.ListenHost), + "port": port, "dbname": "arvados_test", "user": "arvados", "password": "insecure_arvados_test", @@ -739,21 +826,23 @@ func randomHexString(chars int) string { return fmt.Sprintf("%x", b) } -func internalPort(svc arvados.Service) (string, error) { +func internalPort(svc arvados.Service) (host, port string, err error) { if len(svc.InternalURLs) > 1 { - return "", errors.New("internalPort() doesn't work with multiple InternalURLs") + return "", "", errors.New("internalPort() doesn't work with multiple InternalURLs") } for u := range svc.InternalURLs { u := url.URL(u) - if p := u.Port(); p != "" { - return p, nil - } else if u.Scheme == "https" || u.Scheme == "ws" { - return "443", nil - } else { - return "80", nil + host, port = u.Hostname(), u.Port() + switch { + case port != "": + case u.Scheme == "https", u.Scheme == "ws": + port = "443" + default: + port = "80" } + return } - return "", fmt.Errorf("service has no InternalURLs") + return "", "", fmt.Errorf("service has no InternalURLs") } func externalPort(svc arvados.Service) (string, error) {