From: Tom Clegg Date: Mon, 18 Jan 2021 18:55:27 +0000 (-0500) Subject: 16306: Merge branch 'master' X-Git-Tag: 2.2.0~141^2~17 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/41305b5ac71cc9a306dc654c42c11ffcc4258a47?hp=-c 16306: Merge branch 'master' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- 41305b5ac71cc9a306dc654c42c11ffcc4258a47 diff --combined lib/boot/supervisor.go index e892d3e623,f2e715a766..838808df58 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@@ -14,14 -14,12 +14,14 @@@ import "io" "io/ioutil" "net" + "net/url" "os" "os/exec" "os/signal" "os/user" "path/filepath" "reflect" + "strconv" "strings" "sync" "syscall" @@@ -56,9 -54,7 +56,9 @@@ type Supervisor struct tasksReady map[string]chan bool waitShutdown sync.WaitGroup + bindir string tempdir string + wwwtempdir string configfile string environ []string // for child processes } @@@ -135,26 -131,13 +135,26 @@@ func (super *Supervisor) run(cfg *arvad return err } - super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") - if err != nil { - return err - } - defer os.RemoveAll(super.tempdir) - if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil { - return err + // Choose bin and temp dirs: /var/lib/arvados/... in + // production, transient tempdir otherwise. + if super.ClusterType == "production" { + // These dirs have already been created by + // "arvados-server install" (or by extracting a + // package). + super.tempdir = "/var/lib/arvados/tmp" + super.wwwtempdir = "/var/lib/arvados/wwwtmp" + super.bindir = "/var/lib/arvados/bin" + } else { + super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") + if err != nil { + return err + } + defer os.RemoveAll(super.tempdir) + super.wwwtempdir = super.tempdir + super.bindir = filepath.Join(super.tempdir, "bin") + if err := os.Mkdir(super.bindir, 0755); err != nil { + return err + } } // Fill in any missing config keys, and write the resulting @@@ -163,7 -146,7 +163,7 @@@ if err != nil { return err } - conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) + conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } @@@ -183,10 -166,7 +183,10 @@@ super.setEnv("ARVADOS_CONFIG", super.configfile) super.setEnv("RAILS_ENV", super.ClusterType) super.setEnv("TMPDIR", super.tempdir) - super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:") + super.prependEnv("PATH", "/var/lib/arvados/bin:") + if super.ClusterType != "production" { + super.prependEnv("PATH", super.tempdir+"/bin:") + } super.cluster, err = cfg.GetCluster("") if err != nil { @@@ -202,18 -182,16 +202,18 @@@ "PID": os.Getpid(), }) - if super.SourceVersion == "" { + if super.SourceVersion == "" && super.ClusterType == "production" { + // don't need SourceVersion + } else if super.SourceVersion == "" { // Find current source tree version. var buf bytes.Buffer - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat") if err != nil { return err } dirty := buf.Len() > 0 buf.Reset() - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H") if err != nil { return err } @@@ -238,23 -216,23 +238,23 @@@ createCertificates{}, runPostgreSQL{}, runNginx{}, - runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}}, + runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}}, runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP}, runGoProgram{src: "services/health", svc: super.cluster.Services.Health}, runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}}, runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore}, runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, - runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}}, + runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}}, installPassenger{src: "services/api"}, - runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, - installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup - runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, ++ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, + installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup - runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, seedDatabase{}, } if super.ClusterType != "test" { tasks = append(tasks, - runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller}, - runGoProgram{src: "services/keep-balance"}, + runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, + runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance}, ) } super.tasksReady = map[string]chan bool{} @@@ -404,11 -382,9 +404,11 @@@ func dedupEnv(in []string) []string func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) { _, basename := filepath.Split(srcpath) - bindir := filepath.Join(super.tempdir, "bin") - binfile := filepath.Join(bindir, basename) - err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) + binfile := filepath.Join(super.bindir, basename) + if super.ClusterType == "production" { + return binfile, nil + } + err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) return binfile, err } @@@ -425,19 -401,10 +425,19 @@@ func (super *Supervisor) setupRubyEnv( "GEM_PATH=", }) gem := "gem" - if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil { + if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" { gem = "/var/lib/arvados/bin/gem" } cmd := exec.Command(gem, "env", "gempath") + if super.ClusterType == "production" { + cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...) + path, err := exec.LookPath("sudo") + if err != nil { + return fmt.Errorf("LookPath(\"sudo\"): %w", err) + } + cmd.Path = path + } + cmd.Stderr = super.Stderr cmd.Env = super.environ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:... if err != nil || len(buf) == 0 { @@@ -471,12 -438,6 +471,12 @@@ func (super *Supervisor) lookPath(prog return prog } +type runOptions struct { + output io.Writer // attach stdout + env []string // add/replace environment variables + user string // run as specified user +} + // RunProgram runs prog with args, using dir as working directory. If ctx is // cancelled while the child is running, RunProgram terminates the child, waits // for it to exit, then returns. @@@ -485,36 -446,22 +485,36 @@@ // // Child's stdout will be written to output if non-nil, otherwise the // boot command's stderr. -func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error { +func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error { cmdline := fmt.Sprintf("%s", append([]string{prog}, args...)) super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing") logprefix := prog - if logprefix == "setuidgid" && len(args) >= 2 { - logprefix = args[1] - } - logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") - if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" { - logprefix = args[1] - } else if logprefix == "arvados-server" && len(args) > 1 { - logprefix = args[0] - } - if !strings.HasPrefix(dir, "/") { - logprefix = dir + ": " + logprefix + { + innerargs := args + if logprefix == "sudo" { + for i := 0; i < len(args); i++ { + if args[i] == "-u" { + i++ + } else if args[i] == "-E" || strings.Contains(args[i], "=") { + } else { + logprefix = args[i] + innerargs = args[i+1:] + break + } + } + } + logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/") + logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") + if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" { + _, dirbase := filepath.Split(dir) + logprefix = innerargs[1] + "@" + dirbase + } else if logprefix == "arvados-server" && len(args) > 1 { + logprefix = args[0] + } + if !strings.HasPrefix(dir, "/") { + logprefix = dir + ": " + logprefix + } } cmd := exec.Command(super.lookPath(prog), args...) @@@ -535,10 -482,10 +535,10 @@@ }() copiers.Add(1) go func() { - if output == nil { + if opts.output == nil { io.Copy(logwriter, stdout) } else { - io.Copy(output, stdout) + io.Copy(opts.output, stdout) } copiers.Done() }() @@@ -548,34 -495,10 +548,34 @@@ } else { cmd.Dir = filepath.Join(super.SourcePath, dir) } - env = append([]string(nil), env...) + env := append([]string(nil), opts.env...) env = append(env, super.environ...) cmd.Env = dedupEnv(env) + if opts.user != "" { + // Note: We use this approach instead of "sudo" + // because in certain circumstances (we are pid 1 in a + // docker container, and our passenger child process + // changes to pgid 1) the intermediate sudo process + // notices we have the same pgid as our child and + // refuses to propagate signals from us to our child, + // so we can't signal/shutdown our passenger/rails + // apps. "chpst" or "setuidgid" would work, but these + // few lines avoid depending on runit/daemontools. + u, err := user.Lookup(opts.user) + if err != nil { + return fmt.Errorf("user.Lookup(%q): %w", opts.user, err) + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(uid), + Gid: uint32(gid), + }, + } + } + exited := false defer func() { exited = true }() go func() { @@@ -682,26 -605,27 +682,26 @@@ func (super *Supervisor) autofillConfig } } } - if cluster.SystemRootToken == "" { - cluster.SystemRootToken = randomHexString(64) - } - if cluster.ManagementToken == "" { - cluster.ManagementToken = randomHexString(64) - } - if cluster.Collections.BlobSigningKey == "" { - cluster.Collections.BlobSigningKey = randomHexString(64) - } - if cluster.Users.AnonymousUserToken == "" { - cluster.Users.AnonymousUserToken = randomHexString(64) - } - - if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" { - buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) - if err != nil { - return err - } - cluster.Containers.DispatchPrivateKey = string(buf) - } if super.ClusterType != "production" { + if cluster.SystemRootToken == "" { + cluster.SystemRootToken = randomHexString(64) + } + if cluster.ManagementToken == "" { + cluster.ManagementToken = randomHexString(64) + } - if cluster.API.RailsSessionSecretToken == "" { - cluster.API.RailsSessionSecretToken = randomHexString(64) - } + if cluster.Collections.BlobSigningKey == "" { + cluster.Collections.BlobSigningKey = randomHexString(64) + } ++ if cluster.Users.AnonymousUserToken == "" { ++ cluster.Users.AnonymousUserToken = randomHexString(64) ++ } + if cluster.Containers.DispatchPrivateKey == "" { + buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) + if err != nil { + return err + } + cluster.Containers.DispatchPrivateKey = string(buf) + } cluster.TLS.Insecure = true } if super.ClusterType == "test" { @@@ -771,10 -695,11 +771,10 @@@ func internalPort(svc arvados.Service) return "", errors.New("internalPort() doesn't work with multiple InternalURLs") } for u := range svc.InternalURLs { - if _, p, err := net.SplitHostPort(u.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(u) + if p := u.Port(); p != "" { return p, nil - } else if u.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "ws" { return "443", nil } else { return "80", nil @@@ -784,10 -709,11 +784,10 @@@ } func externalPort(svc arvados.Service) (string, error) { - if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(svc.ExternalURL) + if p := u.Port(); p != "" { return p, nil - } else if svc.ExternalURL.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "wss" { return "443", nil } else { return "80", nil