"os/user"
"path/filepath"
"reflect"
+ "strconv"
"strings"
"sync"
"syscall"
ClusterType string // e.g., production
ListenHost string // e.g., localhost
ControllerAddr string // e.g., 127.0.0.1:8000
+ Workbench2Source string // e.g., /home/username/src/arvados-workbench2
+ NoWorkbench1 bool
+ NoWorkbench2 bool
OwnTemporaryDatabase bool
Stderr io.Writer
waitShutdown sync.WaitGroup
bindir string
- tempdir string
- wwwtempdir string
+ tempdir string // in production mode, this is accessible only to root
+ wwwtempdir string // in production mode, this is accessible only to www-data
configfile string
environ []string // for child processes
}
+func (super *Supervisor) Cluster() *arvados.Cluster { return super.cluster }
+
func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config, cfgPath string) {
super.ctx, super.cancel = context.WithCancel(ctx)
super.done = make(chan struct{})
if err != nil {
return err
}
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
} else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat")
if err != nil {
return err
}
dirty := buf.Len() > 0
buf.Reset()
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H")
if err != nil {
return err
}
createCertificates{},
runPostgreSQL{},
runNginx{},
- runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
+ runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
- runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
- runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
+ runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
+ runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
- runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
+ runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
- installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
- runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
seedDatabase{},
}
+ if !super.NoWorkbench1 {
+ tasks = append(tasks,
+ installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ )
+ }
+ if !super.NoWorkbench2 {
+ tasks = append(tasks,
+ runWorkbench2{svc: super.cluster.Services.Workbench2},
+ )
+ }
if super.ClusterType != "test" {
tasks = append(tasks,
- runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
if super.ClusterType == "production" {
return binfile, nil
}
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
- return fmt.Errorf("gem env gempath: %v", err)
+ return fmt.Errorf("gem env gempath: %w", err)
}
gempath := string(bytes.Split(buf, []byte{':'})[0])
super.prependEnv("PATH", gempath+"/bin:")
return prog
}
-// Run prog with args, using dir as working directory. If ctx is
-// cancelled while the child is running, RunProgram terminates the
-// child, waits for it to exit, then returns.
+type runOptions struct {
+ output io.Writer // attach stdout
+ env []string // add/replace environment variables
+ user string // run as specified user
+ stdin io.Reader
+}
+
+// RunProgram runs prog with args, using dir as working directory. If ctx is
+// cancelled while the child is running, RunProgram terminates the child, waits
+// for it to exit, then returns.
//
// Child's environment will have our env vars, plus any given in env.
//
// Child's stdout will be written to output if non-nil, otherwise the
// boot command's stderr.
-func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error {
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
{
- if logprefix == "setuidgid" && len(args) >= 3 {
- logprefix = args[2]
- }
innerargs := args
if logprefix == "sudo" {
for i := 0; i < len(args); i++ {
}
cmd := exec.Command(super.lookPath(prog), args...)
+ cmd.Stdin = opts.stdin
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}()
copiers.Add(1)
go func() {
- if output == nil {
+ if opts.output == nil {
io.Copy(logwriter, stdout)
} else {
- io.Copy(output, stdout)
+ io.Copy(opts.output, stdout)
}
copiers.Done()
}()
} else {
cmd.Dir = filepath.Join(super.SourcePath, dir)
}
- env = append([]string(nil), env...)
+ env := append([]string(nil), opts.env...)
env = append(env, super.environ...)
cmd.Env = dedupEnv(env)
+ if opts.user != "" {
+ // Note: We use this approach instead of "sudo"
+ // because in certain circumstances (we are pid 1 in a
+ // docker container, and our passenger child process
+ // changes to pgid 1) the intermediate sudo process
+ // notices we have the same pgid as our child and
+ // refuses to propagate signals from us to our child,
+ // so we can't signal/shutdown our passenger/rails
+ // apps. "chpst" or "setuidgid" would work, but these
+ // few lines avoid depending on runit/daemontools.
+ u, err := user.Lookup(opts.user)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ Credential: &syscall.Credential{
+ Uid: uint32(uid),
+ Gid: uint32(gid),
+ },
+ }
+ }
+
exited := false
defer func() { exited = true }()
go func() {
return err
}
usedPort := map[string]bool{}
- nextPort := func(host string) string {
+ nextPort := func(host string) (string, error) {
for {
port, err := availablePort(host)
if err != nil {
- panic(err)
+ port, err = availablePort(super.ListenHost)
+ }
+ if err != nil {
+ return "", err
}
if usedPort[port] {
continue
}
usedPort[port] = true
- return port
+ return port, nil
}
}
if cluster.Services.Controller.ExternalURL.Host == "" {
h, p, err := net.SplitHostPort(super.ControllerAddr)
if err != nil {
- return err
+ return fmt.Errorf("SplitHostPort(ControllerAddr): %w", err)
}
if h == "" {
h = super.ListenHost
}
if p == "0" {
- p = nextPort(h)
+ p, err = nextPort(h)
+ if err != nil {
+ return err
+ }
}
cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
+ u := url.URL(cluster.Services.Controller.ExternalURL)
+ defaultExtHost := u.Hostname()
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
&cluster.Services.DispatchCloud,
&cluster.Services.WebDAVDownload,
&cluster.Services.Websocket,
&cluster.Services.Workbench1,
+ &cluster.Services.Workbench2,
} {
if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
continue
}
if svc.ExternalURL.Host == "" {
+ port, err := nextPort(defaultExtHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(defaultExtHost, port)
if svc == &cluster.Services.Controller ||
svc == &cluster.Services.GitHTTP ||
svc == &cluster.Services.Health ||
svc == &cluster.Services.Keepproxy ||
svc == &cluster.Services.WebDAV ||
svc == &cluster.Services.WebDAVDownload ||
- svc == &cluster.Services.Workbench1 {
- svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
+ svc == &cluster.Services.Workbench1 ||
+ svc == &cluster.Services.Workbench2 {
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: host, Path: "/"}
} else if svc == &cluster.Services.Websocket {
- svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"}
}
}
+ if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 ||
+ super.NoWorkbench2 && svc == &cluster.Services.Workbench2 {
+ // When workbench1 is disabled, it gets an
+ // ExternalURL (so we have a valid listening
+ // port to write in our Nginx config) but no
+ // InternalURLs (so health checker doesn't
+ // complain).
+ continue
+ }
if len(svc.InternalURLs) == 0 {
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
- arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{},
+ {Scheme: "http", Host: host, Path: "/"}: {},
}
}
}
- if cluster.SystemRootToken == "" {
- cluster.SystemRootToken = randomHexString(64)
- }
- if cluster.ManagementToken == "" {
- cluster.ManagementToken = randomHexString(64)
- }
- if cluster.API.RailsSessionSecretToken == "" {
- cluster.API.RailsSessionSecretToken = randomHexString(64)
- }
- if cluster.Collections.BlobSigningKey == "" {
- cluster.Collections.BlobSigningKey = randomHexString(64)
- }
- if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
- buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
- if err != nil {
- return err
- }
- cluster.Containers.DispatchPrivateKey = string(buf)
- }
if super.ClusterType != "production" {
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
+ if cluster.Users.AnonymousUserToken == "" {
+ cluster.Users.AnonymousUserToken = randomHexString(64)
+ }
+ if cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
cluster.TLS.Insecure = true
}
if super.ClusterType == "test" {
// Add a second keepstore process.
- cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: host, Path: "/"}] = arvados.ServiceInstance{}
// Create a directory-backed volume for each keepstore
// process.
AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
url: {},
},
+ StorageClasses: map[string]bool{
+ "default": true,
+ "foo": true,
+ "bar": true,
+ },
}
}
+ cluster.StorageClasses = map[string]arvados.StorageClassConfig{
+ "default": {Default: true},
+ "foo": {},
+ "bar": {},
+ }
}
if super.OwnTemporaryDatabase {
+ port, err := nextPort("localhost")
+ if err != nil {
+ return err
+ }
cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
"client_encoding": "utf8",
"host": "localhost",
- "port": nextPort(super.ListenHost),
+ "port": port,
"dbname": "arvados_test",
"user": "arvados",
"password": "insecure_arvados_test",
}
func addrIsLocal(addr string) (bool, error) {
- return true, nil
listener, err := net.Listen("tcp", addr)
if err == nil {
listener.Close()
return fmt.Sprintf("%x", b)
}
-func internalPort(svc arvados.Service) (string, error) {
+func internalPort(svc arvados.Service) (host, port string, err error) {
if len(svc.InternalURLs) > 1 {
- return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
+ return "", "", errors.New("internalPort() doesn't work with multiple InternalURLs")
}
for u := range svc.InternalURLs {
u := url.URL(u)
- if p := u.Port(); p != "" {
- return p, nil
- } else if u.Scheme == "https" || u.Scheme == "ws" {
- return "443", nil
- } else {
- return "80", nil
+ host, port = u.Hostname(), u.Port()
+ switch {
+ case port != "":
+ case u.Scheme == "https", u.Scheme == "ws":
+ port = "443"
+ default:
+ port = "80"
}
+ return
}
- return "", fmt.Errorf("service has no InternalURLs")
+ return "", "", fmt.Errorf("service has no InternalURLs")
}
func externalPort(svc arvados.Service) (string, error) {
projects / arvados.git / blobdiff