1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
27 "git.arvados.org/arvados.git/lib/service"
28 "git.arvados.org/arvados.git/sdk/go/arvados"
29 "git.arvados.org/arvados.git/sdk/go/ctxlog"
30 "git.arvados.org/arvados.git/sdk/go/health"
31 "github.com/sirupsen/logrus"
34 type Supervisor struct {
35 SourcePath string // e.g., /home/username/src/arvados
36 SourceVersion string // e.g., acbd1324...
37 ClusterType string // e.g., production
38 ListenHost string // e.g., localhost
39 ControllerAddr string // e.g., 127.0.0.1:8000
40 OwnTemporaryDatabase bool
43 logger logrus.FieldLogger
44 cluster *arvados.Cluster
47 cancel context.CancelFunc
49 healthChecker *health.Aggregator
50 tasksReady map[string]chan bool
51 waitShutdown sync.WaitGroup
55 environ []string // for child processes
58 func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
59 super.ctx, super.cancel = context.WithCancel(ctx)
60 super.done = make(chan struct{})
63 sigch := make(chan os.Signal)
64 signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
65 defer signal.Stop(sigch)
67 for sig := range sigch {
68 super.logger.WithField("signal", sig).Info("caught signal")
75 fmt.Fprintln(super.Stderr, err)
81 func (super *Supervisor) run(cfg *arvados.Config) error {
82 cwd, err := os.Getwd()
86 if !strings.HasPrefix(super.SourcePath, "/") {
87 super.SourcePath = filepath.Join(cwd, super.SourcePath)
89 super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
94 super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
98 defer os.RemoveAll(super.tempdir)
99 if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0777); err != nil {
103 // Fill in any missing config keys, and write the resulting
104 // config in the temp dir for child services to use.
105 err = super.autofillConfig(cfg, super.logger)
109 conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0777)
113 defer conffile.Close()
114 err = json.NewEncoder(conffile).Encode(cfg)
118 err = conffile.Close()
122 super.configfile = conffile.Name()
124 super.environ = os.Environ()
126 super.setEnv("ARVADOS_CONFIG", super.configfile)
127 super.setEnv("RAILS_ENV", super.ClusterType)
128 super.setEnv("TMPDIR", super.tempdir)
129 super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
131 super.cluster, err = cfg.GetCluster("")
135 // Now that we have the config, replace the bootstrap logger
136 // with a new one according to the logging config.
137 loglevel := super.cluster.SystemLogs.LogLevel
138 if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
141 super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
145 if super.SourceVersion == "" {
146 // Find current source tree version.
148 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
152 dirty := buf.Len() > 0
154 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
158 super.SourceVersion = strings.TrimSpace(buf.String())
160 super.SourceVersion += "+uncommitted"
163 return errors.New("specifying a version to run is not yet supported")
166 _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
170 err = super.setupRubyEnv()
175 tasks := []supervisedTask{
176 createCertificates{},
179 runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
180 runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
181 runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
182 runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
183 runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
184 runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
185 runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
186 installPassenger{src: "services/api"},
187 runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
188 installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
189 runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
192 if super.ClusterType != "test" {
193 tasks = append(tasks,
194 runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
195 runGoProgram{src: "services/keep-balance"},
198 super.tasksReady = map[string]chan bool{}
199 for _, task := range tasks {
200 super.tasksReady[task.String()] = make(chan bool)
202 for _, task := range tasks {
204 fail := func(err error) {
205 if super.ctx.Err() != nil {
209 super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
212 super.logger.WithField("task", task.String()).Info("starting")
213 err := task.Run(super.ctx, fail, super)
218 close(super.tasksReady[task.String()])
221 err = super.wait(super.ctx, tasks...)
225 super.logger.Info("all startup tasks are complete; starting health checks")
226 super.healthChecker = &health.Aggregator{Cluster: super.cluster}
228 super.logger.Info("shutting down")
229 super.waitShutdown.Wait()
230 return super.ctx.Err()
233 func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
234 for _, task := range tasks {
235 ch, ok := super.tasksReady[task.String()]
237 return fmt.Errorf("no such task: %s", task)
239 super.logger.WithField("task", task.String()).Info("waiting")
242 super.logger.WithField("task", task.String()).Info("ready")
244 super.logger.WithField("task", task.String()).Info("task was never ready")
251 func (super *Supervisor) Stop() {
256 func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
257 ticker := time.NewTicker(time.Second)
259 for waiting := "all"; waiting != ""; {
262 case <-super.ctx.Done():
265 if super.healthChecker == nil {
269 resp := super.healthChecker.ClusterHealth()
270 // The overall health check (resp.Health=="OK") might
271 // never pass due to missing components (like
272 // arvados-dispatch-cloud in a test cluster), so
273 // instead we wait for all configured components to
276 for target, check := range resp.Checks {
277 if check.Health != "OK" {
278 waiting += " " + target
282 super.logger.WithField("targets", waiting[1:]).Info("waiting")
285 u := super.cluster.Services.Controller.ExternalURL
289 func (super *Supervisor) prependEnv(key, prepend string) {
290 for i, s := range super.environ {
291 if strings.HasPrefix(s, key+"=") {
292 super.environ[i] = key + "=" + prepend + s[len(key)+1:]
296 super.environ = append(super.environ, key+"="+prepend)
299 var cleanEnvPrefixes = []string{
305 func (super *Supervisor) cleanEnv() {
307 for _, s := range super.environ {
309 for _, p := range cleanEnvPrefixes {
310 if strings.HasPrefix(s, p) {
316 cleaned = append(cleaned, s)
319 super.environ = cleaned
322 func (super *Supervisor) setEnv(key, val string) {
323 for i, s := range super.environ {
324 if strings.HasPrefix(s, key+"=") {
325 super.environ[i] = key + "=" + val
329 super.environ = append(super.environ, key+"="+val)
332 // Remove all but the first occurrence of each env var.
333 func dedupEnv(in []string) []string {
334 saw := map[string]bool{}
336 for _, kv := range in {
337 if split := strings.Index(kv, "="); split < 1 {
338 panic("invalid environment var: " + kv)
339 } else if saw[kv[:split]] {
342 saw[kv[:split]] = true
343 out = append(out, kv)
349 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
350 _, basename := filepath.Split(srcpath)
351 bindir := filepath.Join(super.tempdir, "bin")
352 binfile := filepath.Join(bindir, basename)
353 err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
357 func (super *Supervisor) setupRubyEnv() error {
358 cmd := exec.Command("gem", "env", "gempath")
359 cmd.Env = super.environ
360 buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
361 if err != nil || len(buf) == 0 {
362 return fmt.Errorf("gem env gempath: %v", err)
364 gempath := string(bytes.Split(buf, []byte{':'})[0])
365 super.prependEnv("PATH", gempath+"/bin:")
366 super.setEnv("GEM_HOME", gempath)
367 super.setEnv("GEM_PATH", gempath)
368 // Passenger install doesn't work unless $HOME is ~user
369 u, err := user.Current()
373 super.setEnv("HOME", u.HomeDir)
377 func (super *Supervisor) lookPath(prog string) string {
378 for _, val := range super.environ {
379 if strings.HasPrefix(val, "PATH=") {
380 for _, dir := range filepath.SplitList(val[5:]) {
381 path := filepath.Join(dir, prog)
382 if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
391 // Run prog with args, using dir as working directory. If ctx is
392 // cancelled while the child is running, RunProgram terminates the
393 // child, waits for it to exit, then returns.
395 // Child's environment will have our env vars, plus any given in env.
397 // Child's stdout will be written to output if non-nil, otherwise the
398 // boot command's stderr.
399 func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
400 cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
401 super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
403 logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
404 if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
406 } else if logprefix == "arvados-server" && len(args) > 1 {
409 if !strings.HasPrefix(dir, "/") {
410 logprefix = dir + ": " + logprefix
413 cmd := exec.Command(super.lookPath(prog), args...)
414 stdout, err := cmd.StdoutPipe()
418 stderr, err := cmd.StderrPipe()
422 logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
423 var copiers sync.WaitGroup
426 io.Copy(logwriter, stderr)
432 io.Copy(logwriter, stdout)
434 io.Copy(output, stdout)
439 if strings.HasPrefix(dir, "/") {
442 cmd.Dir = filepath.Join(super.SourcePath, dir)
444 env = append([]string(nil), env...)
445 env = append(env, super.environ...)
446 cmd.Env = dedupEnv(env)
449 defer func() { exited = true }()
452 log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
454 if cmd.Process == nil {
455 log.Debug("waiting for child process to start")
456 time.Sleep(time.Second / 2)
458 log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
459 cmd.Process.Signal(syscall.SIGTERM)
460 time.Sleep(5 * time.Second)
464 log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
476 if ctx.Err() != nil {
477 // Return "context canceled", instead of the "killed"
478 // error that was probably caused by the context being
481 } else if err != nil {
482 return fmt.Errorf("%s: error: %v", cmdline, err)
487 func (super *Supervisor) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger) error {
488 cluster, err := cfg.GetCluster("")
492 usedPort := map[string]bool{}
493 nextPort := func() string {
495 port, err := availablePort(super.ListenHost)
502 usedPort[port] = true
506 if cluster.Services.Controller.ExternalURL.Host == "" {
507 h, p, err := net.SplitHostPort(super.ControllerAddr)
515 p, err = availablePort(h)
521 cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
523 for _, svc := range []*arvados.Service{
524 &cluster.Services.Controller,
525 &cluster.Services.DispatchCloud,
526 &cluster.Services.GitHTTP,
527 &cluster.Services.Health,
528 &cluster.Services.Keepproxy,
529 &cluster.Services.Keepstore,
530 &cluster.Services.RailsAPI,
531 &cluster.Services.WebDAV,
532 &cluster.Services.WebDAVDownload,
533 &cluster.Services.Websocket,
534 &cluster.Services.Workbench1,
536 if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
539 if svc.ExternalURL.Host == "" && (svc == &cluster.Services.Controller ||
540 svc == &cluster.Services.GitHTTP ||
541 svc == &cluster.Services.Keepproxy ||
542 svc == &cluster.Services.WebDAV ||
543 svc == &cluster.Services.WebDAVDownload ||
544 svc == &cluster.Services.Websocket ||
545 svc == &cluster.Services.Workbench1) {
546 svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}
548 if len(svc.InternalURLs) == 0 {
549 svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
550 arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}: arvados.ServiceInstance{},
554 if cluster.SystemRootToken == "" {
555 cluster.SystemRootToken = randomHexString(64)
557 if cluster.ManagementToken == "" {
558 cluster.ManagementToken = randomHexString(64)
560 if cluster.API.RailsSessionSecretToken == "" {
561 cluster.API.RailsSessionSecretToken = randomHexString(64)
563 if cluster.Collections.BlobSigningKey == "" {
564 cluster.Collections.BlobSigningKey = randomHexString(64)
566 if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
567 buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
571 cluster.Containers.DispatchPrivateKey = string(buf)
573 if super.ClusterType != "production" {
574 cluster.TLS.Insecure = true
576 if super.ClusterType == "test" {
577 // Add a second keepstore process.
578 cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}] = arvados.ServiceInstance{}
580 // Create a directory-backed volume for each keepstore
582 cluster.Volumes = map[string]arvados.Volume{}
583 for url := range cluster.Services.Keepstore.InternalURLs {
584 volnum := len(cluster.Volumes)
585 datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
586 if _, err = os.Stat(datadir + "/."); err == nil {
587 } else if !os.IsNotExist(err) {
589 } else if err = os.Mkdir(datadir, 0777); err != nil {
592 cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
594 DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
595 AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
601 if super.OwnTemporaryDatabase {
602 cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
603 "client_encoding": "utf8",
606 "dbname": "arvados_test",
608 "password": "insecure_arvados_test",
612 cfg.Clusters[cluster.ClusterID] = *cluster
616 func addrIsLocal(addr string) (bool, error) {
618 listener, err := net.Listen("tcp", addr)
622 } else if strings.Contains(err.Error(), "cannot assign requested address") {
629 func randomHexString(chars int) string {
630 b := make([]byte, chars/2)
631 _, err := rand.Read(b)
635 return fmt.Sprintf("%x", b)
638 func internalPort(svc arvados.Service) (string, error) {
639 if len(svc.InternalURLs) > 1 {
640 return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
642 for u := range svc.InternalURLs {
643 if _, p, err := net.SplitHostPort(u.Host); err != nil {
647 } else if u.Scheme == "https" {
653 return "", fmt.Errorf("service has no InternalURLs")
656 func externalPort(svc arvados.Service) (string, error) {
657 if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
661 } else if svc.ExternalURL.Scheme == "https" {
668 func availablePort(host string) (string, error) {
669 ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
674 _, port, err := net.SplitHostPort(ln.Addr().String())
681 // Try to connect to addr until it works, then close ch. Give up if
683 func waitForConnect(ctx context.Context, addr string) error {
684 dialer := net.Dialer{Timeout: time.Second}
685 for ctx.Err() == nil {
686 conn, err := dialer.DialContext(ctx, "tcp", addr)
688 time.Sleep(time.Second / 10)