lib/boot/supervisor.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package boot
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "os"
  18         "os/exec"
  19         "os/signal"
  20         "os/user"
  21         "path/filepath"
  22         "strings"
  23         "sync"
  24         "syscall"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/lib/service"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/health"
  31         "github.com/sirupsen/logrus"
  32 )
  33
  34 type Supervisor struct {
  35         SourcePath           string // e.g., /home/username/src/arvados
  36         SourceVersion        string // e.g., acbd1324...
  37         ClusterType          string // e.g., production
  38         ListenHost           string // e.g., localhost
  39         ControllerAddr       string // e.g., 127.0.0.1:8000
  40         OwnTemporaryDatabase bool
  41         Stderr               io.Writer
  42
  43         logger  logrus.FieldLogger
  44         cluster *arvados.Cluster
  45
  46         ctx           context.Context
  47         cancel        context.CancelFunc
  48         done          chan struct{}
  49         healthChecker *health.Aggregator
  50         tasksReady    map[string]chan bool
  51         waitShutdown  sync.WaitGroup
  52
  53         tempdir    string
  54         configfile string
  55         environ    []string // for child processes
  56 }
  57
  58 func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
  59         super.ctx, super.cancel = context.WithCancel(ctx)
  60         super.done = make(chan struct{})
  61
  62         go func() {
  63                 sigch := make(chan os.Signal)
  64                 signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
  65                 defer signal.Stop(sigch)
  66                 go func() {
  67                         for sig := range sigch {
  68                                 super.logger.WithField("signal", sig).Info("caught signal")
  69                                 super.cancel()
  70                         }
  71                 }()
  72
  73                 err := super.run(cfg)
  74                 if err != nil {
  75                         super.logger.WithError(err).Warn("supervisor shut down")
  76                 }
  77                 close(super.done)
  78         }()
  79 }
  80
  81 func (super *Supervisor) run(cfg *arvados.Config) error {
  82         cwd, err := os.Getwd()
  83         if err != nil {
  84                 return err
  85         }
  86         if !strings.HasPrefix(super.SourcePath, "/") {
  87                 super.SourcePath = filepath.Join(cwd, super.SourcePath)
  88         }
  89         super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
  90         if err != nil {
  91                 return err
  92         }
  93
  94         super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
  95         if err != nil {
  96                 return err
  97         }
  98         defer os.RemoveAll(super.tempdir)
  99         if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
 100                 return err
 101         }
 102
 103         // Fill in any missing config keys, and write the resulting
 104         // config in the temp dir for child services to use.
 105         err = super.autofillConfig(cfg)
 106         if err != nil {
 107                 return err
 108         }
 109         conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
 110         if err != nil {
 111                 return err
 112         }
 113         defer conffile.Close()
 114         err = json.NewEncoder(conffile).Encode(cfg)
 115         if err != nil {
 116                 return err
 117         }
 118         err = conffile.Close()
 119         if err != nil {
 120                 return err
 121         }
 122         super.configfile = conffile.Name()
 123
 124         super.environ = os.Environ()
 125         super.cleanEnv([]string{"ARVADOS_"})
 126         super.setEnv("ARVADOS_CONFIG", super.configfile)
 127         super.setEnv("RAILS_ENV", super.ClusterType)
 128         super.setEnv("TMPDIR", super.tempdir)
 129         super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
 130
 131         super.cluster, err = cfg.GetCluster("")
 132         if err != nil {
 133                 return err
 134         }
 135         // Now that we have the config, replace the bootstrap logger
 136         // with a new one according to the logging config.
 137         loglevel := super.cluster.SystemLogs.LogLevel
 138         if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
 139                 loglevel = "debug"
 140         }
 141         super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
 142                 "PID": os.Getpid(),
 143         })
 144
 145         if super.SourceVersion == "" {
 146                 // Find current source tree version.
 147                 var buf bytes.Buffer
 148                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
 149                 if err != nil {
 150                         return err
 151                 }
 152                 dirty := buf.Len() > 0
 153                 buf.Reset()
 154                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
 155                 if err != nil {
 156                         return err
 157                 }
 158                 super.SourceVersion = strings.TrimSpace(buf.String())
 159                 if dirty {
 160                         super.SourceVersion += "+uncommitted"
 161                 }
 162         } else {
 163                 return errors.New("specifying a version to run is not yet supported")
 164         }
 165
 166         _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
 167         if err != nil {
 168                 return err
 169         }
 170         err = super.setupRubyEnv()
 171         if err != nil {
 172                 return err
 173         }
 174
 175         tasks := []supervisedTask{
 176                 createCertificates{},
 177                 runPostgreSQL{},
 178                 runNginx{},
 179                 runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
 180                 runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
 181                 runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
 182                 runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
 183                 runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
 184                 runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 185                 runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 186                 installPassenger{src: "services/api"},
 187                 runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 188                 installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
 189                 runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 190                 seedDatabase{},
 191         }
 192         if super.ClusterType != "test" {
 193                 tasks = append(tasks,
 194                         runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
 195                         runGoProgram{src: "services/keep-balance"},
 196                 )
 197         }
 198         super.tasksReady = map[string]chan bool{}
 199         for _, task := range tasks {
 200                 super.tasksReady[task.String()] = make(chan bool)
 201         }
 202         for _, task := range tasks {
 203                 task := task
 204                 fail := func(err error) {
 205                         if super.ctx.Err() != nil {
 206                                 return
 207                         }
 208                         super.cancel()
 209                         super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
 210                 }
 211                 go func() {
 212                         super.logger.WithField("task", task.String()).Info("starting")
 213                         err := task.Run(super.ctx, fail, super)
 214                         if err != nil {
 215                                 fail(err)
 216                                 return
 217                         }
 218                         close(super.tasksReady[task.String()])
 219                 }()
 220         }
 221         err = super.wait(super.ctx, tasks...)
 222         if err != nil {
 223                 return err
 224         }
 225         super.logger.Info("all startup tasks are complete; starting health checks")
 226         super.healthChecker = &health.Aggregator{Cluster: super.cluster}
 227         <-super.ctx.Done()
 228         super.logger.Info("shutting down")
 229         super.waitShutdown.Wait()
 230         return super.ctx.Err()
 231 }
 232
 233 func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
 234         for _, task := range tasks {
 235                 ch, ok := super.tasksReady[task.String()]
 236                 if !ok {
 237                         return fmt.Errorf("no such task: %s", task)
 238                 }
 239                 super.logger.WithField("task", task.String()).Info("waiting")
 240                 select {
 241                 case <-ch:
 242                         super.logger.WithField("task", task.String()).Info("ready")
 243                 case <-ctx.Done():
 244                         super.logger.WithField("task", task.String()).Info("task was never ready")
 245                         return ctx.Err()
 246                 }
 247         }
 248         return nil
 249 }
 250
 251 func (super *Supervisor) Stop() {
 252         super.cancel()
 253         <-super.done
 254 }
 255
 256 func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
 257         ticker := time.NewTicker(time.Second)
 258         defer ticker.Stop()
 259         for waiting := "all"; waiting != ""; {
 260                 select {
 261                 case <-ticker.C:
 262                 case <-super.ctx.Done():
 263                         return nil, false
 264                 }
 265                 if super.healthChecker == nil {
 266                         // not set up yet
 267                         continue
 268                 }
 269                 resp := super.healthChecker.ClusterHealth()
 270                 // The overall health check (resp.Health=="OK") might
 271                 // never pass due to missing components (like
 272                 // arvados-dispatch-cloud in a test cluster), so
 273                 // instead we wait for all configured components to
 274                 // pass.
 275                 waiting = ""
 276                 for target, check := range resp.Checks {
 277                         if check.Health != "OK" {
 278                                 waiting += " " + target
 279                         }
 280                 }
 281                 if waiting != "" {
 282                         super.logger.WithField("targets", waiting[1:]).Info("waiting")
 283                 }
 284         }
 285         u := super.cluster.Services.Controller.ExternalURL
 286         return &u, true
 287 }
 288
 289 func (super *Supervisor) prependEnv(key, prepend string) {
 290         for i, s := range super.environ {
 291                 if strings.HasPrefix(s, key+"=") {
 292                         super.environ[i] = key + "=" + prepend + s[len(key)+1:]
 293                         return
 294                 }
 295         }
 296         super.environ = append(super.environ, key+"="+prepend)
 297 }
 298
 299 func (super *Supervisor) cleanEnv(prefixes []string) {
 300         var cleaned []string
 301         for _, s := range super.environ {
 302                 drop := false
 303                 for _, p := range prefixes {
 304                         if strings.HasPrefix(s, p) {
 305                                 drop = true
 306                                 break
 307                         }
 308                 }
 309                 if !drop {
 310                         cleaned = append(cleaned, s)
 311                 }
 312         }
 313         super.environ = cleaned
 314 }
 315
 316 func (super *Supervisor) setEnv(key, val string) {
 317         for i, s := range super.environ {
 318                 if strings.HasPrefix(s, key+"=") {
 319                         super.environ[i] = key + "=" + val
 320                         return
 321                 }
 322         }
 323         super.environ = append(super.environ, key+"="+val)
 324 }
 325
 326 // Remove all but the first occurrence of each env var.
 327 func dedupEnv(in []string) []string {
 328         saw := map[string]bool{}
 329         var out []string
 330         for _, kv := range in {
 331                 if split := strings.Index(kv, "="); split < 1 {
 332                         panic("invalid environment var: " + kv)
 333                 } else if saw[kv[:split]] {
 334                         continue
 335                 } else {
 336                         saw[kv[:split]] = true
 337                         out = append(out, kv)
 338                 }
 339         }
 340         return out
 341 }
 342
 343 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 344         _, basename := filepath.Split(srcpath)
 345         bindir := filepath.Join(super.tempdir, "bin")
 346         binfile := filepath.Join(bindir, basename)
 347         err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 348         return binfile, err
 349 }
 350
 351 func (super *Supervisor) usingRVM() bool {
 352         return os.Getenv("rvm_path") != ""
 353 }
 354
 355 func (super *Supervisor) setupRubyEnv() error {
 356         if !super.usingRVM() {
 357                 // (If rvm is in use, assume the caller has everything
 358                 // set up as desired)
 359                 super.cleanEnv([]string{
 360                         "GEM_HOME=",
 361                         "GEM_PATH=",
 362                 })
 363                 gem := "gem"
 364                 if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
 365                         gem = "/var/lib/arvados/bin/gem"
 366                 }
 367                 cmd := exec.Command(gem, "env", "gempath")
 368                 cmd.Env = super.environ
 369                 buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 370                 if err != nil || len(buf) == 0 {
 371                         return fmt.Errorf("gem env gempath: %v", err)
 372                 }
 373                 gempath := string(bytes.Split(buf, []byte{':'})[0])
 374                 super.prependEnv("PATH", gempath+"/bin:")
 375                 super.setEnv("GEM_HOME", gempath)
 376                 super.setEnv("GEM_PATH", gempath)
 377         }
 378         // Passenger install doesn't work unless $HOME is ~user
 379         u, err := user.Current()
 380         if err != nil {
 381                 return err
 382         }
 383         super.setEnv("HOME", u.HomeDir)
 384         return nil
 385 }
 386
 387 func (super *Supervisor) lookPath(prog string) string {
 388         for _, val := range super.environ {
 389                 if strings.HasPrefix(val, "PATH=") {
 390                         for _, dir := range filepath.SplitList(val[5:]) {
 391                                 path := filepath.Join(dir, prog)
 392                                 if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
 393                                         return path
 394                                 }
 395                         }
 396                 }
 397         }
 398         return prog
 399 }
 400
 401 // Run prog with args, using dir as working directory. If ctx is
 402 // cancelled while the child is running, RunProgram terminates the
 403 // child, waits for it to exit, then returns.
 404 //
 405 // Child's environment will have our env vars, plus any given in env.
 406 //
 407 // Child's stdout will be written to output if non-nil, otherwise the
 408 // boot command's stderr.
 409 func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
 410         cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
 411         super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 412
 413         logprefix := prog
 414         if logprefix == "setuidgid" && len(args) >= 3 {
 415                 logprefix = args[2]
 416         }
 417         logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
 418         if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
 419                 logprefix = args[1]
 420         } else if logprefix == "arvados-server" && len(args) > 1 {
 421                 logprefix = args[0]
 422         }
 423         if !strings.HasPrefix(dir, "/") {
 424                 logprefix = dir + ": " + logprefix
 425         }
 426
 427         cmd := exec.Command(super.lookPath(prog), args...)
 428         stdout, err := cmd.StdoutPipe()
 429         if err != nil {
 430                 return err
 431         }
 432         stderr, err := cmd.StderrPipe()
 433         if err != nil {
 434                 return err
 435         }
 436         logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
 437         var copiers sync.WaitGroup
 438         copiers.Add(1)
 439         go func() {
 440                 io.Copy(logwriter, stderr)
 441                 copiers.Done()
 442         }()
 443         copiers.Add(1)
 444         go func() {
 445                 if output == nil {
 446                         io.Copy(logwriter, stdout)
 447                 } else {
 448                         io.Copy(output, stdout)
 449                 }
 450                 copiers.Done()
 451         }()
 452
 453         if strings.HasPrefix(dir, "/") {
 454                 cmd.Dir = dir
 455         } else {
 456                 cmd.Dir = filepath.Join(super.SourcePath, dir)
 457         }
 458         env = append([]string(nil), env...)
 459         env = append(env, super.environ...)
 460         cmd.Env = dedupEnv(env)
 461
 462         exited := false
 463         defer func() { exited = true }()
 464         go func() {
 465                 <-ctx.Done()
 466                 log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
 467                 for !exited {
 468                         if cmd.Process == nil {
 469                                 log.Debug("waiting for child process to start")
 470                                 time.Sleep(time.Second / 2)
 471                         } else {
 472                                 log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
 473                                 cmd.Process.Signal(syscall.SIGTERM)
 474                                 time.Sleep(5 * time.Second)
 475                                 if !exited {
 476                                         stdout.Close()
 477                                         stderr.Close()
 478                                         log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
 479                                 }
 480                         }
 481                 }
 482         }()
 483
 484         err = cmd.Start()
 485         if err != nil {
 486                 return err
 487         }
 488         copiers.Wait()
 489         err = cmd.Wait()
 490         if ctx.Err() != nil {
 491                 // Return "context canceled", instead of the "killed"
 492                 // error that was probably caused by the context being
 493                 // canceled.
 494                 return ctx.Err()
 495         } else if err != nil {
 496                 return fmt.Errorf("%s: error: %v", cmdline, err)
 497         }
 498         return nil
 499 }
 500
 501 func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 502         cluster, err := cfg.GetCluster("")
 503         if err != nil {
 504                 return err
 505         }
 506         usedPort := map[string]bool{}
 507         nextPort := func(host string) string {
 508                 for {
 509                         port, err := availablePort(host)
 510                         if err != nil {
 511                                 panic(err)
 512                         }
 513                         if usedPort[port] {
 514                                 continue
 515                         }
 516                         usedPort[port] = true
 517                         return port
 518                 }
 519         }
 520         if cluster.Services.Controller.ExternalURL.Host == "" {
 521                 h, p, err := net.SplitHostPort(super.ControllerAddr)
 522                 if err != nil {
 523                         return err
 524                 }
 525                 if h == "" {
 526                         h = super.ListenHost
 527                 }
 528                 if p == "0" {
 529                         p = nextPort(h)
 530                 }
 531                 cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
 532         }
 533         for _, svc := range []*arvados.Service{
 534                 &cluster.Services.Controller,
 535                 &cluster.Services.DispatchCloud,
 536                 &cluster.Services.GitHTTP,
 537                 &cluster.Services.Health,
 538                 &cluster.Services.Keepproxy,
 539                 &cluster.Services.Keepstore,
 540                 &cluster.Services.RailsAPI,
 541                 &cluster.Services.WebDAV,
 542                 &cluster.Services.WebDAVDownload,
 543                 &cluster.Services.Websocket,
 544                 &cluster.Services.Workbench1,
 545         } {
 546                 if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
 547                         continue
 548                 }
 549                 if svc.ExternalURL.Host == "" {
 550                         if svc == &cluster.Services.Controller ||
 551                                 svc == &cluster.Services.GitHTTP ||
 552                                 svc == &cluster.Services.Keepproxy ||
 553                                 svc == &cluster.Services.WebDAV ||
 554                                 svc == &cluster.Services.WebDAVDownload ||
 555                                 svc == &cluster.Services.Workbench1 {
 556                                 svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
 557                         } else if svc == &cluster.Services.Websocket {
 558                                 svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
 559                         }
 560                 }
 561                 if len(svc.InternalURLs) == 0 {
 562                         svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
 563                                 arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
 564                         }
 565                 }
 566         }
 567         if cluster.SystemRootToken == "" {
 568                 cluster.SystemRootToken = randomHexString(64)
 569         }
 570         if cluster.ManagementToken == "" {
 571                 cluster.ManagementToken = randomHexString(64)
 572         }
 573         if cluster.API.RailsSessionSecretToken == "" {
 574                 cluster.API.RailsSessionSecretToken = randomHexString(64)
 575         }
 576         if cluster.Collections.BlobSigningKey == "" {
 577                 cluster.Collections.BlobSigningKey = randomHexString(64)
 578         }
 579         if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
 580                 buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 581                 if err != nil {
 582                         return err
 583                 }
 584                 cluster.Containers.DispatchPrivateKey = string(buf)
 585         }
 586         if super.ClusterType != "production" {
 587                 cluster.TLS.Insecure = true
 588         }
 589         if super.ClusterType == "test" {
 590                 // Add a second keepstore process.
 591                 cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
 592
 593                 // Create a directory-backed volume for each keepstore
 594                 // process.
 595                 cluster.Volumes = map[string]arvados.Volume{}
 596                 for url := range cluster.Services.Keepstore.InternalURLs {
 597                         volnum := len(cluster.Volumes)
 598                         datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
 599                         if _, err = os.Stat(datadir + "/."); err == nil {
 600                         } else if !os.IsNotExist(err) {
 601                                 return err
 602                         } else if err = os.Mkdir(datadir, 0755); err != nil {
 603                                 return err
 604                         }
 605                         cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
 606                                 Driver:           "Directory",
 607                                 DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
 608                                 AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
 609                                         url: {},
 610                                 },
 611                         }
 612                 }
 613         }
 614         if super.OwnTemporaryDatabase {
 615                 cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
 616                         "client_encoding": "utf8",
 617                         "host":            "localhost",
 618                         "port":            nextPort(super.ListenHost),
 619                         "dbname":          "arvados_test",
 620                         "user":            "arvados",
 621                         "password":        "insecure_arvados_test",
 622                 }
 623         }
 624
 625         cfg.Clusters[cluster.ClusterID] = *cluster
 626         return nil
 627 }
 628
 629 func addrIsLocal(addr string) (bool, error) {
 630         return true, nil
 631         listener, err := net.Listen("tcp", addr)
 632         if err == nil {
 633                 listener.Close()
 634                 return true, nil
 635         } else if strings.Contains(err.Error(), "cannot assign requested address") {
 636                 return false, nil
 637         } else {
 638                 return false, err
 639         }
 640 }
 641
 642 func randomHexString(chars int) string {
 643         b := make([]byte, chars/2)
 644         _, err := rand.Read(b)
 645         if err != nil {
 646                 panic(err)
 647         }
 648         return fmt.Sprintf("%x", b)
 649 }
 650
 651 func internalPort(svc arvados.Service) (string, error) {
 652         if len(svc.InternalURLs) > 1 {
 653                 return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
 654         }
 655         for u := range svc.InternalURLs {
 656                 if _, p, err := net.SplitHostPort(u.Host); err != nil {
 657                         return "", err
 658                 } else if p != "" {
 659                         return p, nil
 660                 } else if u.Scheme == "https" {
 661                         return "443", nil
 662                 } else {
 663                         return "80", nil
 664                 }
 665         }
 666         return "", fmt.Errorf("service has no InternalURLs")
 667 }
 668
 669 func externalPort(svc arvados.Service) (string, error) {
 670         if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
 671                 return "", err
 672         } else if p != "" {
 673                 return p, nil
 674         } else if svc.ExternalURL.Scheme == "https" {
 675                 return "443", nil
 676         } else {
 677                 return "80", nil
 678         }
 679 }
 680
 681 func availablePort(host string) (string, error) {
 682         ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
 683         if err != nil {
 684                 return "", err
 685         }
 686         defer ln.Close()
 687         _, port, err := net.SplitHostPort(ln.Addr().String())
 688         if err != nil {
 689                 return "", err
 690         }
 691         return port, nil
 692 }
 693
 694 // Try to connect to addr until it works, then close ch. Give up if
 695 // ctx cancels.
 696 func waitForConnect(ctx context.Context, addr string) error {
 697         dialer := net.Dialer{Timeout: time.Second}
 698         for ctx.Err() == nil {
 699                 conn, err := dialer.DialContext(ctx, "tcp", addr)
 700                 if err != nil {
 701                         time.Sleep(time.Second / 10)
 702                         continue
 703                 }
 704                 conn.Close()
 705                 return nil
 706         }
 707         return ctx.Err()
 708 }