lib/boot/supervisor.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package boot
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "os"
  18         "os/exec"
  19         "os/signal"
  20         "os/user"
  21         "path/filepath"
  22         "strings"
  23         "sync"
  24         "syscall"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/lib/service"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/health"
  31         "github.com/sirupsen/logrus"
  32 )
  33
  34 type Supervisor struct {
  35         SourcePath           string // e.g., /home/username/src/arvados
  36         SourceVersion        string // e.g., acbd1324...
  37         ClusterType          string // e.g., production
  38         ListenHost           string // e.g., localhost
  39         ControllerAddr       string // e.g., 127.0.0.1:8000
  40         OwnTemporaryDatabase bool
  41         Stderr               io.Writer
  42
  43         logger  logrus.FieldLogger
  44         cluster *arvados.Cluster
  45
  46         ctx           context.Context
  47         cancel        context.CancelFunc
  48         done          chan struct{}
  49         healthChecker *health.Aggregator
  50         tasksReady    map[string]chan bool
  51         waitShutdown  sync.WaitGroup
  52
  53         tempdir    string
  54         configfile string
  55         environ    []string // for child processes
  56 }
  57
  58 func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
  59         super.ctx, super.cancel = context.WithCancel(ctx)
  60         super.done = make(chan struct{})
  61
  62         go func() {
  63                 sigch := make(chan os.Signal)
  64                 signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
  65                 defer signal.Stop(sigch)
  66                 go func() {
  67                         for sig := range sigch {
  68                                 super.logger.WithField("signal", sig).Info("caught signal")
  69                                 super.cancel()
  70                         }
  71                 }()
  72
  73                 err := super.run(cfg)
  74                 if err != nil {
  75                         super.logger.WithError(err).Warn("supervisor shut down")
  76                 }
  77                 close(super.done)
  78         }()
  79 }
  80
  81 func (super *Supervisor) run(cfg *arvados.Config) error {
  82         cwd, err := os.Getwd()
  83         if err != nil {
  84                 return err
  85         }
  86         if !strings.HasPrefix(super.SourcePath, "/") {
  87                 super.SourcePath = filepath.Join(cwd, super.SourcePath)
  88         }
  89         super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
  90         if err != nil {
  91                 return err
  92         }
  93
  94         super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
  95         if err != nil {
  96                 return err
  97         }
  98         defer os.RemoveAll(super.tempdir)
  99         if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
 100                 return err
 101         }
 102
 103         // Fill in any missing config keys, and write the resulting
 104         // config in the temp dir for child services to use.
 105         err = super.autofillConfig(cfg)
 106         if err != nil {
 107                 return err
 108         }
 109         conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
 110         if err != nil {
 111                 return err
 112         }
 113         defer conffile.Close()
 114         err = json.NewEncoder(conffile).Encode(cfg)
 115         if err != nil {
 116                 return err
 117         }
 118         err = conffile.Close()
 119         if err != nil {
 120                 return err
 121         }
 122         super.configfile = conffile.Name()
 123
 124         super.environ = os.Environ()
 125         super.cleanEnv([]string{"ARVADOS_"})
 126         super.setEnv("ARVADOS_CONFIG", super.configfile)
 127         super.setEnv("RAILS_ENV", super.ClusterType)
 128         super.setEnv("TMPDIR", super.tempdir)
 129         super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
 130
 131         super.cluster, err = cfg.GetCluster("")
 132         if err != nil {
 133                 return err
 134         }
 135         // Now that we have the config, replace the bootstrap logger
 136         // with a new one according to the logging config.
 137         loglevel := super.cluster.SystemLogs.LogLevel
 138         if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
 139                 loglevel = "debug"
 140         }
 141         super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
 142                 "PID": os.Getpid(),
 143         })
 144
 145         if super.SourceVersion == "" {
 146                 // Find current source tree version.
 147                 var buf bytes.Buffer
 148                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
 149                 if err != nil {
 150                         return err
 151                 }
 152                 dirty := buf.Len() > 0
 153                 buf.Reset()
 154                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
 155                 if err != nil {
 156                         return err
 157                 }
 158                 super.SourceVersion = strings.TrimSpace(buf.String())
 159                 if dirty {
 160                         super.SourceVersion += "+uncommitted"
 161                 }
 162         } else {
 163                 return errors.New("specifying a version to run is not yet supported")
 164         }
 165
 166         _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
 167         if err != nil {
 168                 return err
 169         }
 170         err = super.setupRubyEnv()
 171         if err != nil {
 172                 return err
 173         }
 174
 175         tasks := []supervisedTask{
 176                 createCertificates{},
 177                 runPostgreSQL{},
 178                 runNginx{},
 179                 runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
 180                 runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
 181                 runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
 182                 runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
 183                 runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
 184                 runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 185                 runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 186                 installPassenger{src: "services/api"},
 187                 runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 188                 installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
 189                 runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 190                 seedDatabase{},
 191         }
 192         if super.ClusterType != "test" {
 193                 tasks = append(tasks,
 194                         runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
 195                         runGoProgram{src: "services/keep-balance"},
 196                 )
 197         }
 198         super.tasksReady = map[string]chan bool{}
 199         for _, task := range tasks {
 200                 super.tasksReady[task.String()] = make(chan bool)
 201         }
 202         for _, task := range tasks {
 203                 task := task
 204                 fail := func(err error) {
 205                         if super.ctx.Err() != nil {
 206                                 return
 207                         }
 208                         super.cancel()
 209                         super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
 210                 }
 211                 go func() {
 212                         super.logger.WithField("task", task.String()).Info("starting")
 213                         err := task.Run(super.ctx, fail, super)
 214                         if err != nil {
 215                                 fail(err)
 216                                 return
 217                         }
 218                         close(super.tasksReady[task.String()])
 219                 }()
 220         }
 221         err = super.wait(super.ctx, tasks...)
 222         if err != nil {
 223                 return err
 224         }
 225         super.logger.Info("all startup tasks are complete; starting health checks")
 226         super.healthChecker = &health.Aggregator{Cluster: super.cluster}
 227         <-super.ctx.Done()
 228         super.logger.Info("shutting down")
 229         super.waitShutdown.Wait()
 230         return super.ctx.Err()
 231 }
 232
 233 func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
 234         for _, task := range tasks {
 235                 ch, ok := super.tasksReady[task.String()]
 236                 if !ok {
 237                         return fmt.Errorf("no such task: %s", task)
 238                 }
 239                 super.logger.WithField("task", task.String()).Info("waiting")
 240                 select {
 241                 case <-ch:
 242                         super.logger.WithField("task", task.String()).Info("ready")
 243                 case <-ctx.Done():
 244                         super.logger.WithField("task", task.String()).Info("task was never ready")
 245                         return ctx.Err()
 246                 }
 247         }
 248         return nil
 249 }
 250
 251 func (super *Supervisor) Stop() {
 252         super.cancel()
 253         <-super.done
 254 }
 255
 256 func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
 257         ticker := time.NewTicker(time.Second)
 258         defer ticker.Stop()
 259         for waiting := "all"; waiting != ""; {
 260                 select {
 261                 case <-ticker.C:
 262                 case <-super.ctx.Done():
 263                         return nil, false
 264                 }
 265                 if super.healthChecker == nil {
 266                         // not set up yet
 267                         continue
 268                 }
 269                 resp := super.healthChecker.ClusterHealth()
 270                 // The overall health check (resp.Health=="OK") might
 271                 // never pass due to missing components (like
 272                 // arvados-dispatch-cloud in a test cluster), so
 273                 // instead we wait for all configured components to
 274                 // pass.
 275                 waiting = ""
 276                 for target, check := range resp.Checks {
 277                         if check.Health != "OK" {
 278                                 waiting += " " + target
 279                         }
 280                 }
 281                 if waiting != "" {
 282                         super.logger.WithField("targets", waiting[1:]).Info("waiting")
 283                 }
 284         }
 285         u := super.cluster.Services.Controller.ExternalURL
 286         return &u, true
 287 }
 288
 289 func (super *Supervisor) prependEnv(key, prepend string) {
 290         for i, s := range super.environ {
 291                 if strings.HasPrefix(s, key+"=") {
 292                         super.environ[i] = key + "=" + prepend + s[len(key)+1:]
 293                         return
 294                 }
 295         }
 296         super.environ = append(super.environ, key+"="+prepend)
 297 }
 298
 299 func (super *Supervisor) cleanEnv(prefixes []string) {
 300         var cleaned []string
 301         for _, s := range super.environ {
 302                 drop := false
 303                 for _, p := range prefixes {
 304                         if strings.HasPrefix(s, p) {
 305                                 drop = true
 306                                 break
 307                         }
 308                 }
 309                 if !drop {
 310                         cleaned = append(cleaned, s)
 311                 }
 312         }
 313         super.environ = cleaned
 314 }
 315
 316 func (super *Supervisor) setEnv(key, val string) {
 317         for i, s := range super.environ {
 318                 if strings.HasPrefix(s, key+"=") {
 319                         super.environ[i] = key + "=" + val
 320                         return
 321                 }
 322         }
 323         super.environ = append(super.environ, key+"="+val)
 324 }
 325
 326 // Remove all but the first occurrence of each env var.
 327 func dedupEnv(in []string) []string {
 328         saw := map[string]bool{}
 329         var out []string
 330         for _, kv := range in {
 331                 if split := strings.Index(kv, "="); split < 1 {
 332                         panic("invalid environment var: " + kv)
 333                 } else if saw[kv[:split]] {
 334                         continue
 335                 } else {
 336                         saw[kv[:split]] = true
 337                         out = append(out, kv)
 338                 }
 339         }
 340         return out
 341 }
 342
 343 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 344         _, basename := filepath.Split(srcpath)
 345         bindir := filepath.Join(super.tempdir, "bin")
 346         binfile := filepath.Join(bindir, basename)
 347         err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 348         return binfile, err
 349 }
 350
 351 func (super *Supervisor) usingRVM() bool {
 352         return os.Getenv("rvm_path") != ""
 353 }
 354
 355 func (super *Supervisor) setupRubyEnv() error {
 356         if !super.usingRVM() {
 357                 // (If rvm is in use, assume the caller has everything
 358                 // set up as desired)
 359                 super.cleanEnv([]string{
 360                         "GEM_HOME=",
 361                         "GEM_PATH=",
 362                 })
 363                 cmd := exec.Command("gem", "env", "gempath")
 364                 cmd.Env = super.environ
 365                 buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 366                 if err != nil || len(buf) == 0 {
 367                         return fmt.Errorf("gem env gempath: %v", err)
 368                 }
 369                 gempath := string(bytes.Split(buf, []byte{':'})[0])
 370                 super.prependEnv("PATH", gempath+"/bin:")
 371                 super.setEnv("GEM_HOME", gempath)
 372                 super.setEnv("GEM_PATH", gempath)
 373         }
 374         // Passenger install doesn't work unless $HOME is ~user
 375         u, err := user.Current()
 376         if err != nil {
 377                 return err
 378         }
 379         super.setEnv("HOME", u.HomeDir)
 380         return nil
 381 }
 382
 383 func (super *Supervisor) lookPath(prog string) string {
 384         for _, val := range super.environ {
 385                 if strings.HasPrefix(val, "PATH=") {
 386                         for _, dir := range filepath.SplitList(val[5:]) {
 387                                 path := filepath.Join(dir, prog)
 388                                 if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
 389                                         return path
 390                                 }
 391                         }
 392                 }
 393         }
 394         return prog
 395 }
 396
 397 // Run prog with args, using dir as working directory. If ctx is
 398 // cancelled while the child is running, RunProgram terminates the
 399 // child, waits for it to exit, then returns.
 400 //
 401 // Child's environment will have our env vars, plus any given in env.
 402 //
 403 // Child's stdout will be written to output if non-nil, otherwise the
 404 // boot command's stderr.
 405 func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
 406         cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
 407         super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 408
 409         logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
 410         if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
 411                 logprefix = args[1]
 412         } else if logprefix == "arvados-server" && len(args) > 1 {
 413                 logprefix = args[0]
 414         }
 415         if !strings.HasPrefix(dir, "/") {
 416                 logprefix = dir + ": " + logprefix
 417         }
 418
 419         cmd := exec.Command(super.lookPath(prog), args...)
 420         stdout, err := cmd.StdoutPipe()
 421         if err != nil {
 422                 return err
 423         }
 424         stderr, err := cmd.StderrPipe()
 425         if err != nil {
 426                 return err
 427         }
 428         logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
 429         var copiers sync.WaitGroup
 430         copiers.Add(1)
 431         go func() {
 432                 io.Copy(logwriter, stderr)
 433                 copiers.Done()
 434         }()
 435         copiers.Add(1)
 436         go func() {
 437                 if output == nil {
 438                         io.Copy(logwriter, stdout)
 439                 } else {
 440                         io.Copy(output, stdout)
 441                 }
 442                 copiers.Done()
 443         }()
 444
 445         if strings.HasPrefix(dir, "/") {
 446                 cmd.Dir = dir
 447         } else {
 448                 cmd.Dir = filepath.Join(super.SourcePath, dir)
 449         }
 450         env = append([]string(nil), env...)
 451         env = append(env, super.environ...)
 452         cmd.Env = dedupEnv(env)
 453
 454         exited := false
 455         defer func() { exited = true }()
 456         go func() {
 457                 <-ctx.Done()
 458                 log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
 459                 for !exited {
 460                         if cmd.Process == nil {
 461                                 log.Debug("waiting for child process to start")
 462                                 time.Sleep(time.Second / 2)
 463                         } else {
 464                                 log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
 465                                 cmd.Process.Signal(syscall.SIGTERM)
 466                                 time.Sleep(5 * time.Second)
 467                                 if !exited {
 468                                         stdout.Close()
 469                                         stderr.Close()
 470                                         log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
 471                                 }
 472                         }
 473                 }
 474         }()
 475
 476         err = cmd.Start()
 477         if err != nil {
 478                 return err
 479         }
 480         copiers.Wait()
 481         err = cmd.Wait()
 482         if ctx.Err() != nil {
 483                 // Return "context canceled", instead of the "killed"
 484                 // error that was probably caused by the context being
 485                 // canceled.
 486                 return ctx.Err()
 487         } else if err != nil {
 488                 return fmt.Errorf("%s: error: %v", cmdline, err)
 489         }
 490         return nil
 491 }
 492
 493 func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 494         cluster, err := cfg.GetCluster("")
 495         if err != nil {
 496                 return err
 497         }
 498         usedPort := map[string]bool{}
 499         nextPort := func(host string) string {
 500                 for {
 501                         port, err := availablePort(host)
 502                         if err != nil {
 503                                 panic(err)
 504                         }
 505                         if usedPort[port] {
 506                                 continue
 507                         }
 508                         usedPort[port] = true
 509                         return port
 510                 }
 511         }
 512         if cluster.Services.Controller.ExternalURL.Host == "" {
 513                 h, p, err := net.SplitHostPort(super.ControllerAddr)
 514                 if err != nil {
 515                         return err
 516                 }
 517                 if h == "" {
 518                         h = super.ListenHost
 519                 }
 520                 if p == "0" {
 521                         p = nextPort(h)
 522                 }
 523                 cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
 524         }
 525         for _, svc := range []*arvados.Service{
 526                 &cluster.Services.Controller,
 527                 &cluster.Services.DispatchCloud,
 528                 &cluster.Services.GitHTTP,
 529                 &cluster.Services.Health,
 530                 &cluster.Services.Keepproxy,
 531                 &cluster.Services.Keepstore,
 532                 &cluster.Services.RailsAPI,
 533                 &cluster.Services.WebDAV,
 534                 &cluster.Services.WebDAVDownload,
 535                 &cluster.Services.Websocket,
 536                 &cluster.Services.Workbench1,
 537         } {
 538                 if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
 539                         continue
 540                 }
 541                 if svc.ExternalURL.Host == "" {
 542                         if svc == &cluster.Services.Controller ||
 543                                 svc == &cluster.Services.GitHTTP ||
 544                                 svc == &cluster.Services.Keepproxy ||
 545                                 svc == &cluster.Services.WebDAV ||
 546                                 svc == &cluster.Services.WebDAVDownload ||
 547                                 svc == &cluster.Services.Workbench1 {
 548                                 svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
 549                         } else if svc == &cluster.Services.Websocket {
 550                                 svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
 551                         }
 552                 }
 553                 if len(svc.InternalURLs) == 0 {
 554                         svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
 555                                 arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
 556                         }
 557                 }
 558         }
 559         if cluster.SystemRootToken == "" {
 560                 cluster.SystemRootToken = randomHexString(64)
 561         }
 562         if cluster.ManagementToken == "" {
 563                 cluster.ManagementToken = randomHexString(64)
 564         }
 565         if cluster.API.RailsSessionSecretToken == "" {
 566                 cluster.API.RailsSessionSecretToken = randomHexString(64)
 567         }
 568         if cluster.Collections.BlobSigningKey == "" {
 569                 cluster.Collections.BlobSigningKey = randomHexString(64)
 570         }
 571         if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
 572                 buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 573                 if err != nil {
 574                         return err
 575                 }
 576                 cluster.Containers.DispatchPrivateKey = string(buf)
 577         }
 578         if super.ClusterType != "production" {
 579                 cluster.TLS.Insecure = true
 580         }
 581         if super.ClusterType == "test" {
 582                 // Add a second keepstore process.
 583                 cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
 584
 585                 // Create a directory-backed volume for each keepstore
 586                 // process.
 587                 cluster.Volumes = map[string]arvados.Volume{}
 588                 for url := range cluster.Services.Keepstore.InternalURLs {
 589                         volnum := len(cluster.Volumes)
 590                         datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
 591                         if _, err = os.Stat(datadir + "/."); err == nil {
 592                         } else if !os.IsNotExist(err) {
 593                                 return err
 594                         } else if err = os.Mkdir(datadir, 0755); err != nil {
 595                                 return err
 596                         }
 597                         cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
 598                                 Driver:           "Directory",
 599                                 DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
 600                                 AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
 601                                         url: {},
 602                                 },
 603                         }
 604                 }
 605         }
 606         if super.OwnTemporaryDatabase {
 607                 cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
 608                         "client_encoding": "utf8",
 609                         "host":            "localhost",
 610                         "port":            nextPort(super.ListenHost),
 611                         "dbname":          "arvados_test",
 612                         "user":            "arvados",
 613                         "password":        "insecure_arvados_test",
 614                 }
 615         }
 616
 617         cfg.Clusters[cluster.ClusterID] = *cluster
 618         return nil
 619 }
 620
 621 func addrIsLocal(addr string) (bool, error) {
 622         return true, nil
 623         listener, err := net.Listen("tcp", addr)
 624         if err == nil {
 625                 listener.Close()
 626                 return true, nil
 627         } else if strings.Contains(err.Error(), "cannot assign requested address") {
 628                 return false, nil
 629         } else {
 630                 return false, err
 631         }
 632 }
 633
 634 func randomHexString(chars int) string {
 635         b := make([]byte, chars/2)
 636         _, err := rand.Read(b)
 637         if err != nil {
 638                 panic(err)
 639         }
 640         return fmt.Sprintf("%x", b)
 641 }
 642
 643 func internalPort(svc arvados.Service) (string, error) {
 644         if len(svc.InternalURLs) > 1 {
 645                 return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
 646         }
 647         for u := range svc.InternalURLs {
 648                 if _, p, err := net.SplitHostPort(u.Host); err != nil {
 649                         return "", err
 650                 } else if p != "" {
 651                         return p, nil
 652                 } else if u.Scheme == "https" {
 653                         return "443", nil
 654                 } else {
 655                         return "80", nil
 656                 }
 657         }
 658         return "", fmt.Errorf("service has no InternalURLs")
 659 }
 660
 661 func externalPort(svc arvados.Service) (string, error) {
 662         if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
 663                 return "", err
 664         } else if p != "" {
 665                 return p, nil
 666         } else if svc.ExternalURL.Scheme == "https" {
 667                 return "443", nil
 668         } else {
 669                 return "80", nil
 670         }
 671 }
 672
 673 func availablePort(host string) (string, error) {
 674         ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
 675         if err != nil {
 676                 return "", err
 677         }
 678         defer ln.Close()
 679         _, port, err := net.SplitHostPort(ln.Addr().String())
 680         if err != nil {
 681                 return "", err
 682         }
 683         return port, nil
 684 }
 685
 686 // Try to connect to addr until it works, then close ch. Give up if
 687 // ctx cancels.
 688 func waitForConnect(ctx context.Context, addr string) error {
 689         dialer := net.Dialer{Timeout: time.Second}
 690         for ctx.Err() == nil {
 691                 conn, err := dialer.DialContext(ctx, "tcp", addr)
 692                 if err != nil {
 693                         time.Sleep(time.Second / 10)
 694                         continue
 695                 }
 696                 conn.Close()
 697                 return nil
 698         }
 699         return ctx.Err()
 700 }