lib/boot/supervisor.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package boot
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "os"
  18         "os/exec"
  19         "os/signal"
  20         "os/user"
  21         "path/filepath"
  22         "strings"
  23         "sync"
  24         "syscall"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/lib/service"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/health"
  31         "github.com/sirupsen/logrus"
  32 )
  33
  34 type Supervisor struct {
  35         SourcePath           string // e.g., /home/username/src/arvados
  36         SourceVersion        string // e.g., acbd1324...
  37         ClusterType          string // e.g., production
  38         ListenHost           string // e.g., localhost
  39         ControllerAddr       string // e.g., 127.0.0.1:8000
  40         OwnTemporaryDatabase bool
  41         Stderr               io.Writer
  42
  43         logger  logrus.FieldLogger
  44         cluster *arvados.Cluster
  45
  46         ctx           context.Context
  47         cancel        context.CancelFunc
  48         done          chan struct{}
  49         healthChecker *health.Aggregator
  50         tasksReady    map[string]chan bool
  51         waitShutdown  sync.WaitGroup
  52
  53         tempdir    string
  54         configfile string
  55         environ    []string // for child processes
  56 }
  57
  58 func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
  59         super.ctx, super.cancel = context.WithCancel(ctx)
  60         super.done = make(chan struct{})
  61
  62         go func() {
  63                 sigch := make(chan os.Signal)
  64                 signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
  65                 defer signal.Stop(sigch)
  66                 go func() {
  67                         for sig := range sigch {
  68                                 super.logger.WithField("signal", sig).Info("caught signal")
  69                                 super.cancel()
  70                         }
  71                 }()
  72
  73                 err := super.run(cfg)
  74                 if err != nil {
  75                         super.logger.WithError(err).Warn("supervisor shut down")
  76                 }
  77                 close(super.done)
  78         }()
  79 }
  80
  81 func (super *Supervisor) run(cfg *arvados.Config) error {
  82         cwd, err := os.Getwd()
  83         if err != nil {
  84                 return err
  85         }
  86         if !strings.HasPrefix(super.SourcePath, "/") {
  87                 super.SourcePath = filepath.Join(cwd, super.SourcePath)
  88         }
  89         super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
  90         if err != nil {
  91                 return err
  92         }
  93
  94         super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
  95         if err != nil {
  96                 return err
  97         }
  98         defer os.RemoveAll(super.tempdir)
  99         if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
 100                 return err
 101         }
 102
 103         // Fill in any missing config keys, and write the resulting
 104         // config in the temp dir for child services to use.
 105         err = super.autofillConfig(cfg)
 106         if err != nil {
 107                 return err
 108         }
 109         conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
 110         if err != nil {
 111                 return err
 112         }
 113         defer conffile.Close()
 114         err = json.NewEncoder(conffile).Encode(cfg)
 115         if err != nil {
 116                 return err
 117         }
 118         err = conffile.Close()
 119         if err != nil {
 120                 return err
 121         }
 122         super.configfile = conffile.Name()
 123
 124         super.environ = os.Environ()
 125         super.cleanEnv([]string{"ARVADOS_"})
 126         super.setEnv("ARVADOS_CONFIG", super.configfile)
 127         super.setEnv("RAILS_ENV", super.ClusterType)
 128         super.setEnv("TMPDIR", super.tempdir)
 129         super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
 130
 131         super.cluster, err = cfg.GetCluster("")
 132         if err != nil {
 133                 return err
 134         }
 135         // Now that we have the config, replace the bootstrap logger
 136         // with a new one according to the logging config.
 137         loglevel := super.cluster.SystemLogs.LogLevel
 138         if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
 139                 loglevel = "debug"
 140         }
 141         super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
 142                 "PID": os.Getpid(),
 143         })
 144
 145         if super.SourceVersion == "" {
 146                 // Find current source tree version.
 147                 var buf bytes.Buffer
 148                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
 149                 if err != nil {
 150                         return err
 151                 }
 152                 dirty := buf.Len() > 0
 153                 buf.Reset()
 154                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
 155                 if err != nil {
 156                         return err
 157                 }
 158                 super.SourceVersion = strings.TrimSpace(buf.String())
 159                 if dirty {
 160                         super.SourceVersion += "+uncommitted"
 161                 }
 162         } else {
 163                 return errors.New("specifying a version to run is not yet supported")
 164         }
 165
 166         _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
 167         if err != nil {
 168                 return err
 169         }
 170         err = super.setupRubyEnv()
 171         if err != nil {
 172                 return err
 173         }
 174
 175         tasks := []supervisedTask{
 176                 createCertificates{},
 177                 runPostgreSQL{},
 178                 runNginx{},
 179                 runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
 180                 runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
 181                 runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
 182                 runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
 183                 runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
 184                 runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 185                 runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 186                 installPassenger{src: "services/api"},
 187                 runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 188                 installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
 189                 runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 190                 seedDatabase{},
 191         }
 192         if super.ClusterType != "test" {
 193                 tasks = append(tasks,
 194                         runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
 195                         runGoProgram{src: "services/keep-balance"},
 196                 )
 197         }
 198         super.tasksReady = map[string]chan bool{}
 199         for _, task := range tasks {
 200                 super.tasksReady[task.String()] = make(chan bool)
 201         }
 202         for _, task := range tasks {
 203                 task := task
 204                 fail := func(err error) {
 205                         if super.ctx.Err() != nil {
 206                                 return
 207                         }
 208                         super.cancel()
 209                         super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
 210                 }
 211                 go func() {
 212                         super.logger.WithField("task", task.String()).Info("starting")
 213                         err := task.Run(super.ctx, fail, super)
 214                         if err != nil {
 215                                 fail(err)
 216                                 return
 217                         }
 218                         close(super.tasksReady[task.String()])
 219                 }()
 220         }
 221         err = super.wait(super.ctx, tasks...)
 222         if err != nil {
 223                 return err
 224         }
 225         super.logger.Info("all startup tasks are complete; starting health checks")
 226         super.healthChecker = &health.Aggregator{Cluster: super.cluster}
 227         <-super.ctx.Done()
 228         super.logger.Info("shutting down")
 229         super.waitShutdown.Wait()
 230         return super.ctx.Err()
 231 }
 232
 233 func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
 234         for _, task := range tasks {
 235                 ch, ok := super.tasksReady[task.String()]
 236                 if !ok {
 237                         return fmt.Errorf("no such task: %s", task)
 238                 }
 239                 super.logger.WithField("task", task.String()).Info("waiting")
 240                 select {
 241                 case <-ch:
 242                         super.logger.WithField("task", task.String()).Info("ready")
 243                 case <-ctx.Done():
 244                         super.logger.WithField("task", task.String()).Info("task was never ready")
 245                         return ctx.Err()
 246                 }
 247         }
 248         return nil
 249 }
 250
 251 func (super *Supervisor) Stop() {
 252         super.cancel()
 253         <-super.done
 254 }
 255
 256 func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
 257         ticker := time.NewTicker(time.Second)
 258         defer ticker.Stop()
 259         for waiting := "all"; waiting != ""; {
 260                 select {
 261                 case <-ticker.C:
 262                 case <-super.ctx.Done():
 263                         return nil, false
 264                 }
 265                 if super.healthChecker == nil {
 266                         // not set up yet
 267                         continue
 268                 }
 269                 resp := super.healthChecker.ClusterHealth()
 270                 // The overall health check (resp.Health=="OK") might
 271                 // never pass due to missing components (like
 272                 // arvados-dispatch-cloud in a test cluster), so
 273                 // instead we wait for all configured components to
 274                 // pass.
 275                 waiting = ""
 276                 for target, check := range resp.Checks {
 277                         if check.Health != "OK" {
 278                                 waiting += " " + target
 279                         }
 280                 }
 281                 if waiting != "" {
 282                         super.logger.WithField("targets", waiting[1:]).Info("waiting")
 283                 }
 284         }
 285         u := super.cluster.Services.Controller.ExternalURL
 286         return &u, true
 287 }
 288
 289 func (super *Supervisor) prependEnv(key, prepend string) {
 290         for i, s := range super.environ {
 291                 if strings.HasPrefix(s, key+"=") {
 292                         super.environ[i] = key + "=" + prepend + s[len(key)+1:]
 293                         return
 294                 }
 295         }
 296         super.environ = append(super.environ, key+"="+prepend)
 297 }
 298
 299 func (super *Supervisor) cleanEnv(prefixes []string) {
 300         var cleaned []string
 301         for _, s := range super.environ {
 302                 drop := false
 303                 for _, p := range prefixes {
 304                         if strings.HasPrefix(s, p) {
 305                                 drop = true
 306                                 break
 307                         }
 308                 }
 309                 if !drop {
 310                         cleaned = append(cleaned, s)
 311                 }
 312         }
 313         super.environ = cleaned
 314 }
 315
 316 func (super *Supervisor) setEnv(key, val string) {
 317         for i, s := range super.environ {
 318                 if strings.HasPrefix(s, key+"=") {
 319                         super.environ[i] = key + "=" + val
 320                         return
 321                 }
 322         }
 323         super.environ = append(super.environ, key+"="+val)
 324 }
 325
 326 // Remove all but the first occurrence of each env var.
 327 func dedupEnv(in []string) []string {
 328         saw := map[string]bool{}
 329         var out []string
 330         for _, kv := range in {
 331                 if split := strings.Index(kv, "="); split < 1 {
 332                         panic("invalid environment var: " + kv)
 333                 } else if saw[kv[:split]] {
 334                         continue
 335                 } else {
 336                         saw[kv[:split]] = true
 337                         out = append(out, kv)
 338                 }
 339         }
 340         return out
 341 }
 342
 343 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 344         _, basename := filepath.Split(srcpath)
 345         bindir := filepath.Join(super.tempdir, "bin")
 346         binfile := filepath.Join(bindir, basename)
 347         err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 348         return binfile, err
 349 }
 350
 351 func (super *Supervisor) usingRVM() bool {
 352         return os.Getenv("rvm_path") != ""
 353 }
 354
 355 func (super *Supervisor) setupRubyEnv() error {
 356         if !super.usingRVM() {
 357                 // (If rvm is in use, assume the caller has everything
 358                 // set up as desired)
 359                 super.cleanEnv([]string{
 360                         "GEM_HOME=",
 361                         "GEM_PATH=",
 362                 })
 363                 cmd := exec.Command("gem", "env", "gempath")
 364                 cmd.Env = super.environ
 365                 buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 366                 if err != nil || len(buf) == 0 {
 367                         return fmt.Errorf("gem env gempath: %v", err)
 368                 }
 369                 gempath := string(bytes.Split(buf, []byte{':'})[0])
 370                 super.prependEnv("PATH", gempath+"/bin:")
 371                 super.setEnv("GEM_HOME", gempath)
 372                 super.setEnv("GEM_PATH", gempath)
 373         }
 374         // Passenger install doesn't work unless $HOME is ~user
 375         u, err := user.Current()
 376         if err != nil {
 377                 return err
 378         }
 379         super.setEnv("HOME", u.HomeDir)
 380         return nil
 381 }
 382
 383 func (super *Supervisor) lookPath(prog string) string {
 384         for _, val := range super.environ {
 385                 if strings.HasPrefix(val, "PATH=") {
 386                         for _, dir := range filepath.SplitList(val[5:]) {
 387                                 path := filepath.Join(dir, prog)
 388                                 if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
 389                                         return path
 390                                 }
 391                         }
 392                 }
 393         }
 394         return prog
 395 }
 396
 397 // Run prog with args, using dir as working directory. If ctx is
 398 // cancelled while the child is running, RunProgram terminates the
 399 // child, waits for it to exit, then returns.
 400 //
 401 // Child's environment will have our env vars, plus any given in env.
 402 //
 403 // Child's stdout will be written to output if non-nil, otherwise the
 404 // boot command's stderr.
 405 func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
 406         cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
 407         super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 408
 409         logprefix := prog
 410         if logprefix == "sudo" && len(args) >= 3 && args[0] == "-u" {
 411                 logprefix = args[2]
 412         }
 413         logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
 414         if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
 415                 logprefix = args[1]
 416         } else if logprefix == "arvados-server" && len(args) > 1 {
 417                 logprefix = args[0]
 418         }
 419         if !strings.HasPrefix(dir, "/") {
 420                 logprefix = dir + ": " + logprefix
 421         }
 422
 423         cmd := exec.Command(super.lookPath(prog), args...)
 424         stdout, err := cmd.StdoutPipe()
 425         if err != nil {
 426                 return err
 427         }
 428         stderr, err := cmd.StderrPipe()
 429         if err != nil {
 430                 return err
 431         }
 432         logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
 433         var copiers sync.WaitGroup
 434         copiers.Add(1)
 435         go func() {
 436                 io.Copy(logwriter, stderr)
 437                 copiers.Done()
 438         }()
 439         copiers.Add(1)
 440         go func() {
 441                 if output == nil {
 442                         io.Copy(logwriter, stdout)
 443                 } else {
 444                         io.Copy(output, stdout)
 445                 }
 446                 copiers.Done()
 447         }()
 448
 449         if strings.HasPrefix(dir, "/") {
 450                 cmd.Dir = dir
 451         } else {
 452                 cmd.Dir = filepath.Join(super.SourcePath, dir)
 453         }
 454         env = append([]string(nil), env...)
 455         env = append(env, super.environ...)
 456         cmd.Env = dedupEnv(env)
 457
 458         exited := false
 459         defer func() { exited = true }()
 460         go func() {
 461                 <-ctx.Done()
 462                 log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
 463                 for !exited {
 464                         if cmd.Process == nil {
 465                                 log.Debug("waiting for child process to start")
 466                                 time.Sleep(time.Second / 2)
 467                         } else {
 468                                 log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
 469                                 cmd.Process.Signal(syscall.SIGTERM)
 470                                 time.Sleep(5 * time.Second)
 471                                 if !exited {
 472                                         stdout.Close()
 473                                         stderr.Close()
 474                                         log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
 475                                 }
 476                         }
 477                 }
 478         }()
 479
 480         err = cmd.Start()
 481         if err != nil {
 482                 return err
 483         }
 484         copiers.Wait()
 485         err = cmd.Wait()
 486         if ctx.Err() != nil {
 487                 // Return "context canceled", instead of the "killed"
 488                 // error that was probably caused by the context being
 489                 // canceled.
 490                 return ctx.Err()
 491         } else if err != nil {
 492                 return fmt.Errorf("%s: error: %v", cmdline, err)
 493         }
 494         return nil
 495 }
 496
 497 func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 498         cluster, err := cfg.GetCluster("")
 499         if err != nil {
 500                 return err
 501         }
 502         usedPort := map[string]bool{}
 503         nextPort := func(host string) string {
 504                 for {
 505                         port, err := availablePort(host)
 506                         if err != nil {
 507                                 panic(err)
 508                         }
 509                         if usedPort[port] {
 510                                 continue
 511                         }
 512                         usedPort[port] = true
 513                         return port
 514                 }
 515         }
 516         if cluster.Services.Controller.ExternalURL.Host == "" {
 517                 h, p, err := net.SplitHostPort(super.ControllerAddr)
 518                 if err != nil {
 519                         return err
 520                 }
 521                 if h == "" {
 522                         h = super.ListenHost
 523                 }
 524                 if p == "0" {
 525                         p = nextPort(h)
 526                 }
 527                 cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
 528         }
 529         for _, svc := range []*arvados.Service{
 530                 &cluster.Services.Controller,
 531                 &cluster.Services.DispatchCloud,
 532                 &cluster.Services.GitHTTP,
 533                 &cluster.Services.Health,
 534                 &cluster.Services.Keepproxy,
 535                 &cluster.Services.Keepstore,
 536                 &cluster.Services.RailsAPI,
 537                 &cluster.Services.WebDAV,
 538                 &cluster.Services.WebDAVDownload,
 539                 &cluster.Services.Websocket,
 540                 &cluster.Services.Workbench1,
 541         } {
 542                 if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
 543                         continue
 544                 }
 545                 if svc.ExternalURL.Host == "" && (svc == &cluster.Services.Controller ||
 546                         svc == &cluster.Services.GitHTTP ||
 547                         svc == &cluster.Services.Keepproxy ||
 548                         svc == &cluster.Services.WebDAV ||
 549                         svc == &cluster.Services.WebDAVDownload ||
 550                         svc == &cluster.Services.Websocket ||
 551                         svc == &cluster.Services.Workbench1) {
 552                         svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
 553                 }
 554                 if len(svc.InternalURLs) == 0 {
 555                         svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
 556                                 arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
 557                         }
 558                 }
 559         }
 560         if cluster.SystemRootToken == "" {
 561                 cluster.SystemRootToken = randomHexString(64)
 562         }
 563         if cluster.ManagementToken == "" {
 564                 cluster.ManagementToken = randomHexString(64)
 565         }
 566         if cluster.API.RailsSessionSecretToken == "" {
 567                 cluster.API.RailsSessionSecretToken = randomHexString(64)
 568         }
 569         if cluster.Collections.BlobSigningKey == "" {
 570                 cluster.Collections.BlobSigningKey = randomHexString(64)
 571         }
 572         if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
 573                 buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 574                 if err != nil {
 575                         return err
 576                 }
 577                 cluster.Containers.DispatchPrivateKey = string(buf)
 578         }
 579         if super.ClusterType != "production" {
 580                 cluster.TLS.Insecure = true
 581         }
 582         if super.ClusterType == "test" {
 583                 // Add a second keepstore process.
 584                 cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
 585
 586                 // Create a directory-backed volume for each keepstore
 587                 // process.
 588                 cluster.Volumes = map[string]arvados.Volume{}
 589                 for url := range cluster.Services.Keepstore.InternalURLs {
 590                         volnum := len(cluster.Volumes)
 591                         datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
 592                         if _, err = os.Stat(datadir + "/."); err == nil {
 593                         } else if !os.IsNotExist(err) {
 594                                 return err
 595                         } else if err = os.Mkdir(datadir, 0755); err != nil {
 596                                 return err
 597                         }
 598                         cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
 599                                 Driver:           "Directory",
 600                                 DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
 601                                 AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
 602                                         url: {},
 603                                 },
 604                         }
 605                 }
 606         }
 607         if super.OwnTemporaryDatabase {
 608                 cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
 609                         "client_encoding": "utf8",
 610                         "host":            "localhost",
 611                         "port":            nextPort(super.ListenHost),
 612                         "dbname":          "arvados_test",
 613                         "user":            "arvados",
 614                         "password":        "insecure_arvados_test",
 615                 }
 616         }
 617
 618         cfg.Clusters[cluster.ClusterID] = *cluster
 619         return nil
 620 }
 621
 622 func addrIsLocal(addr string) (bool, error) {
 623         return true, nil
 624         listener, err := net.Listen("tcp", addr)
 625         if err == nil {
 626                 listener.Close()
 627                 return true, nil
 628         } else if strings.Contains(err.Error(), "cannot assign requested address") {
 629                 return false, nil
 630         } else {
 631                 return false, err
 632         }
 633 }
 634
 635 func randomHexString(chars int) string {
 636         b := make([]byte, chars/2)
 637         _, err := rand.Read(b)
 638         if err != nil {
 639                 panic(err)
 640         }
 641         return fmt.Sprintf("%x", b)
 642 }
 643
 644 func internalPort(svc arvados.Service) (string, error) {
 645         if len(svc.InternalURLs) > 1 {
 646                 return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
 647         }
 648         for u := range svc.InternalURLs {
 649                 if _, p, err := net.SplitHostPort(u.Host); err != nil {
 650                         return "", err
 651                 } else if p != "" {
 652                         return p, nil
 653                 } else if u.Scheme == "https" {
 654                         return "443", nil
 655                 } else {
 656                         return "80", nil
 657                 }
 658         }
 659         return "", fmt.Errorf("service has no InternalURLs")
 660 }
 661
 662 func externalPort(svc arvados.Service) (string, error) {
 663         if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
 664                 return "", err
 665         } else if p != "" {
 666                 return p, nil
 667         } else if svc.ExternalURL.Scheme == "https" {
 668                 return "443", nil
 669         } else {
 670                 return "80", nil
 671         }
 672 }
 673
 674 func availablePort(host string) (string, error) {
 675         ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
 676         if err != nil {
 677                 return "", err
 678         }
 679         defer ln.Close()
 680         _, port, err := net.SplitHostPort(ln.Addr().String())
 681         if err != nil {
 682                 return "", err
 683         }
 684         return port, nil
 685 }
 686
 687 // Try to connect to addr until it works, then close ch. Give up if
 688 // ctx cancels.
 689 func waitForConnect(ctx context.Context, addr string) error {
 690         dialer := net.Dialer{Timeout: time.Second}
 691         for ctx.Err() == nil {
 692                 conn, err := dialer.DialContext(ctx, "tcp", addr)
 693                 if err != nil {
 694                         time.Sleep(time.Second / 10)
 695                         continue
 696                 }
 697                 conn.Close()
 698                 return nil
 699         }
 700         return ctx.Err()
 701 }