lib/boot/supervisor.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package boot
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/rand"
  11         "encoding/json"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "os"
  18         "os/exec"
  19         "os/signal"
  20         "os/user"
  21         "path/filepath"
  22         "strings"
  23         "sync"
  24         "syscall"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/lib/service"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/health"
  31         "github.com/sirupsen/logrus"
  32 )
  33
  34 type Supervisor struct {
  35         SourcePath           string // e.g., /home/username/src/arvados
  36         SourceVersion        string // e.g., acbd1324...
  37         LibPath              string // e.g., /var/lib/arvados
  38         ClusterType          string // e.g., production
  39         ListenHost           string // e.g., localhost
  40         ControllerAddr       string // e.g., 127.0.0.1:8000
  41         OwnTemporaryDatabase bool
  42         Stderr               io.Writer
  43
  44         logger  logrus.FieldLogger
  45         cluster *arvados.Cluster
  46
  47         ctx           context.Context
  48         cancel        context.CancelFunc
  49         done          chan struct{}
  50         healthChecker *health.Aggregator
  51         tasksReady    map[string]chan bool
  52         waitShutdown  sync.WaitGroup
  53
  54         tempdir    string
  55         configfile string
  56         environ    []string // for child processes
  57 }
  58
  59 func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
  60         super.ctx, super.cancel = context.WithCancel(ctx)
  61         super.done = make(chan struct{})
  62
  63         go func() {
  64                 sigch := make(chan os.Signal)
  65                 signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
  66                 defer signal.Stop(sigch)
  67                 go func() {
  68                         for sig := range sigch {
  69                                 super.logger.WithField("signal", sig).Info("caught signal")
  70                                 super.cancel()
  71                         }
  72                 }()
  73
  74                 err := super.run(cfg)
  75                 if err != nil {
  76                         fmt.Fprintln(super.Stderr, err)
  77                 }
  78                 close(super.done)
  79         }()
  80 }
  81
  82 func (super *Supervisor) run(cfg *arvados.Config) error {
  83         cwd, err := os.Getwd()
  84         if err != nil {
  85                 return err
  86         }
  87         if !strings.HasPrefix(super.SourcePath, "/") {
  88                 super.SourcePath = filepath.Join(cwd, super.SourcePath)
  89         }
  90         super.SourcePath, err = filepath.EvalSymlinks(super.SourcePath)
  91         if err != nil {
  92                 return err
  93         }
  94
  95         super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
  96         if err != nil {
  97                 return err
  98         }
  99         defer os.RemoveAll(super.tempdir)
 100         if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0777); err != nil {
 101                 return err
 102         }
 103
 104         // Fill in any missing config keys, and write the resulting
 105         // config in the temp dir for child services to use.
 106         err = super.autofillConfig(cfg, super.logger)
 107         if err != nil {
 108                 return err
 109         }
 110         conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0777)
 111         if err != nil {
 112                 return err
 113         }
 114         defer conffile.Close()
 115         err = json.NewEncoder(conffile).Encode(cfg)
 116         if err != nil {
 117                 return err
 118         }
 119         err = conffile.Close()
 120         if err != nil {
 121                 return err
 122         }
 123         super.configfile = conffile.Name()
 124
 125         super.environ = os.Environ()
 126         super.cleanEnv()
 127         super.setEnv("ARVADOS_CONFIG", super.configfile)
 128         super.setEnv("RAILS_ENV", super.ClusterType)
 129         super.setEnv("TMPDIR", super.tempdir)
 130         super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
 131         super.prependEnv("PATH", filepath.Join(super.LibPath, "bin")+":")
 132
 133         super.cluster, err = cfg.GetCluster("")
 134         if err != nil {
 135                 return err
 136         }
 137         // Now that we have the config, replace the bootstrap logger
 138         // with a new one according to the logging config.
 139         loglevel := super.cluster.SystemLogs.LogLevel
 140         if s := os.Getenv("ARVADOS_DEBUG"); s != "" && s != "0" {
 141                 loglevel = "debug"
 142         }
 143         super.logger = ctxlog.New(super.Stderr, super.cluster.SystemLogs.Format, loglevel).WithFields(logrus.Fields{
 144                 "PID": os.Getpid(),
 145         })
 146
 147         if super.SourceVersion == "" {
 148                 // Find current source tree version.
 149                 var buf bytes.Buffer
 150                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
 151                 if err != nil {
 152                         return err
 153                 }
 154                 dirty := buf.Len() > 0
 155                 buf.Reset()
 156                 err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
 157                 if err != nil {
 158                         return err
 159                 }
 160                 super.SourceVersion = strings.TrimSpace(buf.String())
 161                 if dirty {
 162                         super.SourceVersion += "+uncommitted"
 163                 }
 164         } else {
 165                 return errors.New("specifying a version to run is not yet supported")
 166         }
 167         for _, dir := range []string{super.LibPath, filepath.Join(super.LibPath, "bin")} {
 168                 if _, err = os.Stat(filepath.Join(dir, ".")); os.IsNotExist(err) {
 169                         err = os.Mkdir(dir, 0755)
 170                         if err != nil {
 171                                 return err
 172                         }
 173                 } else if err != nil {
 174                         return err
 175                 }
 176         }
 177         _, err = super.installGoProgram(super.ctx, "cmd/arvados-server")
 178         if err != nil {
 179                 return err
 180         }
 181         err = super.setupRubyEnv()
 182         if err != nil {
 183                 return err
 184         }
 185
 186         tasks := []supervisedTask{
 187                 createCertificates{},
 188                 runPostgreSQL{},
 189                 runNginx{},
 190                 runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{runPostgreSQL{}}},
 191                 runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
 192                 runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
 193                 runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
 194                 runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
 195                 runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 196                 runGoProgram{src: "services/ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 197                 installPassenger{src: "services/api"},
 198                 runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 199                 installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
 200                 runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 201                 seedDatabase{},
 202         }
 203         if super.ClusterType != "test" {
 204                 tasks = append(tasks,
 205                         runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
 206                         runGoProgram{src: "services/keep-balance"},
 207                 )
 208         }
 209         super.tasksReady = map[string]chan bool{}
 210         for _, task := range tasks {
 211                 super.tasksReady[task.String()] = make(chan bool)
 212         }
 213         for _, task := range tasks {
 214                 task := task
 215                 fail := func(err error) {
 216                         if super.ctx.Err() != nil {
 217                                 return
 218                         }
 219                         super.cancel()
 220                         super.logger.WithField("task", task.String()).WithError(err).Error("task failed")
 221                 }
 222                 go func() {
 223                         super.logger.WithField("task", task.String()).Info("starting")
 224                         err := task.Run(super.ctx, fail, super)
 225                         if err != nil {
 226                                 fail(err)
 227                                 return
 228                         }
 229                         close(super.tasksReady[task.String()])
 230                 }()
 231         }
 232         err = super.wait(super.ctx, tasks...)
 233         if err != nil {
 234                 return err
 235         }
 236         super.logger.Info("all startup tasks are complete; starting health checks")
 237         super.healthChecker = &health.Aggregator{Cluster: super.cluster}
 238         <-super.ctx.Done()
 239         super.logger.Info("shutting down")
 240         super.waitShutdown.Wait()
 241         return super.ctx.Err()
 242 }
 243
 244 func (super *Supervisor) wait(ctx context.Context, tasks ...supervisedTask) error {
 245         for _, task := range tasks {
 246                 ch, ok := super.tasksReady[task.String()]
 247                 if !ok {
 248                         return fmt.Errorf("no such task: %s", task)
 249                 }
 250                 super.logger.WithField("task", task.String()).Info("waiting")
 251                 select {
 252                 case <-ch:
 253                         super.logger.WithField("task", task.String()).Info("ready")
 254                 case <-ctx.Done():
 255                         super.logger.WithField("task", task.String()).Info("task was never ready")
 256                         return ctx.Err()
 257                 }
 258         }
 259         return nil
 260 }
 261
 262 func (super *Supervisor) Stop() {
 263         super.cancel()
 264         <-super.done
 265 }
 266
 267 func (super *Supervisor) WaitReady() (*arvados.URL, bool) {
 268         ticker := time.NewTicker(time.Second)
 269         defer ticker.Stop()
 270         for waiting := "all"; waiting != ""; {
 271                 select {
 272                 case <-ticker.C:
 273                 case <-super.ctx.Done():
 274                         return nil, false
 275                 }
 276                 if super.healthChecker == nil {
 277                         // not set up yet
 278                         continue
 279                 }
 280                 resp := super.healthChecker.ClusterHealth()
 281                 // The overall health check (resp.Health=="OK") might
 282                 // never pass due to missing components (like
 283                 // arvados-dispatch-cloud in a test cluster), so
 284                 // instead we wait for all configured components to
 285                 // pass.
 286                 waiting = ""
 287                 for target, check := range resp.Checks {
 288                         if check.Health != "OK" {
 289                                 waiting += " " + target
 290                         }
 291                 }
 292                 if waiting != "" {
 293                         super.logger.WithField("targets", waiting[1:]).Info("waiting")
 294                 }
 295         }
 296         u := super.cluster.Services.Controller.ExternalURL
 297         return &u, true
 298 }
 299
 300 func (super *Supervisor) prependEnv(key, prepend string) {
 301         for i, s := range super.environ {
 302                 if strings.HasPrefix(s, key+"=") {
 303                         super.environ[i] = key + "=" + prepend + s[len(key)+1:]
 304                         return
 305                 }
 306         }
 307         super.environ = append(super.environ, key+"="+prepend)
 308 }
 309
 310 var cleanEnvPrefixes = []string{
 311         "GEM_HOME=",
 312         "GEM_PATH=",
 313         "ARVADOS_",
 314 }
 315
 316 func (super *Supervisor) cleanEnv() {
 317         var cleaned []string
 318         for _, s := range super.environ {
 319                 drop := false
 320                 for _, p := range cleanEnvPrefixes {
 321                         if strings.HasPrefix(s, p) {
 322                                 drop = true
 323                                 break
 324                         }
 325                 }
 326                 if !drop {
 327                         cleaned = append(cleaned, s)
 328                 }
 329         }
 330         super.environ = cleaned
 331 }
 332
 333 func (super *Supervisor) setEnv(key, val string) {
 334         for i, s := range super.environ {
 335                 if strings.HasPrefix(s, key+"=") {
 336                         super.environ[i] = key + "=" + val
 337                         return
 338                 }
 339         }
 340         super.environ = append(super.environ, key+"="+val)
 341 }
 342
 343 // Remove all but the first occurrence of each env var.
 344 func dedupEnv(in []string) []string {
 345         saw := map[string]bool{}
 346         var out []string
 347         for _, kv := range in {
 348                 if split := strings.Index(kv, "="); split < 1 {
 349                         panic("invalid environment var: " + kv)
 350                 } else if saw[kv[:split]] {
 351                         continue
 352                 } else {
 353                         saw[kv[:split]] = true
 354                         out = append(out, kv)
 355                 }
 356         }
 357         return out
 358 }
 359
 360 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 361         _, basename := filepath.Split(srcpath)
 362         bindir := filepath.Join(super.tempdir, "bin")
 363         binfile := filepath.Join(bindir, basename)
 364         err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 365         return binfile, err
 366 }
 367
 368 func (super *Supervisor) setupRubyEnv() error {
 369         cmd := exec.Command("gem", "env", "gempath")
 370         cmd.Env = super.environ
 371         buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 372         if err != nil || len(buf) == 0 {
 373                 return fmt.Errorf("gem env gempath: %v", err)
 374         }
 375         gempath := string(bytes.Split(buf, []byte{':'})[0])
 376         super.prependEnv("PATH", gempath+"/bin:")
 377         super.setEnv("GEM_HOME", gempath)
 378         super.setEnv("GEM_PATH", gempath)
 379         // Passenger install doesn't work unless $HOME is ~user
 380         u, err := user.Current()
 381         if err != nil {
 382                 return err
 383         }
 384         super.setEnv("HOME", u.HomeDir)
 385         return nil
 386 }
 387
 388 func (super *Supervisor) lookPath(prog string) string {
 389         for _, val := range super.environ {
 390                 if strings.HasPrefix(val, "PATH=") {
 391                         for _, dir := range filepath.SplitList(val[5:]) {
 392                                 path := filepath.Join(dir, prog)
 393                                 if fi, err := os.Stat(path); err == nil && fi.Mode()&0111 != 0 {
 394                                         return path
 395                                 }
 396                         }
 397                 }
 398         }
 399         return prog
 400 }
 401
 402 // Run prog with args, using dir as working directory. If ctx is
 403 // cancelled while the child is running, RunProgram terminates the
 404 // child, waits for it to exit, then returns.
 405 //
 406 // Child's environment will have our env vars, plus any given in env.
 407 //
 408 // Child's stdout will be written to output if non-nil, otherwise the
 409 // boot command's stderr.
 410 func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
 411         cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
 412         super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 413
 414         logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
 415         if prog == "bundle" && len(args) > 2 && args[0] == "exec" {
 416                 logprefix = args[1]
 417         } else if prog == "arvados-server" && len(args) > 1 {
 418                 logprefix = args[0]
 419         }
 420         if !strings.HasPrefix(dir, "/") {
 421                 logprefix = dir + ": " + logprefix
 422         }
 423
 424         cmd := exec.Command(super.lookPath(prog), args...)
 425         stdout, err := cmd.StdoutPipe()
 426         if err != nil {
 427                 return err
 428         }
 429         stderr, err := cmd.StderrPipe()
 430         if err != nil {
 431                 return err
 432         }
 433         logwriter := &service.LogPrefixer{Writer: super.Stderr, Prefix: []byte("[" + logprefix + "] ")}
 434         var copiers sync.WaitGroup
 435         copiers.Add(1)
 436         go func() {
 437                 io.Copy(logwriter, stderr)
 438                 copiers.Done()
 439         }()
 440         copiers.Add(1)
 441         go func() {
 442                 if output == nil {
 443                         io.Copy(logwriter, stdout)
 444                 } else {
 445                         io.Copy(output, stdout)
 446                 }
 447                 copiers.Done()
 448         }()
 449
 450         if strings.HasPrefix(dir, "/") {
 451                 cmd.Dir = dir
 452         } else {
 453                 cmd.Dir = filepath.Join(super.SourcePath, dir)
 454         }
 455         env = append([]string(nil), env...)
 456         env = append(env, super.environ...)
 457         cmd.Env = dedupEnv(env)
 458
 459         exited := false
 460         defer func() { exited = true }()
 461         go func() {
 462                 <-ctx.Done()
 463                 log := ctxlog.FromContext(ctx).WithFields(logrus.Fields{"dir": dir, "cmdline": cmdline})
 464                 for !exited {
 465                         if cmd.Process == nil {
 466                                 log.Debug("waiting for child process to start")
 467                                 time.Sleep(time.Second / 2)
 468                         } else {
 469                                 log.WithField("PID", cmd.Process.Pid).Debug("sending SIGTERM")
 470                                 cmd.Process.Signal(syscall.SIGTERM)
 471                                 time.Sleep(5 * time.Second)
 472                                 if !exited {
 473                                         stdout.Close()
 474                                         stderr.Close()
 475                                         log.WithField("PID", cmd.Process.Pid).Warn("still waiting for child process to exit 5s after SIGTERM")
 476                                 }
 477                         }
 478                 }
 479         }()
 480
 481         err = cmd.Start()
 482         if err != nil {
 483                 return err
 484         }
 485         copiers.Wait()
 486         err = cmd.Wait()
 487         if ctx.Err() != nil {
 488                 // Return "context canceled", instead of the "killed"
 489                 // error that was probably caused by the context being
 490                 // canceled.
 491                 return ctx.Err()
 492         } else if err != nil {
 493                 return fmt.Errorf("%s: error: %v", cmdline, err)
 494         }
 495         return nil
 496 }
 497
 498 func (super *Supervisor) autofillConfig(cfg *arvados.Config, log logrus.FieldLogger) error {
 499         cluster, err := cfg.GetCluster("")
 500         if err != nil {
 501                 return err
 502         }
 503         usedPort := map[string]bool{}
 504         nextPort := func() string {
 505                 for {
 506                         port, err := availablePort(super.ListenHost + ":0")
 507                         if err != nil {
 508                                 panic(err)
 509                         }
 510                         if usedPort[port] {
 511                                 continue
 512                         }
 513                         usedPort[port] = true
 514                         return port
 515                 }
 516         }
 517         if cluster.Services.Controller.ExternalURL.Host == "" {
 518                 h, p, err := net.SplitHostPort(super.ControllerAddr)
 519                 if err != nil {
 520                         return err
 521                 }
 522                 if h == "" {
 523                         h = super.ListenHost
 524                 }
 525                 if p == "0" {
 526                         p, err = availablePort(":0")
 527                         if err != nil {
 528                                 return err
 529                         }
 530                         usedPort[p] = true
 531                 }
 532                 cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
 533         }
 534         for _, svc := range []*arvados.Service{
 535                 &cluster.Services.Controller,
 536                 &cluster.Services.DispatchCloud,
 537                 &cluster.Services.GitHTTP,
 538                 &cluster.Services.Health,
 539                 &cluster.Services.Keepproxy,
 540                 &cluster.Services.Keepstore,
 541                 &cluster.Services.RailsAPI,
 542                 &cluster.Services.WebDAV,
 543                 &cluster.Services.WebDAVDownload,
 544                 &cluster.Services.Websocket,
 545                 &cluster.Services.Workbench1,
 546         } {
 547                 if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
 548                         continue
 549                 }
 550                 if svc.ExternalURL.Host == "" && (svc == &cluster.Services.Controller ||
 551                         svc == &cluster.Services.GitHTTP ||
 552                         svc == &cluster.Services.Keepproxy ||
 553                         svc == &cluster.Services.WebDAV ||
 554                         svc == &cluster.Services.WebDAVDownload ||
 555                         svc == &cluster.Services.Websocket ||
 556                         svc == &cluster.Services.Workbench1) {
 557                         svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}
 558                 }
 559                 if len(svc.InternalURLs) == 0 {
 560                         svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
 561                                 arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}: arvados.ServiceInstance{},
 562                         }
 563                 }
 564         }
 565         if cluster.SystemRootToken == "" {
 566                 cluster.SystemRootToken = randomHexString(64)
 567         }
 568         if cluster.ManagementToken == "" {
 569                 cluster.ManagementToken = randomHexString(64)
 570         }
 571         if cluster.API.RailsSessionSecretToken == "" {
 572                 cluster.API.RailsSessionSecretToken = randomHexString(64)
 573         }
 574         if cluster.Collections.BlobSigningKey == "" {
 575                 cluster.Collections.BlobSigningKey = randomHexString(64)
 576         }
 577         if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
 578                 buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
 579                 if err != nil {
 580                         return err
 581                 }
 582                 cluster.Containers.DispatchPrivateKey = string(buf)
 583         }
 584         if super.ClusterType != "production" {
 585                 cluster.TLS.Insecure = true
 586         }
 587         if super.ClusterType == "test" {
 588                 // Add a second keepstore process.
 589                 cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort())}] = arvados.ServiceInstance{}
 590
 591                 // Create a directory-backed volume for each keepstore
 592                 // process.
 593                 cluster.Volumes = map[string]arvados.Volume{}
 594                 for url := range cluster.Services.Keepstore.InternalURLs {
 595                         volnum := len(cluster.Volumes)
 596                         datadir := fmt.Sprintf("%s/keep%d.data", super.tempdir, volnum)
 597                         if _, err = os.Stat(datadir + "/."); err == nil {
 598                         } else if !os.IsNotExist(err) {
 599                                 return err
 600                         } else if err = os.Mkdir(datadir, 0777); err != nil {
 601                                 return err
 602                         }
 603                         cluster.Volumes[fmt.Sprintf(cluster.ClusterID+"-nyw5e-%015d", volnum)] = arvados.Volume{
 604                                 Driver:           "Directory",
 605                                 DriverParameters: json.RawMessage(fmt.Sprintf(`{"Root":%q}`, datadir)),
 606                                 AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
 607                                         url: {},
 608                                 },
 609                         }
 610                 }
 611         }
 612         if super.OwnTemporaryDatabase {
 613                 cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
 614                         "client_encoding": "utf8",
 615                         "host":            "localhost",
 616                         "port":            nextPort(),
 617                         "dbname":          "arvados_test",
 618                         "user":            "arvados",
 619                         "password":        "insecure_arvados_test",
 620                 }
 621         }
 622
 623         cfg.Clusters[cluster.ClusterID] = *cluster
 624         return nil
 625 }
 626
 627 func randomHexString(chars int) string {
 628         b := make([]byte, chars/2)
 629         _, err := rand.Read(b)
 630         if err != nil {
 631                 panic(err)
 632         }
 633         return fmt.Sprintf("%x", b)
 634 }
 635
 636 func internalPort(svc arvados.Service) (string, error) {
 637         for u := range svc.InternalURLs {
 638                 if _, p, err := net.SplitHostPort(u.Host); err != nil {
 639                         return "", err
 640                 } else if p != "" {
 641                         return p, nil
 642                 } else if u.Scheme == "https" {
 643                         return "443", nil
 644                 } else {
 645                         return "80", nil
 646                 }
 647         }
 648         return "", fmt.Errorf("service has no InternalURLs")
 649 }
 650
 651 func externalPort(svc arvados.Service) (string, error) {
 652         if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
 653                 return "", err
 654         } else if p != "" {
 655                 return p, nil
 656         } else if svc.ExternalURL.Scheme == "https" {
 657                 return "443", nil
 658         } else {
 659                 return "80", nil
 660         }
 661 }
 662
 663 func availablePort(addr string) (string, error) {
 664         ln, err := net.Listen("tcp", addr)
 665         if err != nil {
 666                 return "", err
 667         }
 668         defer ln.Close()
 669         _, port, err := net.SplitHostPort(ln.Addr().String())
 670         if err != nil {
 671                 return "", err
 672         }
 673         return port, nil
 674 }
 675
 676 // Try to connect to addr until it works, then close ch. Give up if
 677 // ctx cancels.
 678 func waitForConnect(ctx context.Context, addr string) error {
 679         dialer := net.Dialer{Timeout: time.Second}
 680         for ctx.Err() == nil {
 681                 conn, err := dialer.DialContext(ctx, "tcp", addr)
 682                 if err != nil {
 683                         time.Sleep(time.Second / 10)
 684                         continue
 685                 }
 686                 conn.Close()
 687                 return nil
 688         }
 689         return ctx.Err()
 690 }