From: Tom Clegg <tom@curii.com>
Date: Tue, 22 Dec 2020 21:48:37 +0000 (-0500)
Subject: 16306: Merge branch 'master'
X-Git-Tag: 2.2.0~141^2~32
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3aaefcb3c76ff470b475d950398d01255e87712a?hp=2c8874c0625a2126813d4df7170076f95cdc7327

16306: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 595f721080..7bd4e618dd 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -550,7 +550,7 @@ setup_ruby_environment() {
         (
             export HOME=$GEMHOME
             bundlers="$(gem list --details bundler)"
-            versions=(1.11.0 1.17.3 2.0.2)
+            versions=(1.16.6 1.17.3 2.0.2)
             for v in ${versions[@]}; do
                 if ! echo "$bundlers" | fgrep -q "($v)"; then
                     gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
new file mode 100644
index 0000000000..a6e0b544a0
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage.go
@@ -0,0 +1,124 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+
+	"git.arvados.org/arvados.git/lib/install"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+type buildPackage struct{}
+
+func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "text", "info")
+	err := (&builder{
+		PackageVersion: "0.0.0",
+		logger:         logger,
+	}).run(context.Background(), prog, args, stdin, stdout, stderr)
+	if err != nil {
+		logger.WithError(err).Error("failed")
+		return 1
+	}
+	return 0
+}
+
+type builder struct {
+	PackageVersion string
+	SourcePath     string
+	OutputDir      string
+	SkipInstall    bool
+	logger         logrus.FieldLogger
+}
+
+func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
+	flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
+	flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
+	flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
+	err := flags.Parse(args)
+	if err != nil {
+		return err
+	}
+	if len(flags.Args()) > 0 {
+		return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+	}
+	if !bldr.SkipInstall {
+		exitcode := install.Command.RunCommand("arvados-server install", []string{
+			"-type", "package",
+			"-package-version", bldr.PackageVersion,
+			"-source", bldr.SourcePath,
+		}, stdin, stdout, stderr)
+		if exitcode != 0 {
+			return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+		}
+	}
+	cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("gem install fpm: %w", err)
+	}
+
+	if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+		// Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+		cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return fmt.Errorf("monkeypatch fpm: %w", err)
+		}
+	}
+
+	// Remove unneeded files. This is much faster than "fpm
+	// --exclude X" because fpm copies everything into a staging
+	// area before looking at the --exclude args.
+	cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git /var/lib/arvados/go")
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("rm -rf /var/www/.gem/ruby/*/cache/: %w", err)
+	}
+
+	format := "deb" // TODO: rpm
+
+	cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+		"--name", "arvados-server-easy",
+		"--version", bldr.PackageVersion,
+		"--input-type", "dir",
+		"--output-type", format)
+	deps, err := install.ProductionDependencies()
+	if err != nil {
+		return err
+	}
+	for _, pkg := range deps {
+		cmd.Args = append(cmd.Args, "--depends", pkg)
+	}
+	cmd.Args = append(cmd.Args,
+		"--verbose",
+		"--deb-use-file-permissions",
+		"--rpm-use-file-permissions",
+		"/var/lib/arvados",
+		"/var/www/.gem",
+		"/var/www/.passenger",
+		"/var/www/.bundle",
+	)
+	fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+	cmd.Dir = bldr.OutputDir
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	return cmd.Run()
+}
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-dev/buildpackage_test.go
new file mode 100644
index 0000000000..61bdeae82f
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage_test.go
@@ -0,0 +1,77 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+	os.Args = append(os.Args, "-test.timeout=30m") // kludge
+
+	// This enables a hack to speed up repeated tests: hit "docker
+	// commit --pause {containername} checkpointtag" after the
+	// test container has downloaded/compiled some stuff, then run
+	// tests with "-test.buildimage=checkpointtag" next time to
+	// retry/resume/update from that point.
+	flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
+}
+
+type BuildpackageSuite struct{}
+
+var _ = check.Suite(&BuildpackageSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+	if testing.Short() {
+		c.Skip("skipping docker tests in short mode")
+	} else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+		c.Skip("skipping docker tests because docker is not available")
+	}
+	tmpdir := c.MkDir()
+	defer os.RemoveAll(tmpdir)
+
+	err := os.Mkdir(tmpdir+"/pkg", 0755)
+	c.Assert(err, check.IsNil)
+	err = os.Mkdir(tmpdir+"/bin", 0755)
+	c.Assert(err, check.IsNil)
+
+	cmd := exec.Command("go", "install")
+	cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+
+	srctree, err := filepath.Abs("../..")
+	c.Assert(err, check.IsNil)
+
+	cmd = exec.Command("docker", "run", "--rm",
+		"-v", tmpdir+"/pkg:/pkg",
+		"-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+		"-v", srctree+":/usr/local/src/arvados:ro",
+		buildimage,
+		"/arvados-dev", "buildpackage",
+		"-package-version", "0.9.99",
+		"-source", "/usr/local/src/arvados",
+		"-output-directory", "/pkg")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+
+	fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+	c.Assert(err, check.IsNil)
+	c.Logf("%#v", fi)
+}
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-dev/cmd.go
new file mode 100644
index 0000000000..e0d5144e57
--- /dev/null
+++ b/cmd/arvados-dev/cmd.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"os"
+
+	"git.arvados.org/arvados.git/lib/cmd"
+)
+
+var (
+	handler = cmd.Multi(map[string]cmd.Handler{
+		"version":   cmd.Version,
+		"-version":  cmd.Version,
+		"--version": cmd.Version,
+
+		"buildpackage": buildPackage{},
+	})
+)
+
+func main() {
+	os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
new file mode 100755
index 0000000000..39a8a56c8a
--- /dev/null
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Bring up a docker container with some locally-built commands (e.g.,
+# cmd/arvados-server) replacing the ones that came with
+# arvados-server-easy when the arvados-installpackage-* image was
+# built.
+#
+# Assumes docker-build-install.sh has already succeeded.
+#
+# Example:
+#
+#    docker-boot.sh cmd/arvados-server services/keep-balance
+
+set -e -o pipefail
+
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --os)
+            shift
+            opts[os]="$1"
+            shift
+            ;;
+        --shell)
+            shift
+            opts[shell]=1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+cleanup() {
+    if [[ -n "${tmpdir}" ]]; then
+        rm -rf "${tmpdir}"
+    fi
+}
+trap cleanup ERR EXIT
+
+tmpdir=$(mktemp -d)
+version=$(git describe --tag --dirty)
+
+declare -a volargs=()
+for inject in "$@"; do
+    case "$inject" in
+        nginx.conf)
+            volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
+            ;;
+        *)
+            echo >&2 "building $inject..."
+            (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+            cmd="$(basename "$inject")"
+            volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+            ;;
+    esac
+done
+
+osbase=${opts[os]:-debian:10}
+installimage=arvados-installpackage-${osbase}
+command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
+if [[ "${opts[shell]}" ]]; then
+    command="bash -login"
+fi
+docker run -it --rm \
+       "${volargs[@]}" \
+       "${installimage}" \
+       bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
new file mode 100755
index 0000000000..ff514a3b68
--- /dev/null
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Build an arvados-server-easy package, then install and run it on a
+# base OS image.
+#
+# Examples:
+#
+#    docker-build-install.sh --force-buildimage --force-installimage     # always build fresh docker images
+#
+#    docker-build-install.sh                                             # reuse cached docker images if possible
+
+set -e -o pipefail
+
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+    arg="$1"
+    shift
+    case "$arg" in
+        --force-buildimage)
+            opts[force-buildimage]=1
+            ;;
+        --force-installimage)
+            opts[force-installimage]=1
+            ;;
+        --os)
+            opts[os]="$1"
+            shift
+            ;;
+        *)
+            echo >&2 "invalid argument '$arg'"
+            exit 2
+            ;;
+    esac
+done
+
+cleanup() {
+    if [[ -n "${buildctr}" ]]; then
+        docker rm "${buildctr}" || true
+    fi
+    if [[ -n "${installctr}" ]]; then
+        docker rm "${installctr}" || true
+    fi
+}
+trap cleanup ERR EXIT
+
+version=$(git describe --tag --dirty)
+osbase=${opts[os]:-debian:10}
+
+mkdir -p /tmp/pkg
+
+buildimage=arvados-buildpackage-${osbase}
+if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
+    (
+        echo >&2 building arvados-server...
+        cd cmd/arvados-server
+        go install
+    )
+    echo >&2 building ${buildimage}...
+    buildctr=${buildimage/:/-}
+    docker rm "${buildctr}" || true
+    docker run \
+           --name "${buildctr}" \
+           -v /tmp/pkg:/pkg \
+           -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
+           -v "$(pwd)":/arvados:ro \
+           "${osbase}" \
+           /arvados-server install \
+           -type package \
+           -source /arvados \
+           -package-version "${version}"
+    docker commit "${buildctr}" "${buildimage}"
+    docker rm "${buildctr}"
+    buildctr=
+fi
+
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+rm -v -f "${pkgfile}"
+
+(
+    echo >&2 building arvados-dev...
+    cd cmd/arvados-dev
+    go install
+)
+echo >&2 building ${pkgfile}...
+docker run --rm \
+       -v /tmp/pkg:/pkg \
+       -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+       -v "$(pwd)":/arvados:ro \
+       "${buildimage}" \
+       /arvados-dev buildpackage \
+       -source /arvados \
+       -package-version "${version}" \
+       -output-directory /pkg
+
+ls -l ${pkgfile}
+(
+    echo >&2 dpkg-scanpackages...
+    cd /tmp/pkg
+    dpkg-scanpackages . | gzip > Packages.gz
+)
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+installimage="arvados-installpackage-${osbase}"
+if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+    echo >&2 building ${installimage}...
+    installctr=${installimage/:/-}
+    docker rm "${installctr}" || true
+    docker run -it \
+           --name "${installctr}" \
+           -v /tmp/pkg:/pkg:ro \
+           -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+           "${osbase}" \
+           bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+    docker commit "${installctr}" "${installimage}"
+    docker rm "${installctr}"
+    installctr=
+fi
+
+echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
+docker run -it --rm \
+       -v /tmp/pkg:/pkg:ro \
+       -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+       "${installimage}" \
+       bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0'
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index ff99de75c4..d0aa9da94d 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -34,6 +34,7 @@ var (
 		"crunch-run":         crunchrun.Command,
 		"dispatch-cloud":     dispatchcloud.Command,
 		"install":            install.Command,
+		"init":               install.InitCommand,
 		"recover-collection": recovercollection.Command,
 		"ws":                 ws.Command,
 	})
diff --git a/doc/_config.yml b/doc/_config.yml
index 75a55b469d..5386e8797a 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -208,6 +208,8 @@ navbar:
       - install/arvados-on-kubernetes.html.textile.liquid
       - install/arvados-on-kubernetes-minikube.html.textile.liquid
       - install/arvados-on-kubernetes-GKE.html.textile.liquid
+    - Automatic installation:
+      - install/automatic.html.textile.liquid
     - Manual installation:
       - install/install-manual-prerequisites.html.textile.liquid
       - install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
new file mode 100644
index 0000000000..049bf68917
--- /dev/null
+++ b/doc/install/automatic.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example@gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# apt-get install arvados-server
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example@gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 1b27ca6ed9..f16ae2dad2 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -25,6 +25,7 @@ table(table table-bordered table-condensed).
 |"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
 |"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
 |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
 |"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
 |"Cluster Operation Subscription supported by Curii":mailto:info@curii.com|N/A ^3^|yes|yes|yes|yes|yes|
 </div>
diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index f0797c2ac5..8f6339e63e 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
 	"path/filepath"
 )
 
@@ -27,9 +28,14 @@ func (createCertificates) String() string {
 func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
 	var san string
 	if net.ParseIP(super.ListenHost) != nil {
-		san = fmt.Sprintf("IP:%s", super.ListenHost)
+		san += fmt.Sprintf(",IP:%s", super.ListenHost)
 	} else {
-		san = fmt.Sprintf("DNS:%s", super.ListenHost)
+		san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+	}
+	if hostname, err := os.Hostname(); err != nil {
+		return fmt.Errorf("hostname: %w", err)
+	} else {
+		san += ",DNS:" + hostname
 	}
 
 	// Generate root key
@@ -52,7 +58,7 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+	err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 0f105d6b6c..d49fabe567 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -9,8 +9,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"regexp"
 
@@ -36,7 +38,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 		"SSLKEY":     filepath.Join(super.tempdir, "server.key"),
 		"ACCESSLOG":  filepath.Join(super.tempdir, "nginx_access.log"),
 		"ERRORLOG":   filepath.Join(super.tempdir, "nginx_error.log"),
-		"TMPDIR":     super.tempdir,
+		"TMPDIR":     super.wwwtempdir,
 	}
 	for _, cmpt := range []struct {
 		varname string
@@ -53,7 +55,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 	} {
 		port, err := internalPort(cmpt.svc)
 		if err != nil {
-			return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+			return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
 		}
 		if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
 			return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
@@ -62,14 +64,20 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 
 		port, err = externalPort(cmpt.svc)
 		if err != nil {
-			return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+			return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
 		}
 		if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
 			return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
 		}
 		vars[cmpt.varname+"SSLPORT"] = port
 	}
-	tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+	var conftemplate string
+	if super.ClusterType == "production" {
+		conftemplate = "/var/lib/arvados/share/nginx.conf"
+	} else {
+		conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+	}
+	tmpl, err := ioutil.ReadFile(conftemplate)
 	if err != nil {
 		return err
 	}
@@ -93,13 +101,32 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 			}
 		}
 	}
+
+	args := []string{
+		"-g", "error_log stderr info;",
+		"-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
+		"-c", conffile,
+	}
+	// Nginx ignores "user www-data;" when running as a non-root
+	// user... except that it causes it to ignore our other -g
+	// options. So we still have to decide for ourselves whether
+	// it's needed.
+	if u, err := user.Current(); err != nil {
+		return fmt.Errorf("user.Current(): %w", err)
+	} else if u.Uid == "0" {
+		args = append([]string{"-g", "user www-data;"}, args...)
+	}
+
 	super.waitShutdown.Add(1)
 	go func() {
 		defer super.waitShutdown.Done()
-		fail(super.RunProgram(ctx, ".", nil, nil, nginx,
-			"-g", "error_log stderr info;",
-			"-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
-			"-c", conffile))
+		fail(super.RunProgram(ctx, ".", nil, nil, nginx, args...))
 	}()
-	return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+	// Choose one of the ports where Nginx should listen, and wait
+	// here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+	testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+	if testurl.Port() == "" {
+		testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+	}
+	return waitForConnect(ctx, testurl.Host)
 }
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 6a2c4b61f5..03464aaf7c 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -37,6 +37,10 @@ func (runner installPassenger) String() string {
 }
 
 func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+	if super.ClusterType == "production" {
+		// passenger has already been installed via package
+		return nil
+	}
 	err := super.wait(ctx, runner.depends...)
 	if err != nil {
 		return err
@@ -50,9 +54,9 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 	if err != nil {
 		return err
 	}
-	for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+	for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
 		if !strings.Contains(buf.String(), "("+version+")") {
-			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+			err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
 			if err != nil {
 				return err
 			}
@@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
 }
 
 type runPassenger struct {
-	src     string
-	svc     arvados.Service
-	depends []supervisedTask
+	src       string // path to app in source tree
+	varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+	svc       arvados.Service
+	depends   []supervisedTask
 }
 
 func (runner runPassenger) String() string {
@@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 	if err != nil {
 		return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
 	}
+	var appdir string
+	if super.ClusterType == "production" {
+		appdir = "/var/lib/arvados/" + runner.varlibdir
+	} else {
+		appdir = runner.src
+	}
 	loglevel := "4"
 	if lvl, ok := map[string]string{
 		"debug":   "5",
@@ -116,13 +127,30 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
 	super.waitShutdown.Add(1)
 	go func() {
 		defer super.waitShutdown.Done()
-		err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+		cmdline := []string{
+			"bundle", "exec",
 			"passenger", "start",
 			"-p", port,
-			"--log-file", "/dev/stderr",
 			"--log-level", loglevel,
 			"--no-friendly-error-pages",
-			"--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+			"--disable-anonymous-telemetry",
+			"--disable-security-update-check",
+			"--no-compile-runtime",
+			"--no-install-runtime",
+			"--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+		}
+		if super.ClusterType == "production" {
+			cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+		} else {
+			// This would be desirable in the production
+			// case too, but it fails with sudo because
+			// /dev/stderr is a symlink to a pty owned by
+			// root: "nginx: [emerg] open() "/dev/stderr"
+			// failed (13: Permission denied)"
+			cmdline = append(cmdline, "--log-file", "/dev/stderr")
+		}
+		env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
+		err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
 		fail(err)
 	}()
 	return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 7661c6b587..e45c4e1686 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -36,9 +36,13 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
 		return err
 	}
 
+	if super.ClusterType == "production" {
+		return nil
+	}
+
 	iamroot := false
 	if u, err := user.Current(); err != nil {
-		return fmt.Errorf("user.Current(): %s", err)
+		return fmt.Errorf("user.Current(): %w", err)
 	} else if u.Uid == "0" {
 		iamroot = true
 	}
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index 2afccc45b6..1f6cb764e0 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -20,6 +20,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
 	if err != nil {
 		return err
 	}
+	if super.ClusterType == "production" {
+		return nil
+	}
 	err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
 	if err != nil {
 		return err
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 5afacfe716..77fdc98be0 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string {
 }
 
 func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
-	binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
-	err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+	binfile := filepath.Join(super.bindir, "arvados-server")
+	err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
 	if err != nil {
 		return err
 	}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 1e8e83ff3b..20576b6b97 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -14,6 +14,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -54,7 +55,9 @@ type Supervisor struct {
 	tasksReady    map[string]chan bool
 	waitShutdown  sync.WaitGroup
 
+	bindir     string
 	tempdir    string
+	wwwtempdir string
 	configfile string
 	environ    []string // for child processes
 }
@@ -131,13 +134,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		return err
 	}
 
-	super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(super.tempdir)
-	if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
-		return err
+	// Choose bin and temp dirs: /var/lib/arvados/... in
+	// production, transient tempdir otherwise.
+	if super.ClusterType == "production" {
+		// These dirs have already been created by
+		// "arvados-server install" (or by extracting a
+		// package).
+		super.tempdir = "/var/lib/arvados/tmp"
+		super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+		super.bindir = "/var/lib/arvados/bin"
+	} else {
+		super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(super.tempdir)
+		super.wwwtempdir = super.tempdir
+		super.bindir = filepath.Join(super.tempdir, "bin")
+		if err := os.Mkdir(super.bindir, 0755); err != nil {
+			return err
+		}
 	}
 
 	// Fill in any missing config keys, and write the resulting
@@ -166,7 +182,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 	super.setEnv("ARVADOS_CONFIG", super.configfile)
 	super.setEnv("RAILS_ENV", super.ClusterType)
 	super.setEnv("TMPDIR", super.tempdir)
-	super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+	super.prependEnv("PATH", "/var/lib/arvados/bin:")
+	if super.ClusterType != "production" {
+		super.prependEnv("PATH", super.tempdir+"/bin:")
+	}
 
 	super.cluster, err = cfg.GetCluster("")
 	if err != nil {
@@ -182,7 +201,9 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		"PID": os.Getpid(),
 	})
 
-	if super.SourceVersion == "" {
+	if super.SourceVersion == "" && super.ClusterType == "production" {
+		// don't need SourceVersion
+	} else if super.SourceVersion == "" {
 		// Find current source tree version.
 		var buf bytes.Buffer
 		err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
@@ -224,15 +245,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
 		runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
 		runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
 		installPassenger{src: "services/api"},
-		runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
+		runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
 		installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
-		runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+		runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
 		seedDatabase{},
 	}
 	if super.ClusterType != "test" {
 		tasks = append(tasks,
-			runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
-			runGoProgram{src: "services/keep-balance"},
+			runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
+			runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
 		)
 	}
 	super.tasksReady = map[string]chan bool{}
@@ -382,9 +403,11 @@ func dedupEnv(in []string) []string {
 
 func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
 	_, basename := filepath.Split(srcpath)
-	bindir := filepath.Join(super.tempdir, "bin")
-	binfile := filepath.Join(bindir, basename)
-	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+	binfile := filepath.Join(super.bindir, basename)
+	if super.ClusterType == "production" {
+		return binfile, nil
+	}
+	err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
 	return binfile, err
 }
 
@@ -401,10 +424,19 @@ func (super *Supervisor) setupRubyEnv() error {
 			"GEM_PATH=",
 		})
 		gem := "gem"
-		if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+		if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
 			gem = "/var/lib/arvados/bin/gem"
 		}
 		cmd := exec.Command(gem, "env", "gempath")
+		if super.ClusterType == "production" {
+			cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+			path, err := exec.LookPath("sudo")
+			if err != nil {
+				return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+			}
+			cmd.Path = path
+		}
+		cmd.Stderr = super.Stderr
 		cmd.Env = super.environ
 		buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
 		if err != nil || len(buf) == 0 {
@@ -451,17 +483,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
 	super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
 
 	logprefix := prog
-	if logprefix == "setuidgid" && len(args) >= 3 {
-		logprefix = args[2]
-	}
-	logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
-	if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
-		logprefix = args[1]
-	} else if logprefix == "arvados-server" && len(args) > 1 {
-		logprefix = args[0]
-	}
-	if !strings.HasPrefix(dir, "/") {
-		logprefix = dir + ": " + logprefix
+	{
+		if logprefix == "setuidgid" && len(args) >= 3 {
+			logprefix = args[2]
+		}
+		innerargs := args
+		if logprefix == "sudo" {
+			for i := 0; i < len(args); i++ {
+				if args[i] == "-u" {
+					i++
+				} else if args[i] == "-E" || strings.Contains(args[i], "=") {
+				} else {
+					logprefix = args[i]
+					innerargs = args[i+1:]
+					break
+				}
+			}
+		}
+		logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+		logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+		if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+			_, dirbase := filepath.Split(dir)
+			logprefix = innerargs[1] + "@" + dirbase
+		} else if logprefix == "arvados-server" && len(args) > 1 {
+			logprefix = args[0]
+		}
+		if !strings.HasPrefix(dir, "/") {
+			logprefix = dir + ": " + logprefix
+		}
 	}
 
 	cmd := exec.Command(super.lookPath(prog), args...)
@@ -605,30 +654,26 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 			}
 		}
 	}
-	if cluster.SystemRootToken == "" {
-		cluster.SystemRootToken = randomHexString(64)
-	}
-	if cluster.ManagementToken == "" {
-		cluster.ManagementToken = randomHexString(64)
-	}
-	if cluster.API.RailsSessionSecretToken == "" {
-		cluster.API.RailsSessionSecretToken = randomHexString(64)
-	}
-	if cluster.Collections.BlobSigningKey == "" {
-		cluster.Collections.BlobSigningKey = randomHexString(64)
-	}
-	if cluster.Users.AnonymousUserToken == "" {
-		cluster.Users.AnonymousUserToken = randomHexString(64)
-	}
-
-	if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
-		buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
-		if err != nil {
-			return err
-		}
-		cluster.Containers.DispatchPrivateKey = string(buf)
-	}
 	if super.ClusterType != "production" {
+		if cluster.SystemRootToken == "" {
+			cluster.SystemRootToken = randomHexString(64)
+		}
+		if cluster.ManagementToken == "" {
+			cluster.ManagementToken = randomHexString(64)
+		}
+		if cluster.API.RailsSessionSecretToken == "" {
+			cluster.API.RailsSessionSecretToken = randomHexString(64)
+		}
+		if cluster.Collections.BlobSigningKey == "" {
+			cluster.Collections.BlobSigningKey = randomHexString(64)
+		}
+		if cluster.Containers.DispatchPrivateKey == "" {
+			buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+			if err != nil {
+				return err
+			}
+			cluster.Containers.DispatchPrivateKey = string(buf)
+		}
 		cluster.TLS.Insecure = true
 	}
 	if super.ClusterType == "test" {
@@ -698,11 +743,10 @@ func internalPort(svc arvados.Service) (string, error) {
 		return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
 	}
 	for u := range svc.InternalURLs {
-		if _, p, err := net.SplitHostPort(u.Host); err != nil {
-			return "", err
-		} else if p != "" {
+		u := url.URL(u)
+		if p := u.Port(); p != "" {
 			return p, nil
-		} else if u.Scheme == "https" {
+		} else if u.Scheme == "https" || u.Scheme == "ws" {
 			return "443", nil
 		} else {
 			return "80", nil
@@ -712,11 +756,10 @@ func internalPort(svc arvados.Service) (string, error) {
 }
 
 func externalPort(svc arvados.Service) (string, error) {
-	if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
-		return "", err
-	} else if p != "" {
+	u := url.URL(svc.ExternalURL)
+	if p := u.Port(); p != "" {
 		return p, nil
-	} else if svc.ExternalURL.Scheme == "https" {
+	} else if u.Scheme == "https" || u.Scheme == "wss" {
 		return "443", nil
 	} else {
 		return "80", nil
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 342ef03a7f..cc9595db64 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,8 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"os/user"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -24,13 +26,17 @@ import (
 	"github.com/lib/pq"
 )
 
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
 
 const devtestDatabasePassword = "insecure_arvados_test"
 
-type installCommand struct{}
+type installCommand struct {
+	ClusterType    string
+	SourcePath     string
+	PackageVersion string
+}
 
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	logger := ctxlog.New(stderr, "text", "info")
 	ctx := ctxlog.Context(context.Background(), logger)
 	ctx, cancel := context.WithCancel(ctx)
@@ -46,7 +52,9 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
 	flags.SetOutput(stderr)
 	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-	clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+	flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+	flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+	flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -55,18 +63,23 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		return 2
 	} else if *versionFlag {
 		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+	} else if len(flags.Args()) > 0 {
+		err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+		return 2
 	}
 
-	var dev, test, prod bool
-	switch *clusterType {
+	var dev, test, prod, pkg bool
+	switch inst.ClusterType {
 	case "development":
 		dev = true
 	case "test":
 		test = true
 	case "production":
 		prod = true
+	case "package":
+		pkg = true
 	default:
-		err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+		err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
 		return 2
 	}
 
@@ -96,34 +109,34 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		}
 	}
 
-	if dev || test {
-		debs := []string{
+	pkgs := prodpkgs(osv)
+
+	if pkg {
+		pkgs = append(pkgs,
+			"dpkg-dev",
+			"rsync",
+		)
+	}
+
+	if dev || test || pkg {
+		pkgs = append(pkgs,
+			"automake",
 			"bison",
 			"bsdmainutils",
 			"build-essential",
-			"ca-certificates",
 			"cadaver",
 			"curl",
 			"cython3",
 			"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
 			"default-jdk-headless",
 			"default-jre-headless",
-			"fuse",
 			"gettext",
-			"git",
-			"gitolite3",
-			"graphviz",
-			"haveged",
 			"iceweasel",
 			"libattr1-dev",
 			"libcrypt-ssleay-perl",
-			"libcrypt-ssleay-perl",
-			"libcurl3-gnutls",
-			"libcurl4-openssl-dev",
 			"libfuse-dev",
 			"libgnutls28-dev",
 			"libjson-perl",
-			"libjson-perl",
 			"libpam-dev",
 			"libpcre3-dev",
 			"libpq-dev",
@@ -131,11 +144,11 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 			"libssl-dev",
 			"libwww-perl",
 			"libxml2-dev",
-			"libxslt1.1",
+			"libxslt1-dev",
 			"linkchecker",
 			"lsof",
+			"make",
 			"net-tools",
-			"nginx",
 			"pandoc",
 			"perl-modules",
 			"pkg-config",
@@ -154,16 +167,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 			"sudo",
 			"wget",
 			"xvfb",
-			"zlib1g-dev",
-		}
+		)
 		switch {
 		case osv.Debian && osv.Major >= 10:
-			debs = append(debs, "libcurl4")
+			pkgs = append(pkgs, "libcurl4")
 		default:
-			debs = append(debs, "libcurl3")
+			pkgs = append(pkgs, "libcurl3")
 		}
 		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
-		cmd.Args = append(cmd.Args, debs...)
+		cmd.Args = append(cmd.Args, pkgs...)
 		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr
@@ -174,20 +186,36 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 	}
 
 	os.Mkdir("/var/lib/arvados", 0755)
+	os.Mkdir("/var/lib/arvados/tmp", 0700)
+	if prod || pkg {
+		os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+		u, er := user.Lookup("www-data")
+		if er != nil {
+			err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+			return 1
+		}
+		uid, _ := strconv.Atoi(u.Uid)
+		gid, _ := strconv.Atoi(u.Gid)
+		err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+		if err != nil {
+			return 1
+		}
+	}
 	rubyversion := "2.5.7"
 	if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
 		logger.Print("ruby " + rubyversion + " already installed")
 	} else {
 		err = runBash(`
-mkdir -p /var/lib/arvados/tmp
 tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
 trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
 cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
+make -j8
 make install
-/var/lib/arvados/bin/gem install bundler
+/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
+# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
 rm -r ${tmp}
 `, stdout, stderr)
 		if err != nil {
@@ -209,7 +237,9 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
 				return 1
 			}
 		}
+	}
 
+	if !prod && !pkg {
 		pjsversion := "1.9.8"
 		if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
 			logger.Print("phantomjs " + pjsversion + " already installed")
@@ -258,7 +288,6 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 		} else {
 			err = runBash(`
 G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
 zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
 trap "rm ${zip}" ERR
 wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
@@ -392,12 +421,104 @@ rm ${zip}
 		}
 	}
 
+	if prod || pkg {
+		// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+		for dstdir, srcdir := range map[string]string{
+			"railsapi":   "services/api",
+			"workbench1": "apps/workbench",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("rsync",
+				"-a", "--no-owner", "--delete-after", "--delete-excluded",
+				"--exclude", "/coverage",
+				"--exclude", "/log",
+				"--exclude", "/tmp",
+				"--exclude", "/vendor",
+				"./", "/var/lib/arvados/"+dstdir+"/")
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+			for _, cmdline := range [][]string{
+				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
+				{"touch", "log/production.log"},
+				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+			} {
+				cmd = exec.Command(cmdline[0], cmdline[1:]...)
+				cmd.Env = append([]string{}, os.Environ()...)
+				cmd.Dir = "/var/lib/arvados/" + dstdir
+				cmd.Stdout = stdout
+				cmd.Stderr = stderr
+				fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+				err = cmd.Run()
+				if err != nil {
+					return 1
+				}
+			}
+			cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+			cmd.Dir = "/var/lib/arvados/" + dstdir
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+				// Exit code 2 indicates there were warnings (like
+				// "other passenger installations have been detected",
+				// which we can't expect to avoid) but no errors.
+				// Other non-zero exit codes (1, 9) indicate errors.
+				return 1
+			}
+		}
+
+		// Install Go programs to /var/lib/arvados/bin/
+		for _, srcdir := range []string{
+			"cmd/arvados-client",
+			"cmd/arvados-server",
+			"services/arv-git-httpd",
+			"services/crunch-dispatch-local",
+			"services/crunch-dispatch-slurm",
+			"services/health",
+			"services/keep-balance",
+			"services/keep-web",
+			"services/keepproxy",
+			"services/keepstore",
+			"services/ws",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+			cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+		}
+
+		// Copy assets from source tree to /var/lib/arvados/share
+		cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return 1
+		}
+	}
+
 	return 0
 }
 
 type osversion struct {
 	Debian bool
 	Ubuntu bool
+	Centos bool
 	Major  int
 }
 
@@ -435,6 +556,8 @@ func identifyOS() (osversion, error) {
 		osv.Ubuntu = true
 	case "debian":
 		osv.Debian = true
+	case "centos":
+		osv.Centos = true
 	default:
 		return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
 	}
@@ -469,3 +592,54 @@ func runBash(script string, stdout, stderr io.Writer) error {
 	cmd.Stderr = stderr
 	return cmd.Run()
 }
+
+func prodpkgs(osv osversion) []string {
+	pkgs := []string{
+		"ca-certificates",
+		"curl",
+		"fuse",
+		"git",
+		"gitolite3",
+		"graphviz",
+		"haveged",
+		"libcurl3-gnutls",
+		"libxslt1.1",
+		"nginx",
+		"python",
+		"sudo",
+	}
+	if osv.Debian || osv.Ubuntu {
+		if osv.Debian && osv.Major == 8 {
+			pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+		} else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+			pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+		}
+		return append(pkgs,
+			"g++",
+			"libcurl4-openssl-dev", // services/api
+			"libpq-dev",
+			"libpython2.7", // services/fuse
+			"mime-support", // keep-web
+			"zlib1g-dev",   // services/api
+		)
+	} else if osv.Centos {
+		return append(pkgs,
+			"fuse-libs", // services/fuse
+			"gcc",
+			"gcc-c++",
+			"libcurl-devel",    // services/api
+			"mailcap",          // keep-web
+			"postgresql-devel", // services/api
+		)
+	} else {
+		panic("os version not supported")
+	}
+}
+
+func ProductionDependencies() ([]string, error) {
+	osv, err := identifyOS()
+	if err != nil {
+		return nil, err
+	}
+	return prodpkgs(osv), nil
+}
diff --git a/lib/install/init.go b/lib/install/init.go
new file mode 100644
index 0000000000..8debf59be5
--- /dev/null
+++ b/lib/install/init.go
@@ -0,0 +1,269 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strconv"
+	"text/template"
+
+	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+	ClusterID          string
+	Domain             string
+	PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "text", "info")
+	ctx := ctxlog.Context(context.Background(), logger)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var err error
+	defer func() {
+		if err != nil {
+			logger.WithError(err).Info("exiting")
+		}
+	}()
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		err = fmt.Errorf("Hostname(): %w", err)
+		return 1
+	}
+
+	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+	flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+	flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+	err = flags.Parse(args)
+	if err == flag.ErrHelp {
+		err = nil
+		return 0
+	} else if err != nil {
+		return 2
+	} else if *versionFlag {
+		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+	} else if len(flags.Args()) > 0 {
+		err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+		return 2
+	} else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+		err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+		return 1
+	}
+
+	wwwuser, err := user.Lookup("www-data")
+	if err != nil {
+		err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+		return 1
+	}
+	wwwgid, err := strconv.Atoi(wwwuser.Gid)
+	if err != nil {
+		return 1
+	}
+	initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+	err = os.Mkdir("/var/lib/arvados/keep", 0600)
+	if err != nil && !os.IsExist(err) {
+		err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+	err = os.Mkdir("/etc/arvados", 0750)
+	if err != nil && !os.IsExist(err) {
+		err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+		return 1
+	}
+	err = os.Chown("/etc/arvados", 0, wwwgid)
+	f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	if err != nil {
+		err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+		return 1
+	}
+	tmpl, err := template.New("config").Parse(`Clusters:
+  {{.ClusterID}}:
+    Services:
+      Controller:
+        InternalURLs:
+          "http://0.0.0.0:8003/": {}
+        ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+      RailsAPI:
+        InternalURLs:
+          "http://0.0.0.0:8004/": {}
+      Websocket:
+        InternalURLs:
+          "http://0.0.0.0:8005/": {}
+        ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+      Keepbalance:
+        InternalURLs:
+          "http://0.0.0.0:9005/": {}
+      GitHTTP:
+        InternalURLs:
+          "http://0.0.0.0:9001/": {}
+        ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+      DispatchCloud:
+        InternalURLs:
+          "http://0.0.0.0:9006/": {}
+      Keepproxy:
+        InternalURLs:
+          "http://0.0.0.0:25108/": {}
+        ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+      WebDAV:
+        InternalURLs:
+          "http://0.0.0.0:9002/": {}
+        ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+      WebDAVDownload:
+        InternalURLs:
+          "http://0.0.0.0:8004/": {}
+        ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+      Keepstore:
+        InternalURLs:
+          "http://0.0.0.0:25107/": {}
+      Composer:
+        ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+      Workbench1:
+        InternalURLs:
+          "http://0.0.0.0:8001/": {}
+        ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+      Workbench2:
+        InternalURLs:
+          "http://0.0.0.0:8002/": {}
+        ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+      Health:
+        InternalURLs:
+          "http://0.0.0.0:9007/": {}
+    API:
+      RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
+    Collections:
+      BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+    Containers:
+      DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+    ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+    PostgreSQL:
+      Connection:
+        dbname: arvados_production
+        host: localhost
+        user: arvados
+        password: {{printf "%q" .PostgreSQLPassword}}
+    SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+    TLS:
+      Insecure: true
+    Volumes:
+      {{.ClusterID}}-nyw5e-000000000000000:
+        Driver: Directory
+        DriverParameters:
+          Root: /var/lib/arvados/keep
+        Replication: 2
+    Workbench:
+      SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+`)
+	if err != nil {
+		return 1
+	}
+	err = tmpl.Execute(f, initcmd)
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+		return 1
+	}
+	err = f.Close()
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+	ldr := config.NewLoader(nil, logger)
+	ldr.SkipLegacy = true
+	cfg, err := ldr.Load()
+	if err != nil {
+		err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+		return 1
+	}
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return 1
+	}
+
+	err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+	if err != nil {
+		return 1
+	}
+
+	cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+	cmd.Dir = "/var/lib/arvados/railsapi"
+	cmd.Stdout = stderr
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	if err != nil {
+		err = fmt.Errorf("rake db:setup: %w", err)
+		return 1
+	}
+	fmt.Fprintln(stderr, "initialized database")
+
+	return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+	privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return "", err
+	}
+	err = privkey.Validate()
+	if err != nil {
+		return "", err
+	}
+	return string(pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+	})), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+	b := make([]byte, chars/2)
+	_, err := rand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+	for _, sql := range []string{
+		`CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+		`CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+		`CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+	} {
+		cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+		cmd.Stdout = stderr
+		cmd.Stderr = stderr
+		err := cmd.Run()
+		if err != nil {
+			return fmt.Errorf("error setting up arvados user/database: %w", err)
+		}
+	}
+	return nil
+}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 85b4f5b37b..a4336049f2 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -11,17 +11,33 @@ http {
     '[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
     '"$http_referer" "$http_user_agent"';
   access_log "{{ACCESSLOG}}" customlog;
-  client_body_temp_path "{{TMPDIR}}";
-  proxy_temp_path "{{TMPDIR}}";
-  fastcgi_temp_path "{{TMPDIR}}";
-  uwsgi_temp_path "{{TMPDIR}}";
-  scgi_temp_path "{{TMPDIR}}";
+  client_body_temp_path "{{TMPDIR}}/nginx";
+  proxy_temp_path "{{TMPDIR}}/nginx";
+  fastcgi_temp_path "{{TMPDIR}}/nginx";
+  uwsgi_temp_path "{{TMPDIR}}/nginx";
+  scgi_temp_path "{{TMPDIR}}/nginx";
+  upstream controller {
+    server {{LISTENHOST}}:{{CONTROLLERPORT}};
+  }
+  server {
+    listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+    server_name controller ~.*;
+    ssl_certificate "{{SSLCERT}}";
+    ssl_certificate_key "{{SSLKEY}}";
+    location  / {
+      proxy_pass http://controller;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto https;
+      proxy_redirect off;
+    }
+  }
   upstream arv-git-http {
     server {{LISTENHOST}}:{{GITPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
-    server_name arv-git-http;
+    listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+    server_name arv-git-http git.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -36,8 +52,8 @@ http {
     server {{LISTENHOST}}:{{KEEPPROXYPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
-    server_name keepproxy;
+    listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+    server_name keepproxy keep.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -55,8 +71,8 @@ http {
     server {{LISTENHOST}}:{{KEEPWEBPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
-    server_name keep-web;
+    listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+    server_name keep-web collections.* ~\.collections\.;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -75,8 +91,8 @@ http {
     server {{LISTENHOST}}:{{HEALTHPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
-    server_name health;
+    listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+    server_name health health.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -91,8 +107,8 @@ http {
     }
   }
   server {
-    listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
-    server_name keep-web-dl ~.*;
+    listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+    server_name keep-web-dl download.* ~.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -111,8 +127,8 @@ http {
     server {{LISTENHOST}}:{{WSPORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
-    server_name websocket;
+    listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+    server_name websocket ws.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -129,8 +145,8 @@ http {
     server {{LISTENHOST}}:{{WORKBENCH1PORT}};
   }
   server {
-    listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
-    server_name workbench1;
+    listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+    server_name workbench1 workbench.*;
     ssl_certificate "{{SSLCERT}}";
     ssl_certificate_key "{{SSLKEY}}";
     location  / {
@@ -141,20 +157,4 @@ http {
       proxy_redirect off;
     }
   }
-  upstream controller {
-    server {{LISTENHOST}}:{{CONTROLLERPORT}};
-  }
-  server {
-    listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
-    server_name controller;
-    ssl_certificate "{{SSLCERT}}";
-    ssl_certificate_key "{{SSLKEY}}";
-    location  / {
-      proxy_pass http://controller;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-      proxy_redirect off;
-    }
-  }
 }
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 65bd8d4cf0..8b4ee84c71 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -9,13 +9,13 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/service"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/health"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
@@ -83,7 +83,6 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
 			}
 
 			srv := &Server{
-				Handler:    http.NotFoundHandler(),
 				Cluster:    cluster,
 				ArvClient:  ac,
 				RunOptions: options,
@@ -91,6 +90,11 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
 				Logger:     options.Logger,
 				Dumper:     options.Dumper,
 			}
+			srv.Handler = &health.Handler{
+				Token:  cluster.ManagementToken,
+				Prefix: "/_health/",
+				Routes: health.Routes{"ping": srv.CheckHealth},
+			}
 
 			go srv.run()
 			return srv