(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- versions=(1.11.0 1.17.3 2.0.2)
+ versions=(1.16.6 1.17.3 2.0.2)
for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "os/user"
+ "path/filepath"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion == "" {
+ var buf bytes.Buffer
+ cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty")
+ cmd.Stdout = &buf
+ cmd.Stderr = stderr
+ cmd.Dir = opts.SourceDir
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("git describe: %w", err)
+ }
+ opts.PackageVersion = strings.TrimSpace(buf.String())
+ ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion)
+ }
+
+ if opts.PackageChown == "" {
+ whoami, err := user.Current()
+ if err != nil {
+ return fmt.Errorf("user.Current: %w", err)
+ }
+ opts.PackageChown = whoami.Uid + ":" + whoami.Gid
+ }
+
+ // Build in a tempdir, then move to the desired destination
+ // dir. Otherwise, errors might cause us to leave a mess:
+ // truncated files, files owned by root, etc.
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpdir)
+
+ selfbin, err := os.Readlink("/proc/self/exe")
+ if err != nil {
+ return fmt.Errorf("readlink /proc/self/exe: %w", err)
+ }
+ buildImageName := "arvados-package-build-" + opts.TargetOS
+ packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb"
+
+ if ok, err := dockerImageExists(ctx, buildImageName); err != nil {
+ return err
+ } else if !ok || opts.RebuildImage {
+ buildCtrName := strings.Replace(buildImageName, ":", "-", -1)
+ err = dockerRm(ctx, buildCtrName)
+ if err != nil {
+ return err
+ }
+
+ defer dockerRm(ctx, buildCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", buildCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ opts.TargetOS,
+ "/arvados-package", "_install",
+ "-eatmydata",
+ "-type", "package",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+
+ ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName)
+ }
+
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", tmpdir+":/pkg",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ buildImageName,
+ "eatmydata", "/arvados-package", "_fpm",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ "-package-dir", "/pkg",
+ "-package-chown", opts.PackageChown,
+ "-package-maintainer", opts.Maintainer,
+ "-package-vendor", opts.Vendor,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func dockerRm(ctx context.Context, name string) error {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return err
+ }
+ ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1})
+ if err != nil {
+ return err
+ }
+ for _, ctr := range ctrs {
+ for _, ctrname := range ctr.Names {
+ if ctrname == "/"+name {
+ err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{})
+ if err != nil {
+ return fmt.Errorf("error removing container %s: %w", ctr.ID, err)
+ }
+ break
+ }
+ }
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "os"
+ "os/exec"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+ os.Args = append(os.Args, "-test.timeout=30m") // kludge
+}
+
+type BuildSuite struct{}
+
+var _ = check.Suite(&BuildSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+ tmpdir := c.MkDir()
+ defer os.RemoveAll(tmpdir)
+ err := os.Chmod(tmpdir, 0755)
+ c.Assert(err, check.IsNil)
+
+ cmd := exec.Command("go", "run", ".",
+ "build",
+ "-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
+ "-source", "../..",
+ )
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Check(err, check.IsNil)
+
+ fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fi)
+
+ buf, _ := exec.Command("ls", "-l", tmpdir).CombinedOutput()
+ c.Logf("%s", buf)
+
+ cmd = exec.Command("go", "run", ".",
+ "testinstall",
+ "-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
+ )
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Check(err, check.IsNil)
+
+ err = os.RemoveAll(tmpdir)
+ c.Check(err, check.IsNil)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+var (
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "version": cmd.Version,
+ "-version": cmd.Version,
+ "--version": cmd.Version,
+
+ "build": cmdFunc(build),
+ "testinstall": cmdFunc(testinstall),
+ "_fpm": cmdFunc(fpm), // internal use
+ "_install": install.Command, // internal use
+ })
+)
+
+func main() {
+ if len(os.Args) < 2 || strings.HasPrefix(os.Args[1], "-") {
+ parseFlags([]string{"-help"})
+ os.Exit(2)
+ }
+ os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error
+
+func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ opts, err := parseFlags(args)
+ if err != nil {
+ logger.WithError(err).Error("error parsing command line flags")
+ return 1
+ }
+ err = cf(ctx, opts, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type opts struct {
+ PackageVersion string
+ PackageDir string
+ PackageChown string
+ RebuildImage bool
+ SourceDir string
+ TargetOS string
+ Maintainer string
+ Vendor string
+}
+
+func parseFlags(args []string) (opts, error) {
+ opts := opts{
+ SourceDir: ".",
+ TargetOS: "debian:10",
+ Maintainer: "Arvados Package Maintainers <packaging@arvados.org>",
+ Vendor: "The Arvados Project",
+ }
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
+ flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location")
+ flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)")
+ flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
+ flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
+ flags.StringVar(&opts.Maintainer, "package-maintainer", opts.Maintainer, "maintainer to be listed in package metadata")
+ flags.StringVar(&opts.Vendor, "package-vendor", opts.Vendor, "vendor to be listed in package metadata")
+ flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+ flags.Usage = func() {
+ fmt.Fprint(flags.Output(), `Usage: arvados-package <subcommand> [options]
+
+Subcommands:
+ build
+ use a docker container to build a package from a checked
+ out version of the arvados source tree
+ testinstall
+ use a docker container to install a package and confirm
+ the resulting installation is functional
+ version
+ show program version
+
+Internally used subcommands:
+ _fpm
+ build a package
+ _install
+ equivalent to "arvados-server install"
+
+Automation/integration notes:
+ The first time a given machine runs "build" or "testinstall" (and
+ any time the -rebuild-image is used), new docker images are built,
+ which is quite slow. If you use on-demand VMs to run automated builds,
+ run "build" and "testinstall" once when setting up your initial VM
+ image, and be prepared to rebuild that VM image when package-building
+ slows down (this will happen when new dependencies are introduced).
+
+ The "build" subcommand, if successful, also runs
+ dpkg-scanpackages to create/replace Packages.gz in the package
+ dir. This enables the "testinstall" subcommand to list the
+ package dir as a source in /etc/apt/sources.*.
+
+Options:
+`)
+ flags.PrintDefaults()
+ }
+ err := flags.Parse(args)
+ if err != nil {
+ return opts, err
+ }
+ if len(flags.Args()) > 0 {
+ return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if opts.SourceDir == "" {
+ d, err := os.Getwd()
+ if err != nil {
+ return opts, fmt.Errorf("Getwd: %w", err)
+ }
+ opts.SourceDir = d
+ }
+ opts.PackageDir = filepath.Clean(opts.PackageDir)
+ opts.SourceDir, err = filepath.Abs(opts.SourceDir)
+ if err != nil {
+ return opts, err
+ }
+ return opts, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "git.arvados.org/arvados.git/lib/install"
+)
+
+func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ var chownUid, chownGid int
+ if opts.PackageChown != "" {
+ _, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid)
+ if err != nil {
+ return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err)
+ }
+ }
+
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", opts.PackageVersion,
+ "-source", opts.SourceDir,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ }
+
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
+ cmd = exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
+ cmd.Stderr = stderr
+ buf, err := cmd.Output() // /root/.gem/ruby/2.7.0:...
+ if err != nil || len(buf) == 0 {
+ return fmt.Errorf("gem env gempath: %w", err)
+ }
+ gempath := string(bytes.TrimRight(bytes.Split(buf, []byte{':'})[0], "\n"))
+
+ if _, err := os.Stat(gempath + "/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+ // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+ cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, gempath+"/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("monkeypatch fpm: %w", err)
+ }
+ }
+
+ // Remove unneeded files. This is much faster than "fpm
+ // --exclude X" because fpm copies everything into a staging
+ // area before looking at the --exclude args.
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("rm -rf [...]: %w", err)
+ }
+
+ format := "deb" // TODO: rpm
+ pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format)
+
+ cmd = exec.Command(gempath+"/bin/fpm",
+ "--package", pkgfile,
+ "--name", "arvados-server-easy",
+ "--version", opts.PackageVersion,
+ "--url", "https://arvados.org",
+ "--maintainer", opts.Maintainer,
+ "--vendor", opts.Vendor,
+ "--license", "GNU Affero General Public License, version 3.0",
+ "--description", "platform for managing, processing, and sharing genomic and other large scientific and biomedical data",
+ "--input-type", "dir",
+ "--output-type", format)
+ deps, err := install.ProductionDependencies()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range deps {
+ cmd.Args = append(cmd.Args, "--depends", pkg)
+ }
+ cmd.Args = append(cmd.Args,
+ "--verbose",
+ "--deb-use-file-permissions",
+ "--rpm-use-file-permissions",
+ "/var/lib/arvados",
+ "/var/www/.gem",
+ "/var/www/.passenger",
+ "/var/www/.bundle",
+ )
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+ cmd.Dir = opts.PackageDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("fpm: %w", err)
+ }
+
+ if opts.PackageChown != "" {
+ err = os.Chown(pkgfile, chownUid, chownGid)
+ if err != nil {
+ return fmt.Errorf("chown %s: %w", pkgfile, err)
+ }
+ }
+
+ cmd = exec.Command("ls", "-l", pkgfile)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ _ = cmd.Run()
+
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ depsImageName := "arvados-package-deps-" + opts.TargetOS
+ depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir("", prog+".")
+ if err != nil {
+ return fmt.Errorf("TempDir: %w", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ if exists, err := dockerImageExists(ctx, depsImageName); err != nil {
+ return err
+ } else if !exists || opts.RebuildImage {
+ err = dockerRm(ctx, depsCtrName)
+ if err != nil {
+ return err
+ }
+ defer dockerRm(ctx, depsCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", depsCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ opts.TargetOS,
+ "bash", "-c", `
+set -e -o pipefail
+apt-get update
+apt-get install -y --no-install-recommends dpkg-dev eatmydata
+
+mkdir /tmp/pkg
+ln -s /pkg/*.deb /tmp/pkg/
+(cd /tmp/pkg; dpkg-scanpackages --multiversion . | gzip > Packages.gz)
+echo >/etc/apt/sources.list.d/arvados-local.list "deb [trusted=yes] file:///tmp/pkg ./"
+apt-get update
+
+eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql
+eatmydata apt-get remove -y dpkg-dev
+SUDO_FORCE_REMOVE=yes apt-get autoremove -y
+eatmydata apt-get remove -y arvados-server-easy
+rm /etc/apt/sources.list.d/arvados-local.list
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+ }
+
+ versionsuffix := ""
+ if opts.PackageVersion != "" {
+ versionsuffix = "=" + opts.PackageVersion
+ }
+ cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ depsImageName,
+ "bash", "-c", `
+set -e -o pipefail
+PATH="/var/lib/arvados/bin:$PATH"
+apt-get update
+apt-get install -y --no-install-recommends dpkg-dev
+mkdir /tmp/pkg
+ln -s /pkg/*.deb /tmp/pkg/
+(cd /tmp/pkg; dpkg-scanpackages --multiversion . | gzip > Packages.gz)
+apt-get remove -y dpkg-dev
+echo
+
+echo >/etc/apt/sources.list.d/arvados-local.list "deb [trusted=yes] file:///tmp/pkg ./"
+apt-get update
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy`+versionsuffix+`
+SUDO_FORCE_REMOVE=yes apt-get autoremove -y
+
+/etc/init.d/postgresql start
+arvados-server init -cluster-id x1234
+exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+ return nil
+}
+
+func dockerImageExists(ctx context.Context, name string) (bool, error) {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return false, err
+ }
+ imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+ if err != nil {
+ return false, err
+ }
+ for _, img := range imgs {
+ for _, tag := range img.RepoTags {
+ if tag == name {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "init": install.InitCommand,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
})
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'notebox_begin' %}
+This installation method is not fully implemented, which is why this page is not yet listed in the "table of installation options":{{site.baseurl}}/install/index.html or in the left nav.
+{% include 'notebox_end' %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example@gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# apt-get update
+# apt-get install arvados-server-easy
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example@gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
"fmt"
"io/ioutil"
"net"
+ "os"
"path/filepath"
)
func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
var san string
if net.ParseIP(super.ListenHost) != nil {
- san = fmt.Sprintf("IP:%s", super.ListenHost)
+ san += fmt.Sprintf(",IP:%s", super.ListenHost)
} else {
- san = fmt.Sprintf("DNS:%s", super.ListenHost)
+ san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+ }
+ if hostname, err := os.Hostname(); err != nil {
+ return fmt.Errorf("hostname: %w", err)
+ } else {
+ san += ",DNS:" + hostname
}
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
// Generate a self-signed root certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
if err != nil {
return err
}
// Generate server key
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "server.key", "2048")
if err != nil {
return err
}
if err != nil {
return err
}
- err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+ err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
if err != nil {
return err
}
// Generate signing request
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
if err != nil {
return err
}
// Sign certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
if err != nil {
return err
}
fmt.Fprintln(stdout, url)
if *shutdown {
super.Stop()
+ // Wait for children to exit. Don't report the
+ // ensuing "context cancelled" error, though:
+ // return nil to indicate successful startup.
+ _ = super.Wait()
+ fmt.Fprintln(stderr, "PASS - all services booted successfully")
+ return nil
}
}
// Wait for signal/crash + orderly shutdown
"fmt"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"regexp"
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
- "TMPDIR": super.tempdir,
+ "TMPDIR": super.wwwtempdir,
}
for _, cmpt := range []struct {
varname string
} {
port, err := internalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
port, err = externalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
}
vars[cmpt.varname+"SSLPORT"] = port
}
- tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ var conftemplate string
+ if super.ClusterType == "production" {
+ conftemplate = "/var/lib/arvados/share/nginx.conf"
+ } else {
+ conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+ }
+ tmpl, err := ioutil.ReadFile(conftemplate)
if err != nil {
return err
}
}
}
}
+
+ args := []string{
+ "-g", "error_log stderr info;",
+ "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
+ "-c", conffile,
+ }
+ // Nginx ignores "user www-data;" when running as a non-root
+ // user... except that it causes it to ignore our other -g
+ // options. So we still have to decide for ourselves whether
+ // it's needed.
+ if u, err := user.Current(); err != nil {
+ return fmt.Errorf("user.Current(): %w", err)
+ } else if u.Uid == "0" {
+ args = append([]string{"-g", "user www-data;"}, args...)
+ }
+
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", nil, nil, nginx,
- "-g", "error_log stderr info;",
- "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
- "-c", conffile))
+ fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
}()
- return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+ // Choose one of the ports where Nginx should listen, and wait
+ // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+ testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+ if testurl.Port() == "" {
+ testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+ }
+ return waitForConnect(ctx, testurl.Host)
}
}
func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if super.ClusterType == "production" {
+ // passenger has already been installed via package
+ return nil
+ }
err := super.wait(ctx, runner.depends...)
if err != nil {
return err
defer passengerInstallMutex.Unlock()
var buf bytes.Buffer
- err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler")
+ err = super.RunProgram(ctx, runner.src, runOptions{output: &buf}, "gem", "list", "--details", "bundler")
if err != nil {
return err
}
- for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+ for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
break
}
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "build-native-support")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "install-standalone-runtime")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "validate-install")
if err != nil && !strings.Contains(err.Error(), "exit status 2") {
// Exit code 2 indicates there were warnings (like
// "other passenger installations have been detected",
}
type runPassenger struct {
- src string
- svc arvados.Service
- depends []supervisedTask
+ src string // path to app in source tree
+ varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+ svc arvados.Service
+ depends []supervisedTask
}
func (runner runPassenger) String() string {
if err != nil {
return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
}
+ var appdir string
+ if super.ClusterType == "production" {
+ appdir = "/var/lib/arvados/" + runner.varlibdir
+ } else {
+ appdir = runner.src
+ }
loglevel := "4"
if lvl, ok := map[string]string{
"debug": "5",
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+ cmdline := []string{
+ "bundle", "exec",
"passenger", "start",
"-p", port,
- "--log-file", "/dev/stderr",
"--log-level", loglevel,
"--no-friendly-error-pages",
- "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+ "--disable-anonymous-telemetry",
+ "--disable-security-update-check",
+ "--no-compile-runtime",
+ "--no-install-runtime",
+ "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+ }
+ opts := runOptions{
+ env: append([]string{
+ "TMPDIR=" + super.wwwtempdir,
+ }, railsEnv...),
+ }
+ if super.ClusterType == "production" {
+ opts.user = "www-data"
+ opts.env = append(opts.env, "HOME=/var/www")
+ } else {
+ // This would be desirable when changing uid
+ // too, but it fails because /dev/stderr is a
+ // symlink to a pty owned by root: "nginx:
+ // [emerg] open() "/dev/stderr" failed (13:
+ // Permission denied)"
+ cmdline = append(cmdline, "--log-file", "/dev/stderr")
+ }
+ err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
fail(err)
}()
return nil
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
+
iamroot := false
if u, err := user.Current(); err != nil {
- return fmt.Errorf("user.Current(): %s", err)
+ return fmt.Errorf("user.Current(): %w", err)
} else if u.Uid == "0" {
iamroot = true
}
buf := bytes.NewBuffer(nil)
- err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{output: buf}, "pg_config", "--bindir")
if err != nil {
return err
}
return err
}
prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+ opts := runOptions{}
if iamroot {
postgresUser, err := user.Lookup("postgres")
if err != nil {
if err != nil {
return err
}
- // We can't use "sudo -u" here because it creates an
- // intermediate process that interferes with our
- // ability to reliably kill postgres. The setuidgid
- // program just calls exec without forking, so it
- // doesn't have this problem.
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
- }
- err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
+ opts.user = "postgres"
+ }
+ err = super.RunProgram(ctx, super.tempdir, opts, prog, args...)
if err != nil {
return err
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir)
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "cp", "server.crt", "server.key", datadir)
if err != nil {
return err
}
if iamroot {
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
if err != nil {
return err
}
"-k", datadir, // socket dir
"-p", super.cluster.PostgreSQL.Connection["port"],
}
+ opts := runOptions{}
if iamroot {
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
+ opts.user = "postgres"
}
- fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
+ fail(super.RunProgram(ctx, super.tempdir, opts, prog, args...))
}()
for {
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
+ if super.ClusterType == "production" {
+ return nil
+ }
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb")
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb")
if err != nil {
return err
}
}
func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
- binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
- err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ binfile := filepath.Join(super.bindir, "arvados-server")
+ err := super.RunProgram(ctx, super.bindir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
}()
}
return nil
return ctx.Err()
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile))
}()
}
return nil
"io"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"os/signal"
"os/user"
"path/filepath"
"reflect"
+ "strconv"
"strings"
"sync"
"syscall"
tasksReady map[string]chan bool
waitShutdown sync.WaitGroup
+ bindir string
tempdir string
+ wwwtempdir string
configfile string
environ []string // for child processes
}
return err
}
- super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
- if err != nil {
- return err
- }
- defer os.RemoveAll(super.tempdir)
- if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
- return err
+ // Choose bin and temp dirs: /var/lib/arvados/... in
+ // production, transient tempdir otherwise.
+ if super.ClusterType == "production" {
+ // These dirs have already been created by
+ // "arvados-server install" (or by extracting a
+ // package).
+ super.tempdir = "/var/lib/arvados/tmp"
+ super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+ super.bindir = "/var/lib/arvados/bin"
+ } else {
+ super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(super.tempdir)
+ super.wwwtempdir = super.tempdir
+ super.bindir = filepath.Join(super.tempdir, "bin")
+ if err := os.Mkdir(super.bindir, 0755); err != nil {
+ return err
+ }
}
// Fill in any missing config keys, and write the resulting
if err != nil {
return err
}
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+ super.prependEnv("PATH", "/var/lib/arvados/bin:")
+ if super.ClusterType != "production" {
+ super.prependEnv("PATH", super.tempdir+"/bin:")
+ }
super.cluster, err = cfg.GetCluster("")
if err != nil {
"PID": os.Getpid(),
})
- if super.SourceVersion == "" {
+ if super.SourceVersion == "" && super.ClusterType == "production" {
+ // don't need SourceVersion
+ } else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat")
if err != nil {
return err
}
dirty := buf.Len() > 0
buf.Reset()
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H")
if err != nil {
return err
}
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
- runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
if super.ClusterType != "test" {
tasks = append(tasks,
- runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
- runGoProgram{src: "services/keep-balance"},
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
+ runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
super.tasksReady = map[string]chan bool{}
func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
_, basename := filepath.Split(srcpath)
- bindir := filepath.Join(super.tempdir, "bin")
- binfile := filepath.Join(bindir, basename)
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ binfile := filepath.Join(super.bindir, basename)
+ if super.ClusterType == "production" {
+ return binfile, nil
+ }
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
"GEM_PATH=",
})
gem := "gem"
- if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
gem = "/var/lib/arvados/bin/gem"
}
cmd := exec.Command(gem, "env", "gempath")
+ if super.ClusterType == "production" {
+ cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+ path, err := exec.LookPath("sudo")
+ if err != nil {
+ return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+ }
+ cmd.Path = path
+ }
+ cmd.Stderr = super.Stderr
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
- return fmt.Errorf("gem env gempath: %v", err)
+ return fmt.Errorf("gem env gempath: %w", err)
}
gempath := string(bytes.Split(buf, []byte{':'})[0])
super.prependEnv("PATH", gempath+"/bin:")
return prog
}
+type runOptions struct {
+ output io.Writer // attach stdout
+ env []string // add/replace environment variables
+ user string // run as specified user
+}
+
// RunProgram runs prog with args, using dir as working directory. If ctx is
// cancelled while the child is running, RunProgram terminates the child, waits
// for it to exit, then returns.
//
// Child's stdout will be written to output if non-nil, otherwise the
// boot command's stderr.
-func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error {
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "setuidgid" && len(args) >= 2 {
- logprefix = args[1]
- }
- logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
- if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
- logprefix = args[1]
- } else if logprefix == "arvados-server" && len(args) > 1 {
- logprefix = args[0]
- }
- if !strings.HasPrefix(dir, "/") {
- logprefix = dir + ": " + logprefix
+ {
+ innerargs := args
+ if logprefix == "sudo" {
+ for i := 0; i < len(args); i++ {
+ if args[i] == "-u" {
+ i++
+ } else if args[i] == "-E" || strings.Contains(args[i], "=") {
+ } else {
+ logprefix = args[i]
+ innerargs = args[i+1:]
+ break
+ }
+ }
+ }
+ logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+ if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+ _, dirbase := filepath.Split(dir)
+ logprefix = innerargs[1] + "@" + dirbase
+ } else if logprefix == "arvados-server" && len(args) > 1 {
+ logprefix = args[0]
+ }
+ if !strings.HasPrefix(dir, "/") {
+ logprefix = dir + ": " + logprefix
+ }
}
cmd := exec.Command(super.lookPath(prog), args...)
}()
copiers.Add(1)
go func() {
- if output == nil {
+ if opts.output == nil {
io.Copy(logwriter, stdout)
} else {
- io.Copy(output, stdout)
+ io.Copy(opts.output, stdout)
}
copiers.Done()
}()
} else {
cmd.Dir = filepath.Join(super.SourcePath, dir)
}
- env = append([]string(nil), env...)
+ env := append([]string(nil), opts.env...)
env = append(env, super.environ...)
cmd.Env = dedupEnv(env)
+ if opts.user != "" {
+ // Note: We use this approach instead of "sudo"
+ // because in certain circumstances (we are pid 1 in a
+ // docker container, and our passenger child process
+ // changes to pgid 1) the intermediate sudo process
+ // notices we have the same pgid as our child and
+ // refuses to propagate signals from us to our child,
+ // so we can't signal/shutdown our passenger/rails
+ // apps. "chpst" or "setuidgid" would work, but these
+ // few lines avoid depending on runit/daemontools.
+ u, err := user.Lookup(opts.user)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ Credential: &syscall.Credential{
+ Uid: uint32(uid),
+ Gid: uint32(gid),
+ },
+ }
+ }
+
exited := false
defer func() { exited = true }()
go func() {
}
}
}
- if cluster.SystemRootToken == "" {
- cluster.SystemRootToken = randomHexString(64)
- }
- if cluster.ManagementToken == "" {
- cluster.ManagementToken = randomHexString(64)
- }
- if cluster.Collections.BlobSigningKey == "" {
- cluster.Collections.BlobSigningKey = randomHexString(64)
- }
- if cluster.Users.AnonymousUserToken == "" {
- cluster.Users.AnonymousUserToken = randomHexString(64)
- }
-
- if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
- buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
- if err != nil {
- return err
- }
- cluster.Containers.DispatchPrivateKey = string(buf)
- }
if super.ClusterType != "production" {
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
+ if cluster.Users.AnonymousUserToken == "" {
+ cluster.Users.AnonymousUserToken = randomHexString(64)
+ }
+ if cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
cluster.TLS.Insecure = true
}
if super.ClusterType == "test" {
return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
}
for u := range svc.InternalURLs {
- if _, p, err := net.SplitHostPort(u.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(u)
+ if p := u.Port(); p != "" {
return p, nil
- } else if u.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "ws" {
return "443", nil
} else {
return "80", nil
}
func externalPort(svc arvados.Service) (string, error) {
- if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(svc.ExternalURL)
+ if p := u.Port(); p != "" {
return p, nil
- } else if svc.ExternalURL.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "wss" {
return "443", nil
} else {
return "80", nil
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
})
mux.Handler("GET", "/metrics", metricsH)
mux.Handler("GET", "/metrics.json", metricsH)
+ mux.Handler("GET", "/_health/:check", &health.Handler{
+ Token: disp.Cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": disp.CheckHealth},
+ })
disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
}
}
"io"
"os"
"os/exec"
+ "os/user"
+ "path/filepath"
"strconv"
"strings"
"syscall"
"github.com/lib/pq"
)
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
const devtestDatabasePassword = "insecure_arvados_test"
-type installCommand struct{}
+type installCommand struct {
+ ClusterType string
+ SourcePath string
+ PackageVersion string
+ EatMyData bool
+}
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.New(stderr, "text", "info")
ctx := ctxlog.Context(context.Background(), logger)
ctx, cancel := context.WithCancel(ctx)
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.SetOutput(stderr)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
- clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+ flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+ flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
+ flags.BoolVar(&inst.EatMyData, "eatmydata", false, "use eatmydata to speed up install")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
return 2
} else if *versionFlag {
return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
}
- var dev, test, prod bool
- switch *clusterType {
+ var dev, test, prod, pkg bool
+ switch inst.ClusterType {
case "development":
dev = true
case "test":
test = true
case "production":
prod = true
+ case "package":
+ pkg = true
default:
- err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+ err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
return 2
}
}
}
- if dev || test {
- debs := []string{
+ if inst.EatMyData {
+ cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends", "eatmydata")
+ cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ pkgs := prodpkgs(osv)
+
+ if pkg {
+ pkgs = append(pkgs,
+ "dpkg-dev",
+ "eatmydata", // install it for later steps, even if we're not using it now
+ "rsync",
+ )
+ }
+
+ if dev || test || pkg {
+ pkgs = append(pkgs,
+ "automake",
"bison",
"bsdmainutils",
"build-essential",
- "ca-certificates",
"cadaver",
"curl",
"cython3",
- "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
- "fuse",
"gettext",
- "git",
- "gitolite3",
- "graphviz",
- "haveged",
"iceweasel",
"libattr1-dev",
"libcrypt-ssleay-perl",
- "libcrypt-ssleay-perl",
- "libcurl3-gnutls",
- "libcurl4-openssl-dev",
"libfuse-dev",
"libgnutls28-dev",
"libjson-perl",
- "libjson-perl",
"libpam-dev",
"libpcre3-dev",
"libpq-dev",
"libssl-dev",
"libwww-perl",
"libxml2-dev",
- "libxslt1.1",
+ "libxslt1-dev",
"linkchecker",
"lsof",
+ "make",
"net-tools",
- "nginx",
"pandoc",
"perl-modules",
"pkg-config",
"sudo",
"wget",
"xvfb",
- "zlib1g-dev",
- }
+ )
switch {
case osv.Debian && osv.Major >= 10:
- debs = append(debs, "libcurl4")
+ pkgs = append(pkgs, "libcurl4")
default:
- debs = append(debs, "libcurl3")
+ pkgs = append(pkgs, "libcurl3")
}
- cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
- cmd.Args = append(cmd.Args, debs...)
+ cmd := exec.CommandContext(ctx, "apt-get")
+ if inst.EatMyData {
+ cmd = exec.CommandContext(ctx, "eatmydata", "apt-get")
+ }
+ cmd.Args = append(cmd.Args, "install", "--yes", "--no-install-recommends")
+ cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
cmd.Stderr = stderr
}
os.Mkdir("/var/lib/arvados", 0755)
- rubyversion := "2.5.7"
+ os.Mkdir("/var/lib/arvados/tmp", 0700)
+ if prod || pkg {
+ os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+ u, er := user.Lookup("www-data")
+ if er != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+ return 1
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+ if err != nil {
+ return 1
+ }
+ }
+ rubyversion := "2.7.2"
+ rubymajorversion := rubyversion[:strings.LastIndex(rubyversion, ".")]
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
} else {
- err = runBash(`
-mkdir -p /var/lib/arvados/tmp
-tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
-trap "rm -r ${tmp}" ERR
-wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+ err = inst.runBash(`
+tmp="$(mktemp -d)"
+trap 'rm -r "${tmp}"' ERR EXIT
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/`+rubymajorversion+`/ruby-`+rubyversion+`.tar.gz | tar -C "${tmp}" -xzf -
+cd "${tmp}/ruby-`+rubyversion+`"
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
+make -j8
make install
-/var/lib/arvados/bin/gem install bundler
-rm -r ${tmp}
+/var/lib/arvados/bin/gem install bundler --no-document
`, stdout, stderr)
if err != nil {
return 1
if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
logger.Print("go " + goversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
cd /tmp
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
return 1
}
}
+ }
+ if !prod && !pkg {
pjsversion := "1.9.8"
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
logger.Print("geckodriver " + geckoversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
GD=v`+geckoversion+`
wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
NJS=`+nodejsversion+`
wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
logger.Print("gradle " + gradleversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
logger.Print("locale " + wantlocale + " already installed")
} else {
- err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+ err = inst.runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
if err != nil {
return 1
}
// locales. Otherwise, it might need a
// restart, so we attempt to restart it with
// systemd.
- if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+ if err = inst.runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
} else if err = waitPostgreSQLReady(); err != nil {
return 1
}
}
+ if prod || pkg {
+ // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+ for dstdir, srcdir := range map[string]string{
+ "railsapi": "services/api",
+ "workbench1": "apps/workbench",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("rsync",
+ "-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded",
+ "--exclude", "/coverage",
+ "--exclude", "/log",
+ "--exclude", "/tmp",
+ "--exclude", "/vendor",
+ "--exclude", "/config/environments",
+ "./", "/var/lib/arvados/"+dstdir+"/")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ for _, cmdline := range [][]string{
+ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
+ {"touch", "log/production.log"},
+ {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+ } {
+ cmd = exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+ // Exit code 2 indicates there were warnings (like
+ // "other passenger installations have been detected",
+ // which we can't expect to avoid) but no errors.
+ // Other non-zero exit codes (1, 9) indicate errors.
+ return 1
+ }
+ }
+
+ // Install Go programs to /var/lib/arvados/bin/
+ for _, srcdir := range []string{
+ "cmd/arvados-client",
+ "cmd/arvados-server",
+ "services/arv-git-httpd",
+ "services/crunch-dispatch-local",
+ "services/crunch-dispatch-slurm",
+ "services/health",
+ "services/keep-balance",
+ "services/keep-web",
+ "services/keepproxy",
+ "services/keepstore",
+ "services/ws",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+ cmd.Env = append(cmd.Env, os.Environ()...)
+ cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ // Copy assets from source tree to /var/lib/arvados/share
+ cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
return 0
}
type osversion struct {
Debian bool
Ubuntu bool
+ Centos bool
Major int
}
osv.Ubuntu = true
case "debian":
osv.Debian = true
+ case "centos":
+ osv.Centos = true
default:
return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
}
}
}
-func runBash(script string, stdout, stderr io.Writer) error {
+func (inst *installCommand) runBash(script string, stdout, stderr io.Writer) error {
cmd := exec.Command("bash", "-")
+ if inst.EatMyData {
+ cmd = exec.Command("eatmydata", "bash", "-")
+ }
cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
cmd.Stdout = stdout
cmd.Stderr = stderr
return cmd.Run()
}
+
+func prodpkgs(osv osversion) []string {
+ pkgs := []string{
+ "ca-certificates",
+ "curl",
+ "fuse",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "libcurl3-gnutls",
+ "libxslt1.1",
+ "nginx",
+ "python",
+ "sudo",
+ }
+ if osv.Debian || osv.Ubuntu {
+ if osv.Debian && osv.Major == 8 {
+ pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+ } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+ pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+ }
+ return append(pkgs,
+ "g++",
+ "libcurl4-openssl-dev", // services/api
+ "libpq-dev",
+ "libpython2.7", // services/fuse
+ "mime-support", // keep-web
+ "zlib1g-dev", // services/api
+ )
+ } else if osv.Centos {
+ return append(pkgs,
+ "fuse-libs", // services/fuse
+ "gcc",
+ "gcc-c++",
+ "libcurl-devel", // services/api
+ "mailcap", // keep-web
+ "postgresql-devel", // services/api
+ )
+ } else {
+ panic("os version not supported")
+ }
+}
+
+func ProductionDependencies() ([]string, error) {
+ osv, err := identifyOS()
+ if err != nil {
+ return nil, err
+ }
+ return prodpkgs(osv), nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/user"
+ "regexp"
+ "strconv"
+ "text/template"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+ ClusterID string
+ Domain string
+ PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("Hostname(): %w", err)
+ return 1
+ }
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+ flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
+ } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+ err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+ return 1
+ }
+
+ wwwuser, err := user.Lookup("www-data")
+ if err != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+ return 1
+ }
+ wwwgid, err := strconv.Atoi(wwwuser.Gid)
+ if err != nil {
+ return 1
+ }
+ initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+ err = os.Mkdir("/var/lib/arvados/keep", 0600)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+ err = os.Mkdir("/etc/arvados", 0750)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ return 1
+ }
+ err = os.Chown("/etc/arvados", 0, wwwgid)
+ f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ tmpl, err := template.New("config").Parse(`Clusters:
+ {{.ClusterID}}:
+ Services:
+ Controller:
+ InternalURLs:
+ "http://0.0.0.0:8003/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ Websocket:
+ InternalURLs:
+ "http://0.0.0.0:8005/": {}
+ ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ Keepbalance:
+ InternalURLs:
+ "http://0.0.0.0:9005/": {}
+ GitHTTP:
+ InternalURLs:
+ "http://0.0.0.0:9001/": {}
+ ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ DispatchCloud:
+ InternalURLs:
+ "http://0.0.0.0:9006/": {}
+ Keepproxy:
+ InternalURLs:
+ "http://0.0.0.0:25108/": {}
+ ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ WebDAV:
+ InternalURLs:
+ "http://0.0.0.0:9002/": {}
+ ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ WebDAVDownload:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ Keepstore:
+ InternalURLs:
+ "http://0.0.0.0:25107/": {}
+ Composer:
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ Workbench1:
+ InternalURLs:
+ "http://0.0.0.0:8001/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+ #Workbench2:
+ # InternalURLs:
+ # "http://0.0.0.0:8002/": {}
+ # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ Health:
+ InternalURLs:
+ "http://0.0.0.0:9007/": {}
+ Collections:
+ BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+ Containers:
+ DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+ ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+ PostgreSQL:
+ Connection:
+ dbname: arvados_production
+ host: localhost
+ user: arvados
+ password: {{printf "%q" .PostgreSQLPassword}}
+ SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ TLS:
+ Insecure: true
+ Volumes:
+ {{.ClusterID}}-nyw5e-000000000000000:
+ Driver: Directory
+ DriverParameters:
+ Root: /var/lib/arvados/keep
+ Replication: 2
+ Workbench:
+ SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+`)
+ if err != nil {
+ return 1
+ }
+ err = tmpl.Execute(f, initcmd)
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ return 1
+ }
+ err = f.Close()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+ ldr := config.NewLoader(nil, logger)
+ ldr.SkipLegacy = true
+ cfg, err := ldr.Load()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+
+ err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+ if err != nil {
+ return 1
+ }
+
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ cmd.Dir = "/var/lib/arvados/railsapi"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("rake db:setup: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "initialized database")
+
+ return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+ privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+ if err != nil {
+ return "", err
+ }
+ err = privkey.Validate()
+ if err != nil {
+ return "", err
+ }
+ return string(pem.EncodeToMemory(&pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+ })), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+ b := make([]byte, chars/2)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+ for _, sql := range []string{
+ `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+ `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+ } {
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
+ }
+ }
+ return nil
+}
}
req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
+ // Avoid workbench1's redirect-http-to-https feature
+ req.Header.Set("X-Forwarded-Proto", "https")
+
ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout))
defer cancel()
req = req.WithContext(ctx)
'[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
- client_body_temp_path "{{TMPDIR}}";
- proxy_temp_path "{{TMPDIR}}";
- fastcgi_temp_path "{{TMPDIR}}";
- uwsgi_temp_path "{{TMPDIR}}";
- scgi_temp_path "{{TMPDIR}}";
+ client_body_temp_path "{{TMPDIR}}/nginx";
+ proxy_temp_path "{{TMPDIR}}/nginx";
+ fastcgi_temp_path "{{TMPDIR}}/nginx";
+ uwsgi_temp_path "{{TMPDIR}}/nginx";
+ scgi_temp_path "{{TMPDIR}}/nginx";
+ upstream controller {
+ server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+ server_name controller ~.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream arv-git-http {
server {{LISTENHOST}}:{{GITPORT}};
}
server {
- listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
- server_name arv-git-http;
+ listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+ server_name arv-git-http git.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{KEEPPROXYPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name keepproxy;
+ listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+ server_name keepproxy keep.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{KEEPWEBPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name keep-web;
+ listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+ server_name keep-web collections.* ~\.collections\.;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{HEALTHPORT}};
}
server {
- listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
- server_name health;
+ listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+ server_name health health.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
}
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name keep-web-dl ~.*;
+ listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+ server_name keep-web-dl download.* ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{WSPORT}};
}
server {
- listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
- server_name websocket;
+ listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+ server_name websocket ws.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{WORKBENCH1PORT}};
}
server {
- listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
- server_name workbench1;
+ listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+ server_name workbench1 workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_redirect off;
}
}
- upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
- }
- server {
- listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
- server_name controller;
- ssl_certificate "{{SSLCERT}}";
- ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://controller;
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
}
"flag"
"fmt"
"io"
- "net/http"
"os"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
}
srv := &Server{
- Handler: http.NotFoundHandler(),
Cluster: cluster,
ArvClient: ac,
RunOptions: options,
Logger: options.Logger,
Dumper: options.Dumper,
}
+ srv.Handler = &health.Handler{
+ Token: cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": srv.CheckHealth},
+ }
go srv.run()
return srv
projects / arvados.git / commitdiff