From: Tom Clegg Date: Thu, 21 Jan 2021 21:33:40 +0000 (-0500) Subject: 16306: Merge branch 'master' X-Git-Tag: 2.2.0~141^2~11 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/8cd08f2ce640e0b1967db489d29e3761ac63f0d7?hp=4c6c49190b5a8949120d822e053657f64146df70 16306: Merge branch 'master' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/build/run-tests.sh b/build/run-tests.sh index 595f721080..7bd4e618dd 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -550,7 +550,7 @@ setup_ruby_environment() { ( export HOME=$GEMHOME bundlers="$(gem list --details bundler)" - versions=(1.11.0 1.17.3 2.0.2) + versions=(1.16.6 1.17.3 2.0.2) for v in ${versions[@]}; do if ! echo "$bundlers" | fgrep -q "($v)"; then gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go new file mode 100644 index 0000000000..cf9cbfa6c3 --- /dev/null +++ b/cmd/arvados-package/build.go @@ -0,0 +1,162 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error { + if opts.PackageVersion == "" { + var buf bytes.Buffer + cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty") + cmd.Stdout = &buf + cmd.Stderr = stderr + cmd.Dir = opts.SourceDir + err := cmd.Run() + if err != nil { + return fmt.Errorf("git describe: %w", err) + } + opts.PackageVersion = strings.TrimSpace(buf.String()) + ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion) + } + + if opts.PackageChown == "" { + whoami, err := user.Current() + if err != nil { + return fmt.Errorf("user.Current: %w", err) + } + opts.PackageChown = whoami.Uid + ":" + whoami.Gid + } + + // Build in a tempdir, then move to the desired destination + // dir. Otherwise, errors might cause us to leave a mess: + // truncated files, files owned by root, etc. + _, prog := filepath.Split(os.Args[0]) + tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".") + if err != nil { + return err + } + defer os.RemoveAll(tmpdir) + + selfbin, err := os.Readlink("/proc/self/exe") + if err != nil { + return fmt.Errorf("readlink /proc/self/exe: %w", err) + } + buildImageName := "arvados-package-build-" + opts.TargetOS + packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb" + + if ok, err := dockerImageExists(ctx, buildImageName); err != nil { + return err + } else if !ok || opts.RebuildImage { + buildCtrName := strings.Replace(buildImageName, ":", "-", -1) + err = dockerRm(ctx, buildCtrName) + if err != nil { + return err + } + + defer dockerRm(ctx, buildCtrName) + cmd := exec.CommandContext(ctx, "docker", "run", + "--name", buildCtrName, + "--tmpfs", "/tmp:exec,mode=01777", + "-v", selfbin+":/arvados-package:ro", + "-v", opts.SourceDir+":/arvados:ro", + opts.TargetOS, + "/arvados-package", "_install", + "-eatmydata", + "-type", "package", + "-source", "/arvados", + "-package-version", opts.PackageVersion, + ) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker run: %w", err) + } + + cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker commit: %w", err) + } + + ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName) + } + + cmd := exec.CommandContext(ctx, "docker", "run", + "--rm", + "--tmpfs", "/tmp:exec,mode=01777", + "-v", tmpdir+":/pkg", + "-v", selfbin+":/arvados-package:ro", + "-v", opts.SourceDir+":/arvados:ro", + buildImageName, + "eatmydata", "/arvados-package", "_fpm", + "-source", "/arvados", + "-package-version", opts.PackageVersion, + "-package-dir", "/pkg", + "-package-chown", opts.PackageChown, + ) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker run: %w", err) + } + + err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename) + if err != nil { + return err + } + + cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz") + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Dir = opts.PackageDir + err = cmd.Run() + if err != nil { + return fmt.Errorf("dpkg-scanpackages: %w", err) + } + + return nil +} + +func dockerRm(ctx context.Context, name string) error { + cli, err := client.NewEnvClient() + if err != nil { + return err + } + ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1}) + if err != nil { + return err + } + for _, ctr := range ctrs { + for _, ctrname := range ctr.Names { + if ctrname == "/"+name { + err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{}) + if err != nil { + return fmt.Errorf("error removing container %s: %w", ctr.ID, err) + } + break + } + } + } + return nil +} diff --git a/cmd/arvados-package/build_test.go b/cmd/arvados-package/build_test.go new file mode 100644 index 0000000000..b4ad68b1df --- /dev/null +++ b/cmd/arvados-package/build_test.go @@ -0,0 +1,67 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "os" + "os/exec" + "testing" + + "gopkg.in/check.v1" +) + +var buildimage string + +func init() { + os.Args = append(os.Args, "-test.timeout=30m") // kludge +} + +type BuildSuite struct{} + +var _ = check.Suite(&BuildSuite{}) + +func Test(t *testing.T) { check.TestingT(t) } + +func (s *BuildSuite) TestBuildAndInstall(c *check.C) { + if testing.Short() { + c.Skip("skipping docker tests in short mode") + } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil { + c.Skip("skipping docker tests because docker is not available") + } + tmpdir := c.MkDir() + defer os.RemoveAll(tmpdir) + err := os.Chmod(tmpdir, 0755) + c.Assert(err, check.IsNil) + + cmd := exec.Command("go", "run", ".", + "build", + "-package-dir", tmpdir, + "-package-version", "1.2.3~rc4", + "-source", "../..", + ) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + err = cmd.Run() + c.Check(err, check.IsNil) + + fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb") + c.Assert(err, check.IsNil) + c.Logf("%#v", fi) + + buf, _ := exec.Command("ls", "-l", tmpdir).CombinedOutput() + c.Logf("%s", buf) + + cmd = exec.Command("go", "run", ".", + "testinstall", + "-package-dir", tmpdir, + ) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + err = cmd.Run() + c.Check(err, check.IsNil) + + err = os.RemoveAll(tmpdir) + c.Check(err, check.IsNil) +} diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go new file mode 100644 index 0000000000..b0c0e9e3c7 --- /dev/null +++ b/cmd/arvados-package/cmd.go @@ -0,0 +1,132 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/install" + "git.arvados.org/arvados.git/sdk/go/ctxlog" +) + +var ( + handler = cmd.Multi(map[string]cmd.Handler{ + "version": cmd.Version, + "-version": cmd.Version, + "--version": cmd.Version, + + "build": cmdFunc(build), + "testinstall": cmdFunc(testinstall), + "_fpm": cmdFunc(fpm), // internal use + "_install": install.Command, // internal use + }) +) + +func main() { + if len(os.Args) < 2 || strings.HasPrefix(os.Args[1], "-") { + parseFlags([]string{"-help"}) + os.Exit(2) + } + os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} + +type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error + +func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + logger := ctxlog.New(stderr, "text", "info") + ctx := ctxlog.Context(context.Background(), logger) + opts, err := parseFlags(args) + if err != nil { + logger.WithError(err).Error("error parsing command line flags") + return 1 + } + err = cf(ctx, opts, stdin, stdout, stderr) + if err != nil { + logger.WithError(err).Error("failed") + return 1 + } + return 0 +} + +type opts struct { + PackageVersion string + PackageDir string + PackageChown string + RebuildImage bool + SourceDir string + TargetOS string +} + +func parseFlags(args []string) (opts, error) { + opts := opts{ + SourceDir: ".", + TargetOS: "debian:10", + } + flags := flag.NewFlagSet("", flag.ContinueOnError) + flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"") + flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location") + flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)") + flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)") + flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version") + flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing") + flags.Usage = func() { + fmt.Fprint(flags.Output(), `Usage: arvados-package [options] + +Subcommands: + build + use a docker container to build a package from a checked + out version of the arvados source tree + testinstall + use a docker container to install a package and confirm + the resulting installation is functional + version + show program version + +Internally used subcommands: + _fpm + build a package + _install + equivalent to "arvados-server install" + +Automation/integration notes: + The first time a given machine runs "build" or "testinstall" (and + any time the -rebuild-image is used), new docker images are built, + which is quite slow. If you use on-demand VMs to run automated builds, + run "build" and "testinstall" once when setting up your initial VM + image, and be prepared to rebuild that VM image when package-building + slows down (this will happen when new dependencies are introduced). + +Options: +`) + flags.PrintDefaults() + } + err := flags.Parse(args) + if err != nil { + return opts, err + } + if len(flags.Args()) > 0 { + return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) + } + if opts.SourceDir == "" { + d, err := os.Getwd() + if err != nil { + return opts, fmt.Errorf("Getwd: %w", err) + } + opts.SourceDir = d + } + opts.PackageDir = filepath.Clean(opts.PackageDir) + opts.SourceDir, err = filepath.Abs(opts.SourceDir) + if err != nil { + return opts, err + } + return opts, nil +} diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go new file mode 100644 index 0000000000..a862320360 --- /dev/null +++ b/cmd/arvados-package/fpm.go @@ -0,0 +1,113 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "git.arvados.org/arvados.git/lib/install" +) + +func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error { + var chownUid, chownGid int + if opts.PackageChown != "" { + _, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid) + if err != nil { + return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err) + } + } + + exitcode := install.Command.RunCommand("arvados-server install", []string{ + "-type", "package", + "-package-version", opts.PackageVersion, + "-source", opts.SourceDir, + }, stdin, stdout, stderr) + if exitcode != 0 { + return fmt.Errorf("arvados-server install failed: exit code %d", exitcode) + } + + cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm") + cmd.Stdout = stdout + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("gem install fpm: %w", err) + } + + if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil { + // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739 + cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb") + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("monkeypatch fpm: %w", err) + } + } + + // Remove unneeded files. This is much faster than "fpm + // --exclude X" because fpm copies everything into a staging + // area before looking at the --exclude args. + cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go") + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("rm -rf [...]: %w", err) + } + + format := "deb" // TODO: rpm + pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format) + + cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm", + "--package", pkgfile, + "--name", "arvados-server-easy", + "--version", opts.PackageVersion, + "--input-type", "dir", + "--output-type", format) + deps, err := install.ProductionDependencies() + if err != nil { + return err + } + for _, pkg := range deps { + cmd.Args = append(cmd.Args, "--depends", pkg) + } + cmd.Args = append(cmd.Args, + "--verbose", + "--deb-use-file-permissions", + "--rpm-use-file-permissions", + "/var/lib/arvados", + "/var/www/.gem", + "/var/www/.passenger", + "/var/www/.bundle", + ) + fmt.Fprintf(stderr, "... %s\n", cmd.Args) + cmd.Dir = opts.PackageDir + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("fpm: %w", err) + } + + if opts.PackageChown != "" { + err = os.Chown(pkgfile, chownUid, chownGid) + if err != nil { + return fmt.Errorf("chown %s: %w", pkgfile, err) + } + } + + cmd = exec.Command("ls", "-l", pkgfile) + cmd.Stdout = stdout + cmd.Stderr = stderr + _ = cmd.Run() + + return nil +} diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go new file mode 100644 index 0000000000..774771efb0 --- /dev/null +++ b/cmd/arvados-package/install.go @@ -0,0 +1,122 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error { + if opts.PackageVersion != "" { + return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir") + } + depsImageName := "arvados-package-deps-" + opts.TargetOS + depsCtrName := strings.Replace(depsImageName, ":", "-", -1) + + _, prog := filepath.Split(os.Args[0]) + tmpdir, err := ioutil.TempDir("", prog+".") + if err != nil { + return fmt.Errorf("TempDir: %w", err) + } + defer os.RemoveAll(tmpdir) + + sourcesFile := tmpdir + "/arvados-local.list" + err = ioutil.WriteFile(sourcesFile, []byte("deb [trusted=yes] file:///pkg ./\n"), 0644) + if err != nil { + return fmt.Errorf("Write %s: %w", sourcesFile, err) + } + + if exists, err := dockerImageExists(ctx, depsImageName); err != nil { + return err + } else if !exists || opts.RebuildImage { + err = dockerRm(ctx, depsCtrName) + if err != nil { + return err + } + defer dockerRm(ctx, depsCtrName) + cmd := exec.CommandContext(ctx, "docker", "run", + "--name", depsCtrName, + "--tmpfs", "/tmp:exec,mode=01777", + "-v", opts.PackageDir+":/pkg:ro", + "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro", + "--env", "DEBIAN_FRONTEND=noninteractive", + opts.TargetOS, + "bash", "-c", ` +set -e +apt-get update +apt-get install -y eatmydata +eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql +eatmydata apt-get remove -y arvados-server-easy +`) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker run: %w", err) + } + + cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker commit: %w", err) + } + } + + cmd := exec.CommandContext(ctx, "docker", "run", "--rm", + "--tmpfs", "/tmp:exec,mode=01777", + "-v", opts.PackageDir+":/pkg:ro", + "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro", + "--env", "DEBIAN_FRONTEND=noninteractive", + depsImageName, + "bash", "-c", ` +set -e +PATH="/var/lib/arvados/bin:$PATH" +apt-get update +eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy +apt-get -y autoremove +/etc/init.d/postgresql start +arvados-server init -cluster-id x1234 +exec arvados-server boot -listen-host 0.0.0.0 -shutdown +`) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("docker run: %w", err) + } + return nil +} + +func dockerImageExists(ctx context.Context, name string) (bool, error) { + cli, err := client.NewEnvClient() + if err != nil { + return false, err + } + imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true}) + if err != nil { + return false, err + } + for _, img := range imgs { + for _, tag := range img.RepoTags { + if tag == name { + return true, nil + } + } + } + return false, nil +} diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index ff99de75c4..d0aa9da94d 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -34,6 +34,7 @@ var ( "crunch-run": crunchrun.Command, "dispatch-cloud": dispatchcloud.Command, "install": install.Command, + "init": install.InitCommand, "recover-collection": recovercollection.Command, "ws": ws.Command, }) diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid new file mode 100644 index 0000000000..79e8505387 --- /dev/null +++ b/doc/install/automatic.html.textile.liquid @@ -0,0 +1,47 @@ +--- +layout: default +navsection: installguide +title: Automatic single-node install +... +{% comment %} +Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: CC-BY-SA-3.0 +{% endcomment %} + +{% include 'notebox_begin' %} +This installation method is not fully implemented, which is why this page is not yet listed in the "table of installation options":{{site.baseurl}}/install/index.html or in the left nav. +{% include 'notebox_end' %} + +This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running. + +A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps. + +h2. Prerequisites + +You will need: +* a server host running Debian 10 (buster). +* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@). +* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host). +* a Google account (use it in place of example@gmail.com.example in the instructions below). + +h2. Initialize the cluster + +
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# apt-get update
+# apt-get install arvados-server-easy
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example@gmail.com.example
+
+ +When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account. + +h2. Enable login + +Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option. + +After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect: + +
+# systemctl restart arvados-server
+
diff --git a/lib/boot/cert.go b/lib/boot/cert.go index f0797c2ac5..b2b8c896c2 100644 --- a/lib/boot/cert.go +++ b/lib/boot/cert.go @@ -9,6 +9,7 @@ import ( "fmt" "io/ioutil" "net" + "os" "path/filepath" ) @@ -27,23 +28,28 @@ func (createCertificates) String() string { func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error { var san string if net.ParseIP(super.ListenHost) != nil { - san = fmt.Sprintf("IP:%s", super.ListenHost) + san += fmt.Sprintf(",IP:%s", super.ListenHost) } else { - san = fmt.Sprintf("DNS:%s", super.ListenHost) + san += fmt.Sprintf(",DNS:%s", super.ListenHost) + } + if hostname, err := os.Hostname(); err != nil { + return fmt.Errorf("hostname: %w", err) + } else { + san += ",DNS:" + hostname } // Generate root key - err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096") + err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096") if err != nil { return err } // Generate a self-signed root certificate - err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost") if err != nil { return err } // Generate server key - err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "server.key", "2048") if err != nil { return err } @@ -52,17 +58,17 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe if err != nil { return err } - err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644) + err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644) if err != nil { return err } // Generate signing request - err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr") if err != nil { return err } // Sign certificate - err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256") if err != nil { return err } diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go index e0e2755220..373613bb35 100644 --- a/lib/boot/cmd.go +++ b/lib/boot/cmd.go @@ -108,6 +108,11 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std fmt.Fprintln(stdout, url) if *shutdown { super.Stop() + // Wait for children to exit. Don't report the + // ensuing "context cancelled" error, though: + // return nil to indicate successful startup. + _ = super.Wait() + return nil } } // Wait for signal/crash + orderly shutdown diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go index 0f105d6b6c..d14d051520 100644 --- a/lib/boot/nginx.go +++ b/lib/boot/nginx.go @@ -9,8 +9,10 @@ import ( "fmt" "io/ioutil" "net" + "net/url" "os" "os/exec" + "os/user" "path/filepath" "regexp" @@ -36,7 +38,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er "SSLKEY": filepath.Join(super.tempdir, "server.key"), "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"), "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"), - "TMPDIR": super.tempdir, + "TMPDIR": super.wwwtempdir, } for _, cmpt := range []struct { varname string @@ -53,7 +55,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er } { port, err := internalPort(cmpt.svc) if err != nil { - return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc) + return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc) } if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil { return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err) @@ -62,14 +64,20 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er port, err = externalPort(cmpt.svc) if err != nil { - return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc) + return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc) } if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil { return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err) } vars[cmpt.varname+"SSLPORT"] = port } - tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")) + var conftemplate string + if super.ClusterType == "production" { + conftemplate = "/var/lib/arvados/share/nginx.conf" + } else { + conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf") + } + tmpl, err := ioutil.ReadFile(conftemplate) if err != nil { return err } @@ -93,13 +101,32 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er } } } + + args := []string{ + "-g", "error_log stderr info;", + "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";", + "-c", conffile, + } + // Nginx ignores "user www-data;" when running as a non-root + // user... except that it causes it to ignore our other -g + // options. So we still have to decide for ourselves whether + // it's needed. + if u, err := user.Current(); err != nil { + return fmt.Errorf("user.Current(): %w", err) + } else if u.Uid == "0" { + args = append([]string{"-g", "user www-data;"}, args...) + } + super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - fail(super.RunProgram(ctx, ".", nil, nil, nginx, - "-g", "error_log stderr info;", - "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";", - "-c", conffile)) + fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...)) }() - return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host) + // Choose one of the ports where Nginx should listen, and wait + // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https" + testurl := url.URL(super.cluster.Services.Controller.ExternalURL) + if testurl.Port() == "" { + testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme) + } + return waitForConnect(ctx, testurl.Host) } diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go index 6a2c4b61f5..84f51d0a71 100644 --- a/lib/boot/passenger.go +++ b/lib/boot/passenger.go @@ -37,6 +37,10 @@ func (runner installPassenger) String() string { } func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error { + if super.ClusterType == "production" { + // passenger has already been installed via package + return nil + } err := super.wait(ctx, runner.depends...) if err != nil { return err @@ -46,32 +50,32 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super defer passengerInstallMutex.Unlock() var buf bytes.Buffer - err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler") + err = super.RunProgram(ctx, runner.src, runOptions{output: &buf}, "gem", "list", "--details", "bundler") if err != nil { return err } - for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} { + for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} { if !strings.Contains(buf.String(), "("+version+")") { - err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2") + err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2") if err != nil { return err } break } } - err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem")) + err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem")) if err != nil { return err } - err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support") + err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "build-native-support") if err != nil { return err } - err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime") + err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "install-standalone-runtime") if err != nil { return err } - err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install") + err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "validate-install") if err != nil && !strings.Contains(err.Error(), "exit status 2") { // Exit code 2 indicates there were warnings (like // "other passenger installations have been detected", @@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super } type runPassenger struct { - src string - svc arvados.Service - depends []supervisedTask + src string // path to app in source tree + varlibdir string // path to app (relative to /var/lib/arvados) in OS package + svc arvados.Service + depends []supervisedTask } func (runner runPassenger) String() string { @@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup if err != nil { return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc) } + var appdir string + if super.ClusterType == "production" { + appdir = "/var/lib/arvados/" + runner.varlibdir + } else { + appdir = runner.src + } loglevel := "4" if lvl, ok := map[string]string{ "debug": "5", @@ -116,13 +127,35 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec", + cmdline := []string{ + "bundle", "exec", "passenger", "start", "-p", port, - "--log-file", "/dev/stderr", "--log-level", loglevel, "--no-friendly-error-pages", - "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid")) + "--disable-anonymous-telemetry", + "--disable-security-update-check", + "--no-compile-runtime", + "--no-install-runtime", + "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"), + } + opts := runOptions{ + env: append([]string{ + "HOME=/var/www", + "TMPDIR=" + super.wwwtempdir, + }, railsEnv...), + } + if super.ClusterType == "production" { + opts.user = "www-data" + } else { + // This would be desirable when changing uid + // too, but it fails because /dev/stderr is a + // symlink to a pty owned by root: "nginx: + // [emerg] open() "/dev/stderr" failed (13: + // Permission denied)" + cmdline = append(cmdline, "--log-file", "/dev/stderr") + } + err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...) fail(err) }() return nil diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go index 7661c6b587..4ed7603d2a 100644 --- a/lib/boot/postgresql.go +++ b/lib/boot/postgresql.go @@ -36,15 +36,19 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso return err } + if super.ClusterType == "production" { + return nil + } + iamroot := false if u, err := user.Current(); err != nil { - return fmt.Errorf("user.Current(): %s", err) + return fmt.Errorf("user.Current(): %w", err) } else if u.Uid == "0" { iamroot = true } buf := bytes.NewBuffer(nil) - err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir") + err = super.RunProgram(ctx, super.tempdir, runOptions{output: buf}, "pg_config", "--bindir") if err != nil { return err } @@ -56,6 +60,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso return err } prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"} + opts := runOptions{} if iamroot { postgresUser, err := user.Lookup("postgres") if err != nil { @@ -81,25 +86,19 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso if err != nil { return err } - // We can't use "sudo -u" here because it creates an - // intermediate process that interferes with our - // ability to reliably kill postgres. The setuidgid - // program just calls exec without forking, so it - // doesn't have this problem. - args = append([]string{"postgres", prog}, args...) - prog = "setuidgid" - } - err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...) + opts.user = "postgres" + } + err = super.RunProgram(ctx, super.tempdir, opts, prog, args...) if err != nil { return err } - err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir) + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "cp", "server.crt", "server.key", datadir) if err != nil { return err } if iamroot { - err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key") if err != nil { return err } @@ -116,11 +115,11 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso "-k", datadir, // socket dir "-p", super.cluster.PostgreSQL.Connection["port"], } + opts := runOptions{} if iamroot { - args = append([]string{"postgres", prog}, args...) - prog = "setuidgid" + opts.user = "postgres" } - fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)) + fail(super.RunProgram(ctx, super.tempdir, opts, prog, args...)) }() for { diff --git a/lib/boot/seed.go b/lib/boot/seed.go index 2afccc45b6..bd1e942658 100644 --- a/lib/boot/seed.go +++ b/lib/boot/seed.go @@ -20,11 +20,14 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor if err != nil { return err } - err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup") + if super.ClusterType == "production" { + return nil + } + err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup") if err != nil { return err } - err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb") + err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb") if err != nil { return err } diff --git a/lib/boot/service.go b/lib/boot/service.go index 5afacfe716..090e852446 100644 --- a/lib/boot/service.go +++ b/lib/boot/service.go @@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string { } func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error { - binfile := filepath.Join(super.tempdir, "bin", "arvados-server") - err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version") + binfile := filepath.Join(super.bindir, "arvados-server") + err := super.RunProgram(ctx, super.bindir, runOptions{}, binfile, "-version") if err != nil { return err } @@ -46,7 +46,7 @@ func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile)) + fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile)) }() } return nil @@ -77,7 +77,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup return ctx.Err() } - err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version") + err = super.RunProgram(ctx, super.tempdir, runOptions{}, binfile, "-version") if err != nil { return err } @@ -93,7 +93,7 @@ func (runner runGoProgram) Run(ctx context.Context, fail func(error), super *Sup super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile)) + fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile)) }() } return nil diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index 752466c2a2..c75e7f146e 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -14,12 +14,14 @@ import ( "io" "io/ioutil" "net" + "net/url" "os" "os/exec" "os/signal" "os/user" "path/filepath" "reflect" + "strconv" "strings" "sync" "syscall" @@ -54,7 +56,9 @@ type Supervisor struct { tasksReady map[string]chan bool waitShutdown sync.WaitGroup + bindir string tempdir string + wwwtempdir string configfile string environ []string // for child processes } @@ -133,13 +137,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error { return err } - super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") - if err != nil { - return err - } - defer os.RemoveAll(super.tempdir) - if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil { - return err + // Choose bin and temp dirs: /var/lib/arvados/... in + // production, transient tempdir otherwise. + if super.ClusterType == "production" { + // These dirs have already been created by + // "arvados-server install" (or by extracting a + // package). + super.tempdir = "/var/lib/arvados/tmp" + super.wwwtempdir = "/var/lib/arvados/wwwtmp" + super.bindir = "/var/lib/arvados/bin" + } else { + super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") + if err != nil { + return err + } + defer os.RemoveAll(super.tempdir) + super.wwwtempdir = super.tempdir + super.bindir = filepath.Join(super.tempdir, "bin") + if err := os.Mkdir(super.bindir, 0755); err != nil { + return err + } } // Fill in any missing config keys, and write the resulting @@ -148,7 +165,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error { if err != nil { return err } - conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) + conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } @@ -168,7 +185,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error { super.setEnv("ARVADOS_CONFIG", super.configfile) super.setEnv("RAILS_ENV", super.ClusterType) super.setEnv("TMPDIR", super.tempdir) - super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:") + super.prependEnv("PATH", "/var/lib/arvados/bin:") + if super.ClusterType != "production" { + super.prependEnv("PATH", super.tempdir+"/bin:") + } super.cluster, err = cfg.GetCluster("") if err != nil { @@ -184,16 +204,18 @@ func (super *Supervisor) run(cfg *arvados.Config) error { "PID": os.Getpid(), }) - if super.SourceVersion == "" { + if super.SourceVersion == "" && super.ClusterType == "production" { + // don't need SourceVersion + } else if super.SourceVersion == "" { // Find current source tree version. var buf bytes.Buffer - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat") if err != nil { return err } dirty := buf.Len() > 0 buf.Reset() - err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H") + err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H") if err != nil { return err } @@ -226,15 +248,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error { runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}}, installPassenger{src: "services/api"}, - runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, + runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}}, installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup - runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, seedDatabase{}, } if super.ClusterType != "test" { tasks = append(tasks, - runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller}, - runGoProgram{src: "services/keep-balance"}, + runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, + runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance}, ) } super.tasksReady = map[string]chan bool{} @@ -384,9 +406,11 @@ func dedupEnv(in []string) []string { func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) { _, basename := filepath.Split(srcpath) - bindir := filepath.Join(super.tempdir, "bin") - binfile := filepath.Join(bindir, basename) - err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) + binfile := filepath.Join(super.bindir, basename) + if super.ClusterType == "production" { + return binfile, nil + } + err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) return binfile, err } @@ -403,10 +427,19 @@ func (super *Supervisor) setupRubyEnv() error { "GEM_PATH=", }) gem := "gem" - if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil { + if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" { gem = "/var/lib/arvados/bin/gem" } cmd := exec.Command(gem, "env", "gempath") + if super.ClusterType == "production" { + cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...) + path, err := exec.LookPath("sudo") + if err != nil { + return fmt.Errorf("LookPath(\"sudo\"): %w", err) + } + cmd.Path = path + } + cmd.Stderr = super.Stderr cmd.Env = super.environ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:... if err != nil || len(buf) == 0 { @@ -440,6 +473,12 @@ func (super *Supervisor) lookPath(prog string) string { return prog } +type runOptions struct { + output io.Writer // attach stdout + env []string // add/replace environment variables + user string // run as specified user +} + // RunProgram runs prog with args, using dir as working directory. If ctx is // cancelled while the child is running, RunProgram terminates the child, waits // for it to exit, then returns. @@ -448,22 +487,36 @@ func (super *Supervisor) lookPath(prog string) string { // // Child's stdout will be written to output if non-nil, otherwise the // boot command's stderr. -func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error { +func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error { cmdline := fmt.Sprintf("%s", append([]string{prog}, args...)) super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing") logprefix := prog - if logprefix == "setuidgid" && len(args) >= 2 { - logprefix = args[1] - } - logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") - if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" { - logprefix = args[1] - } else if logprefix == "arvados-server" && len(args) > 1 { - logprefix = args[0] - } - if !strings.HasPrefix(dir, "/") { - logprefix = dir + ": " + logprefix + { + innerargs := args + if logprefix == "sudo" { + for i := 0; i < len(args); i++ { + if args[i] == "-u" { + i++ + } else if args[i] == "-E" || strings.Contains(args[i], "=") { + } else { + logprefix = args[i] + innerargs = args[i+1:] + break + } + } + } + logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/") + logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") + if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" { + _, dirbase := filepath.Split(dir) + logprefix = innerargs[1] + "@" + dirbase + } else if logprefix == "arvados-server" && len(args) > 1 { + logprefix = args[0] + } + if !strings.HasPrefix(dir, "/") { + logprefix = dir + ": " + logprefix + } } cmd := exec.Command(super.lookPath(prog), args...) @@ -484,10 +537,10 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W }() copiers.Add(1) go func() { - if output == nil { + if opts.output == nil { io.Copy(logwriter, stdout) } else { - io.Copy(output, stdout) + io.Copy(opts.output, stdout) } copiers.Done() }() @@ -497,10 +550,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W } else { cmd.Dir = filepath.Join(super.SourcePath, dir) } - env = append([]string(nil), env...) + env := append([]string(nil), opts.env...) env = append(env, super.environ...) cmd.Env = dedupEnv(env) + if opts.user != "" { + // Note: We use this approach instead of "sudo" + // because in certain circumstances (we are pid 1 in a + // docker container, and our passenger child process + // changes to pgid 1) the intermediate sudo process + // notices we have the same pgid as our child and + // refuses to propagate signals from us to our child, + // so we can't signal/shutdown our passenger/rails + // apps. "chpst" or "setuidgid" would work, but these + // few lines avoid depending on runit/daemontools. + u, err := user.Lookup(opts.user) + if err != nil { + return fmt.Errorf("user.Lookup(%q): %w", opts.user, err) + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(uid), + Gid: uint32(gid), + }, + } + } + exited := false defer func() { exited = true }() go func() { @@ -607,27 +684,26 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { } } } - if cluster.SystemRootToken == "" { - cluster.SystemRootToken = randomHexString(64) - } - if cluster.ManagementToken == "" { - cluster.ManagementToken = randomHexString(64) - } - if cluster.Collections.BlobSigningKey == "" { - cluster.Collections.BlobSigningKey = randomHexString(64) - } - if cluster.Users.AnonymousUserToken == "" { - cluster.Users.AnonymousUserToken = randomHexString(64) - } - - if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" { - buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) - if err != nil { - return err - } - cluster.Containers.DispatchPrivateKey = string(buf) - } if super.ClusterType != "production" { + if cluster.SystemRootToken == "" { + cluster.SystemRootToken = randomHexString(64) + } + if cluster.ManagementToken == "" { + cluster.ManagementToken = randomHexString(64) + } + if cluster.Collections.BlobSigningKey == "" { + cluster.Collections.BlobSigningKey = randomHexString(64) + } + if cluster.Users.AnonymousUserToken == "" { + cluster.Users.AnonymousUserToken = randomHexString(64) + } + if cluster.Containers.DispatchPrivateKey == "" { + buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) + if err != nil { + return err + } + cluster.Containers.DispatchPrivateKey = string(buf) + } cluster.TLS.Insecure = true } if super.ClusterType == "test" { @@ -697,11 +773,10 @@ func internalPort(svc arvados.Service) (string, error) { return "", errors.New("internalPort() doesn't work with multiple InternalURLs") } for u := range svc.InternalURLs { - if _, p, err := net.SplitHostPort(u.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(u) + if p := u.Port(); p != "" { return p, nil - } else if u.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "ws" { return "443", nil } else { return "80", nil @@ -711,11 +786,10 @@ func internalPort(svc arvados.Service) (string, error) { } func externalPort(svc arvados.Service) (string, error) { - if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(svc.ExternalURL) + if p := u.Port(); p != "" { return p, nil - } else if svc.ExternalURL.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "wss" { return "443", nil } else { return "80", nil diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go index 7614a143ab..ae91a710e3 100644 --- a/lib/dispatchcloud/dispatcher.go +++ b/lib/dispatchcloud/dispatcher.go @@ -22,6 +22,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/health" "git.arvados.org/arvados.git/sdk/go/httpserver" "github.com/julienschmidt/httprouter" "github.com/prometheus/client_golang/prometheus" @@ -164,6 +165,11 @@ func (disp *dispatcher) initialize() { }) mux.Handler("GET", "/metrics", metricsH) mux.Handler("GET", "/metrics.json", metricsH) + mux.Handler("GET", "/_health/:check", &health.Handler{ + Token: disp.Cluster.ManagementToken, + Prefix: "/_health/", + Routes: health.Routes{"ping": disp.CheckHealth}, + }) disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux) } } diff --git a/lib/install/deps.go b/lib/install/deps.go index 342ef03a7f..504cdff8f2 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -14,6 +14,8 @@ import ( "io" "os" "os/exec" + "os/user" + "path/filepath" "strconv" "strings" "syscall" @@ -24,13 +26,18 @@ import ( "github.com/lib/pq" ) -var Command cmd.Handler = installCommand{} +var Command cmd.Handler = &installCommand{} const devtestDatabasePassword = "insecure_arvados_test" -type installCommand struct{} +type installCommand struct { + ClusterType string + SourcePath string + PackageVersion string + EatMyData bool +} -func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { logger := ctxlog.New(stderr, "text", "info") ctx := ctxlog.Context(context.Background(), logger) ctx, cancel := context.WithCancel(ctx) @@ -46,7 +53,10 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st flags := flag.NewFlagSet(prog, flag.ContinueOnError) flags.SetOutput(stderr) versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") - clusterType := flags.String("type", "production", "cluster `type`: development, test, or production") + flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package") + flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)") + flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files") + flags.BoolVar(&inst.EatMyData, "eatmydata", false, "use eatmydata to speed up install") err = flags.Parse(args) if err == flag.ErrHelp { err = nil @@ -55,18 +65,23 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st return 2 } else if *versionFlag { return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr) + } else if len(flags.Args()) > 0 { + err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) + return 2 } - var dev, test, prod bool - switch *clusterType { + var dev, test, prod, pkg bool + switch inst.ClusterType { case "development": dev = true case "test": test = true case "production": prod = true + case "package": + pkg = true default: - err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType) + err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType) return 2 } @@ -96,34 +111,45 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st } } - if dev || test { - debs := []string{ + if inst.EatMyData { + cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends", "eatmydata") + cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive") + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + + pkgs := prodpkgs(osv) + + if pkg { + pkgs = append(pkgs, + "dpkg-dev", + "eatmydata", // install it for later steps, even if we're not using it now + "rsync", + ) + } + + if dev || test || pkg { + pkgs = append(pkgs, + "automake", "bison", "bsdmainutils", "build-essential", - "ca-certificates", "cadaver", "curl", "cython3", - "daemontools", // lib/boot uses setuidgid to drop privileges when running as root "default-jdk-headless", "default-jre-headless", - "fuse", "gettext", - "git", - "gitolite3", - "graphviz", - "haveged", "iceweasel", "libattr1-dev", "libcrypt-ssleay-perl", - "libcrypt-ssleay-perl", - "libcurl3-gnutls", - "libcurl4-openssl-dev", "libfuse-dev", "libgnutls28-dev", "libjson-perl", - "libjson-perl", "libpam-dev", "libpcre3-dev", "libpq-dev", @@ -131,11 +157,11 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st "libssl-dev", "libwww-perl", "libxml2-dev", - "libxslt1.1", + "libxslt1-dev", "linkchecker", "lsof", + "make", "net-tools", - "nginx", "pandoc", "perl-modules", "pkg-config", @@ -154,16 +180,19 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st "sudo", "wget", "xvfb", - "zlib1g-dev", - } + ) switch { case osv.Debian && osv.Major >= 10: - debs = append(debs, "libcurl4") + pkgs = append(pkgs, "libcurl4") default: - debs = append(debs, "libcurl3") + pkgs = append(pkgs, "libcurl3") } - cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends") - cmd.Args = append(cmd.Args, debs...) + cmd := exec.CommandContext(ctx, "apt-get") + if inst.EatMyData { + cmd = exec.CommandContext(ctx, "eatmydata", "apt-get") + } + cmd.Args = append(cmd.Args, "install", "--yes", "--no-install-recommends") + cmd.Args = append(cmd.Args, pkgs...) cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive") cmd.Stdout = stdout cmd.Stderr = stderr @@ -174,21 +203,36 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st } os.Mkdir("/var/lib/arvados", 0755) + os.Mkdir("/var/lib/arvados/tmp", 0700) + if prod || pkg { + os.Mkdir("/var/lib/arvados/wwwtmp", 0700) + u, er := user.Lookup("www-data") + if er != nil { + err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er) + return 1 + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid) + if err != nil { + return 1 + } + } rubyversion := "2.5.7" if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) { logger.Print("ruby " + rubyversion + " already installed") } else { - err = runBash(` -mkdir -p /var/lib/arvados/tmp -tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+` -trap "rm -r ${tmp}" ERR -wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf - -cd ${tmp} -./configure --disable-install-doc --prefix /var/lib/arvados -make -j4 + err = inst.runBash(` +tmp="$(mktemp -d)" +trap 'rm -r "${tmp}"' ERR EXIT +wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C "${tmp}" -xzf - +cd "${tmp}/ruby-`+rubyversion+`" +./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados +make -j8 make install -/var/lib/arvados/bin/gem install bundler -rm -r ${tmp} +/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc +# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html +/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc `, stdout, stderr) if err != nil { return 1 @@ -200,7 +244,7 @@ rm -r ${tmp} if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) { logger.Print("go " + goversion + " already installed") } else { - err = runBash(` + err = inst.runBash(` cd /tmp wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf - ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/ @@ -209,12 +253,14 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/ return 1 } } + } + if !prod && !pkg { pjsversion := "1.9.8" if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" { logger.Print("phantomjs " + pjsversion + " already installed") } else { - err = runBash(` + err = inst.runBash(` PJS=phantomjs-`+pjsversion+`-linux-x86_64 wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf - ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/ @@ -228,7 +274,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/ if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") { logger.Print("geckodriver " + geckoversion + " already installed") } else { - err = runBash(` + err = inst.runBash(` GD=v`+geckoversion+` wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/ @@ -242,7 +288,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/ if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" { logger.Print("nodejs " + nodejsversion + " already installed") } else { - err = runBash(` + err = inst.runBash(` NJS=`+nodejsversion+` wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ @@ -256,9 +302,8 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") { logger.Print("gradle " + gradleversion + " already installed") } else { - err = runBash(` + err = inst.runBash(` G=`+gradleversion+` -mkdir -p /var/lib/arvados/tmp zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip trap "rm ${zip}" ERR wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip @@ -278,7 +323,7 @@ rm ${zip} if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) { logger.Print("locale " + wantlocale + " already installed") } else { - err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr) + err = inst.runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr) if err != nil { return 1 } @@ -357,7 +402,7 @@ rm ${zip} // locales. Otherwise, it might need a // restart, so we attempt to restart it with // systemd. - if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil { + if err = inst.runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil { logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted") } else if err = waitPostgreSQLReady(); err != nil { return 1 @@ -392,12 +437,104 @@ rm ${zip} } } + if prod || pkg { + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ + for dstdir, srcdir := range map[string]string{ + "railsapi": "services/api", + "workbench1": "apps/workbench", + } { + fmt.Fprintf(stderr, "building %s...\n", srcdir) + cmd := exec.Command("rsync", + "-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded", + "--exclude", "/coverage", + "--exclude", "/log", + "--exclude", "/tmp", + "--exclude", "/vendor", + "./", "/var/lib/arvados/"+dstdir+"/") + cmd.Dir = filepath.Join(inst.SourcePath, srcdir) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + for _, cmdline := range [][]string{ + {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"}, + {"touch", "log/production.log"}, + {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"}, + } { + cmd = exec.Command(cmdline[0], cmdline[1:]...) + cmd.Dir = "/var/lib/arvados/" + dstdir + cmd.Stdout = stdout + cmd.Stderr = stderr + fmt.Fprintf(stderr, "... %s\n", cmd.Args) + err = cmd.Run() + if err != nil { + return 1 + } + } + cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install") + cmd.Dir = "/var/lib/arvados/" + dstdir + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil && !strings.Contains(err.Error(), "exit status 2") { + // Exit code 2 indicates there were warnings (like + // "other passenger installations have been detected", + // which we can't expect to avoid) but no errors. + // Other non-zero exit codes (1, 9) indicate errors. + return 1 + } + } + + // Install Go programs to /var/lib/arvados/bin/ + for _, srcdir := range []string{ + "cmd/arvados-client", + "cmd/arvados-server", + "services/arv-git-httpd", + "services/crunch-dispatch-local", + "services/crunch-dispatch-slurm", + "services/health", + "services/keep-balance", + "services/keep-web", + "services/keepproxy", + "services/keepstore", + "services/ws", + } { + fmt.Fprintf(stderr, "building %s...\n", srcdir) + cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin") + cmd.Dir = filepath.Join(inst.SourcePath, srcdir) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + + // Copy assets from source tree to /var/lib/arvados/share + cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + return 0 } type osversion struct { Debian bool Ubuntu bool + Centos bool Major int } @@ -435,6 +572,8 @@ func identifyOS() (osversion, error) { osv.Ubuntu = true case "debian": osv.Debian = true + case "centos": + osv.Centos = true default: return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"]) } @@ -462,10 +601,64 @@ func waitPostgreSQLReady() error { } } -func runBash(script string, stdout, stderr io.Writer) error { +func (inst *installCommand) runBash(script string, stdout, stderr io.Writer) error { cmd := exec.Command("bash", "-") + if inst.EatMyData { + cmd = exec.Command("eatmydata", "bash", "-") + } cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script) cmd.Stdout = stdout cmd.Stderr = stderr return cmd.Run() } + +func prodpkgs(osv osversion) []string { + pkgs := []string{ + "ca-certificates", + "curl", + "fuse", + "git", + "gitolite3", + "graphviz", + "haveged", + "libcurl3-gnutls", + "libxslt1.1", + "nginx", + "python", + "sudo", + } + if osv.Debian || osv.Ubuntu { + if osv.Debian && osv.Major == 8 { + pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl + } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 { + pkgs = append(pkgs, "python3-distutils") // sdk/cwl + } + return append(pkgs, + "g++", + "libcurl4-openssl-dev", // services/api + "libpq-dev", + "libpython2.7", // services/fuse + "mime-support", // keep-web + "zlib1g-dev", // services/api + ) + } else if osv.Centos { + return append(pkgs, + "fuse-libs", // services/fuse + "gcc", + "gcc-c++", + "libcurl-devel", // services/api + "mailcap", // keep-web + "postgresql-devel", // services/api + ) + } else { + panic("os version not supported") + } +} + +func ProductionDependencies() ([]string, error) { + osv, err := identifyOS() + if err != nil { + return nil, err + } + return prodpkgs(osv), nil +} diff --git a/lib/install/init.go b/lib/install/init.go new file mode 100644 index 0000000000..d4d9ef47ef --- /dev/null +++ b/lib/install/init.go @@ -0,0 +1,269 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package install + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "flag" + "fmt" + "io" + "os" + "os/exec" + "os/user" + "regexp" + "strconv" + "text/template" + + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/lib/pq" +) + +var InitCommand cmd.Handler = &initCommand{} + +type initCommand struct { + ClusterID string + Domain string + PostgreSQLPassword string +} + +func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + logger := ctxlog.New(stderr, "text", "info") + ctx := ctxlog.Context(context.Background(), logger) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var err error + defer func() { + if err != nil { + logger.WithError(err).Info("exiting") + } + }() + + hostname, err := os.Hostname() + if err != nil { + err = fmt.Errorf("Hostname(): %w", err) + return 1 + } + + flags := flag.NewFlagSet(prog, flag.ContinueOnError) + flags.SetOutput(stderr) + versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") + flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster") + flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com") + err = flags.Parse(args) + if err == flag.ErrHelp { + err = nil + return 0 + } else if err != nil { + return 2 + } else if *versionFlag { + return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr) + } else if len(flags.Args()) > 0 { + err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) + return 2 + } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) { + err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID) + return 1 + } + + wwwuser, err := user.Lookup("www-data") + if err != nil { + err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err) + return 1 + } + wwwgid, err := strconv.Atoi(wwwuser.Gid) + if err != nil { + return 1 + } + initcmd.PostgreSQLPassword = initcmd.RandomHex(32) + + err = os.Mkdir("/var/lib/arvados/keep", 0600) + if err != nil && !os.IsExist(err) { + err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err) + return 1 + } + fmt.Fprintln(stderr, "created /var/lib/arvados/keep") + + err = os.Mkdir("/etc/arvados", 0750) + if err != nil && !os.IsExist(err) { + err = fmt.Errorf("mkdir /etc/arvados: %w", err) + return 1 + } + err = os.Chown("/etc/arvados", 0, wwwgid) + f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + if err != nil { + err = fmt.Errorf("open /etc/arvados/config.yml: %w", err) + return 1 + } + tmpl, err := template.New("config").Parse(`Clusters: + {{.ClusterID}}: + Services: + Controller: + InternalURLs: + "http://0.0.0.0:8003/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }} + RailsAPI: + InternalURLs: + "http://0.0.0.0:8004/": {} + Websocket: + InternalURLs: + "http://0.0.0.0:8005/": {} + ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }} + Keepbalance: + InternalURLs: + "http://0.0.0.0:9005/": {} + GitHTTP: + InternalURLs: + "http://0.0.0.0:9001/": {} + ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }} + DispatchCloud: + InternalURLs: + "http://0.0.0.0:9006/": {} + Keepproxy: + InternalURLs: + "http://0.0.0.0:25108/": {} + ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }} + WebDAV: + InternalURLs: + "http://0.0.0.0:9002/": {} + ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }} + WebDAVDownload: + InternalURLs: + "http://0.0.0.0:8004/": {} + ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }} + Keepstore: + InternalURLs: + "http://0.0.0.0:25107/": {} + Composer: + ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }} + Workbench1: + InternalURLs: + "http://0.0.0.0:8001/": {} + ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }} + #Workbench2: + # InternalURLs: + # "http://0.0.0.0:8002/": {} + # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }} + Health: + InternalURLs: + "http://0.0.0.0:9007/": {} + API: + RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}} + Collections: + BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}} + Containers: + DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}} + ManagementToken: {{printf "%q" ( .RandomHex 50 )}} + PostgreSQL: + Connection: + dbname: arvados_production + host: localhost + user: arvados + password: {{printf "%q" .PostgreSQLPassword}} + SystemRootToken: {{printf "%q" ( .RandomHex 50 )}} + TLS: + Insecure: true + Volumes: + {{.ClusterID}}-nyw5e-000000000000000: + Driver: Directory + DriverParameters: + Root: /var/lib/arvados/keep + Replication: 2 + Workbench: + SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}} +`) + if err != nil { + return 1 + } + err = tmpl.Execute(f, initcmd) + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err) + return 1 + } + err = f.Close() + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err) + return 1 + } + fmt.Fprintln(stderr, "created /etc/arvados/config.yml") + + ldr := config.NewLoader(nil, logger) + ldr.SkipLegacy = true + cfg, err := ldr.Load() + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: %w", err) + return 1 + } + cluster, err := cfg.GetCluster("") + if err != nil { + return 1 + } + + err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr) + if err != nil { + return 1 + } + + cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup") + cmd.Dir = "/var/lib/arvados/railsapi" + cmd.Stdout = stderr + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + err = fmt.Errorf("rake db:setup: %w", err) + return 1 + } + fmt.Fprintln(stderr, "initialized database") + + return 0 +} + +func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) { + privkey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + err = privkey.Validate() + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privkey), + })), nil +} + +func (initcmd *initCommand) RandomHex(chars int) string { + b := make([]byte, chars/2) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", b) +} + +func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error { + for _, sql := range []string{ + `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]), + `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`, + `CREATE EXTENSION IF NOT EXISTS pg_trgm`, + } { + cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql) + cmd.Stdout = stderr + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("error setting up arvados user/database: %w", err) + } + } + return nil +} diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go index 794adabdd3..a666ef8ec0 100644 --- a/sdk/go/health/aggregator.go +++ b/sdk/go/health/aggregator.go @@ -193,6 +193,9 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) { } req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken) + // Avoid workbench1's redirect-http-to-https feature + req.Header.Set("X-Forwarded-Proto", "https") + ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout)) defer cancel() req = req.WithContext(ctx) diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf index 85b4f5b37b..a4336049f2 100644 --- a/sdk/python/tests/nginx.conf +++ b/sdk/python/tests/nginx.conf @@ -11,17 +11,33 @@ http { '[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port ' '"$http_referer" "$http_user_agent"'; access_log "{{ACCESSLOG}}" customlog; - client_body_temp_path "{{TMPDIR}}"; - proxy_temp_path "{{TMPDIR}}"; - fastcgi_temp_path "{{TMPDIR}}"; - uwsgi_temp_path "{{TMPDIR}}"; - scgi_temp_path "{{TMPDIR}}"; + client_body_temp_path "{{TMPDIR}}/nginx"; + proxy_temp_path "{{TMPDIR}}/nginx"; + fastcgi_temp_path "{{TMPDIR}}/nginx"; + uwsgi_temp_path "{{TMPDIR}}/nginx"; + scgi_temp_path "{{TMPDIR}}/nginx"; + upstream controller { + server {{LISTENHOST}}:{{CONTROLLERPORT}}; + } + server { + listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl; + server_name controller ~.*; + ssl_certificate "{{SSLCERT}}"; + ssl_certificate_key "{{SSLKEY}}"; + location / { + proxy_pass http://controller; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_redirect off; + } + } upstream arv-git-http { server {{LISTENHOST}}:{{GITPORT}}; } server { - listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server; - server_name arv-git-http; + listen {{LISTENHOST}}:{{GITSSLPORT}} ssl; + server_name arv-git-http git.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -36,8 +52,8 @@ http { server {{LISTENHOST}}:{{KEEPPROXYPORT}}; } server { - listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server; - server_name keepproxy; + listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl; + server_name keepproxy keep.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -55,8 +71,8 @@ http { server {{LISTENHOST}}:{{KEEPWEBPORT}}; } server { - listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server; - server_name keep-web; + listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl; + server_name keep-web collections.* ~\.collections\.; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -75,8 +91,8 @@ http { server {{LISTENHOST}}:{{HEALTHPORT}}; } server { - listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server; - server_name health; + listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl; + server_name health health.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -91,8 +107,8 @@ http { } } server { - listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server; - server_name keep-web-dl ~.*; + listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl; + server_name keep-web-dl download.* ~.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -111,8 +127,8 @@ http { server {{LISTENHOST}}:{{WSPORT}}; } server { - listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server; - server_name websocket; + listen {{LISTENHOST}}:{{WSSSLPORT}} ssl; + server_name websocket ws.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -129,8 +145,8 @@ http { server {{LISTENHOST}}:{{WORKBENCH1PORT}}; } server { - listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server; - server_name workbench1; + listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl; + server_name workbench1 workbench.*; ssl_certificate "{{SSLCERT}}"; ssl_certificate_key "{{SSLKEY}}"; location / { @@ -141,20 +157,4 @@ http { proxy_redirect off; } } - upstream controller { - server {{LISTENHOST}}:{{CONTROLLERPORT}}; - } - server { - listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server; - server_name controller; - ssl_certificate "{{SSLCERT}}"; - ssl_certificate_key "{{SSLKEY}}"; - location / { - proxy_pass http://controller; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_redirect off; - } - } } diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go index 65bd8d4cf0..8b4ee84c71 100644 --- a/services/keep-balance/main.go +++ b/services/keep-balance/main.go @@ -9,13 +9,13 @@ import ( "flag" "fmt" "io" - "net/http" "os" "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/lib/service" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/health" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -83,7 +83,6 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W } srv := &Server{ - Handler: http.NotFoundHandler(), Cluster: cluster, ArvClient: ac, RunOptions: options, @@ -91,6 +90,11 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W Logger: options.Logger, Dumper: options.Dumper, } + srv.Handler = &health.Handler{ + Token: cluster.ManagementToken, + Prefix: "/_health/", + Routes: health.Routes{"ping": srv.CheckHealth}, + } go srv.run() return srv