(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- versions=(1.11.0 1.17.3 2.0.2)
+ versions=(1.16.6 1.17.3 2.0.2)
for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "os/user"
+ "path/filepath"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion == "" {
+ var buf bytes.Buffer
+ cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty")
+ cmd.Stdout = &buf
+ cmd.Stderr = stderr
+ cmd.Dir = opts.SourceDir
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("git describe: %w", err)
+ }
+ opts.PackageVersion = strings.TrimSpace(buf.String())
+ ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion)
+ }
+
+ if opts.PackageChown == "" {
+ whoami, err := user.Current()
+ if err != nil {
+ return fmt.Errorf("user.Current: %w", err)
+ }
+ opts.PackageChown = whoami.Uid + ":" + whoami.Gid
+ }
+
+ // Build in a tempdir, then move to the desired destination
+ // dir. Otherwise, errors might cause us to leave a mess:
+ // truncated files, files owned by root, etc.
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpdir)
+
+ selfbin, err := os.Readlink("/proc/self/exe")
+ if err != nil {
+ return fmt.Errorf("readlink /proc/self/exe: %w", err)
+ }
+ buildImageName := "arvados-package-build-" + opts.TargetOS
+ packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb"
+
+ if ok, err := dockerImageExists(ctx, buildImageName); err != nil {
+ return err
+ } else if !ok || opts.RebuildImage {
+ buildCtrName := strings.Replace(buildImageName, ":", "-", -1)
+ err = dockerRm(ctx, buildCtrName)
+ if err != nil {
+ return err
+ }
+
+ defer dockerRm(ctx, buildCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", buildCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ opts.TargetOS,
+ "/arvados-package", "_install",
+ "-eatmydata",
+ "-type", "package",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+
+ ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName)
+ }
+
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", tmpdir+":/pkg",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ buildImageName,
+ "eatmydata", "/arvados-package", "_fpm",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ "-package-dir", "/pkg",
+ "-package-chown", opts.PackageChown,
+ "-package-maintainer", opts.Maintainer,
+ "-package-vendor", opts.Vendor,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func dockerRm(ctx context.Context, name string) error {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return err
+ }
+ ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1})
+ if err != nil {
+ return err
+ }
+ for _, ctr := range ctrs {
+ for _, ctrname := range ctr.Names {
+ if ctrname == "/"+name {
+ err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{})
+ if err != nil {
+ return fmt.Errorf("error removing container %s: %w", ctr.ID, err)
+ }
+ break
+ }
+ }
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "os"
+ "os/exec"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+ os.Args = append(os.Args, "-test.timeout=30m") // kludge
+}
+
+type BuildSuite struct{}
+
+var _ = check.Suite(&BuildSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildSuite) TestBuildAndInstall(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+ tmpdir := c.MkDir()
+ defer os.RemoveAll(tmpdir)
+ err := os.Chmod(tmpdir, 0755)
+ c.Assert(err, check.IsNil)
+
+ cmd := exec.Command("go", "run", ".",
+ "build",
+ "-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
+ "-source", "../..",
+ )
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Check(err, check.IsNil)
+
+ fi, err := os.Stat(tmpdir + "/arvados-server-easy_1.2.3~rc4_amd64.deb")
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fi)
+
+ buf, _ := exec.Command("ls", "-l", tmpdir).CombinedOutput()
+ c.Logf("%s", buf)
+
+ cmd = exec.Command("go", "run", ".",
+ "testinstall",
+ "-package-dir", tmpdir,
+ "-package-version", "1.2.3~rc4",
+ )
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Check(err, check.IsNil)
+
+ err = os.RemoveAll(tmpdir)
+ c.Check(err, check.IsNil)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+var (
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "version": cmd.Version,
+ "-version": cmd.Version,
+ "--version": cmd.Version,
+
+ "build": cmdFunc(build),
+ "testinstall": cmdFunc(testinstall),
+ "_fpm": cmdFunc(fpm), // internal use
+ "_install": install.Command, // internal use
+ })
+)
+
+func main() {
+ if len(os.Args) < 2 || strings.HasPrefix(os.Args[1], "-") {
+ parseFlags([]string{"-help"})
+ os.Exit(2)
+ }
+ os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error
+
+func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ opts, err := parseFlags(args)
+ if err != nil {
+ logger.WithError(err).Error("error parsing command line flags")
+ return 1
+ }
+ err = cf(ctx, opts, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type opts struct {
+ PackageVersion string
+ PackageDir string
+ PackageChown string
+ RebuildImage bool
+ SourceDir string
+ TargetOS string
+ Maintainer string
+ Vendor string
+}
+
+func parseFlags(args []string) (opts, error) {
+ opts := opts{
+ SourceDir: ".",
+ TargetOS: "debian:10",
+ Maintainer: "Arvados Package Maintainers <packaging@arvados.org>",
+ Vendor: "The Arvados Project",
+ }
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
+ flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location")
+ flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)")
+ flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
+ flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
+ flags.StringVar(&opts.Maintainer, "package-maintainer", opts.Maintainer, "maintainer to be listed in package metadata")
+ flags.StringVar(&opts.Vendor, "package-vendor", opts.Vendor, "vendor to be listed in package metadata")
+ flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+ flags.Usage = func() {
+ fmt.Fprint(flags.Output(), `Usage: arvados-package <subcommand> [options]
+
+Subcommands:
+ build
+ use a docker container to build a package from a checked
+ out version of the arvados source tree
+ testinstall
+ use a docker container to install a package and confirm
+ the resulting installation is functional
+ version
+ show program version
+
+Internally used subcommands:
+ _fpm
+ build a package
+ _install
+ equivalent to "arvados-server install"
+
+Automation/integration notes:
+ The first time a given machine runs "build" or "testinstall" (and
+ any time the -rebuild-image is used), new docker images are built,
+ which is quite slow. If you use on-demand VMs to run automated builds,
+ run "build" and "testinstall" once when setting up your initial VM
+ image, and be prepared to rebuild that VM image when package-building
+ slows down (this will happen when new dependencies are introduced).
+
+ The "build" subcommand, if successful, also runs
+ dpkg-scanpackages to create/replace Packages.gz in the package
+ dir. This enables the "testinstall" subcommand to list the
+ package dir as a source in /etc/apt/sources.*.
+
+Options:
+`)
+ flags.PrintDefaults()
+ }
+ err := flags.Parse(args)
+ if err != nil {
+ return opts, err
+ }
+ if len(flags.Args()) > 0 {
+ return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if opts.SourceDir == "" {
+ d, err := os.Getwd()
+ if err != nil {
+ return opts, fmt.Errorf("Getwd: %w", err)
+ }
+ opts.SourceDir = d
+ }
+ opts.PackageDir = filepath.Clean(opts.PackageDir)
+ opts.SourceDir, err = filepath.Abs(opts.SourceDir)
+ if err != nil {
+ return opts, err
+ }
+ return opts, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "git.arvados.org/arvados.git/lib/install"
+)
+
+func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ var chownUid, chownGid int
+ if opts.PackageChown != "" {
+ _, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid)
+ if err != nil {
+ return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err)
+ }
+ }
+
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", opts.PackageVersion,
+ "-source", opts.SourceDir,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ }
+
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
+ cmd = exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
+ cmd.Stderr = stderr
+ buf, err := cmd.Output() // /root/.gem/ruby/2.7.0:...
+ if err != nil || len(buf) == 0 {
+ return fmt.Errorf("gem env gempath: %w", err)
+ }
+ gempath := string(bytes.TrimRight(bytes.Split(buf, []byte{':'})[0], "\n"))
+
+ if _, err := os.Stat(gempath + "/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+ // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+ cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, gempath+"/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("monkeypatch fpm: %w", err)
+ }
+ }
+
+ // Remove unneeded files. This is much faster than "fpm
+ // --exclude X" because fpm copies everything into a staging
+ // area before looking at the --exclude args.
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("rm -rf [...]: %w", err)
+ }
+
+ format := "deb" // TODO: rpm
+ pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format)
+
+ cmd = exec.Command(gempath+"/bin/fpm",
+ "--package", pkgfile,
+ "--name", "arvados-server-easy",
+ "--version", opts.PackageVersion,
+ "--url", "https://arvados.org",
+ "--maintainer", opts.Maintainer,
+ "--vendor", opts.Vendor,
+ "--license", "GNU Affero General Public License, version 3.0",
+ "--description", "platform for managing, processing, and sharing genomic and other large scientific and biomedical data",
+ "--input-type", "dir",
+ "--output-type", format)
+ deps, err := install.ProductionDependencies()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range deps {
+ cmd.Args = append(cmd.Args, "--depends", pkg)
+ }
+ cmd.Args = append(cmd.Args,
+ "--verbose",
+ "--deb-use-file-permissions",
+ "--rpm-use-file-permissions",
+ "/var/lib/arvados",
+ "/var/www/.gem",
+ "/var/www/.passenger",
+ "/var/www/.bundle",
+ )
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+ cmd.Dir = opts.PackageDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("fpm: %w", err)
+ }
+
+ if opts.PackageChown != "" {
+ err = os.Chown(pkgfile, chownUid, chownGid)
+ if err != nil {
+ return fmt.Errorf("chown %s: %w", pkgfile, err)
+ }
+ }
+
+ cmd = exec.Command("ls", "-l", pkgfile)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ _ = cmd.Run()
+
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ depsImageName := "arvados-package-deps-" + opts.TargetOS
+ depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir("", prog+".")
+ if err != nil {
+ return fmt.Errorf("TempDir: %w", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ if exists, err := dockerImageExists(ctx, depsImageName); err != nil {
+ return err
+ } else if !exists || opts.RebuildImage {
+ err = dockerRm(ctx, depsCtrName)
+ if err != nil {
+ return err
+ }
+ defer dockerRm(ctx, depsCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", depsCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ opts.TargetOS,
+ "bash", "-c", `
+set -e -o pipefail
+apt-get update
+apt-get install -y --no-install-recommends dpkg-dev eatmydata
+
+mkdir /tmp/pkg
+ln -s /pkg/*.deb /tmp/pkg/
+(cd /tmp/pkg; dpkg-scanpackages --multiversion . | gzip > Packages.gz)
+echo >/etc/apt/sources.list.d/arvados-local.list "deb [trusted=yes] file:///tmp/pkg ./"
+apt-get update
+
+eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql
+eatmydata apt-get remove -y dpkg-dev
+SUDO_FORCE_REMOVE=yes apt-get autoremove -y
+eatmydata apt-get remove -y arvados-server-easy
+rm /etc/apt/sources.list.d/arvados-local.list
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+ }
+
+ versionsuffix := ""
+ if opts.PackageVersion != "" {
+ versionsuffix = "=" + opts.PackageVersion
+ }
+ cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ depsImageName,
+ "bash", "-c", `
+set -e -o pipefail
+PATH="/var/lib/arvados/bin:$PATH"
+apt-get update
+apt-get install -y --no-install-recommends dpkg-dev
+mkdir /tmp/pkg
+ln -s /pkg/*.deb /tmp/pkg/
+(cd /tmp/pkg; dpkg-scanpackages --multiversion . | gzip > Packages.gz)
+apt-get remove -y dpkg-dev
+echo
+
+echo >/etc/apt/sources.list.d/arvados-local.list "deb [trusted=yes] file:///tmp/pkg ./"
+apt-get update
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy`+versionsuffix+`
+SUDO_FORCE_REMOVE=yes apt-get autoremove -y
+
+/etc/init.d/postgresql start
+arvados-server init -cluster-id x1234
+exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+ return nil
+}
+
+func dockerImageExists(ctx context.Context, name string) (bool, error) {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return false, err
+ }
+ imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+ if err != nil {
+ return false, err
+ }
+ for _, img := range imgs {
+ for _, tag := range img.RepoTags {
+ if tag == name {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "init": install.InitCommand,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
})
task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
vars = ['baseurl', 'arvados_cluster_uuid', 'arvados_api_host', 'arvados_workbench_host']
+ if ! ENV.key?('baseurl') || ENV['baseurl'] == ""
+ if !ENV.key?('WORKSPACE') || ENV['WORKSPACE'] == ""
+ puts "The `baseurl` variable was not specified and the `WORKSPACE` environment variable is not set. Defaulting `baseurl` to file://#{pwd}/.site"
+ ENV['baseurl'] = "file://#{pwd}/.site/"
+ else
+ puts "The `baseurl` variable was not specified, defaulting to a value derived from the `WORKSPACE` environment variable"
+ ENV['baseurl'] = "file://#{ENV['WORKSPACE']}/doc/.site/"
+ end
+ end
vars.each do |v|
if ENV[v]
website.config.h[v] = ENV[v]
- install/install-shell-server.html.textile.liquid
- install/install-webshell.html.textile.liquid
- install/install-arv-git-httpd.html.textile.liquid
- - Containers API:
+ - Containers API (cloud):
- install/install-jobs-image.html.textile.liquid
- install/crunch2-cloud/install-compute-node.html.textile.liquid
- install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid
- - install/crunch2-slurm/install-compute-node.html.textile.liquid
+ - Containers API (slurm):
- install/crunch2-slurm/install-dispatch.html.textile.liquid
+ - install/crunch2-slurm/configure-slurm.html.textile.liquid
+ - install/crunch2-slurm/install-compute-node.html.textile.liquid
- install/crunch2-slurm/install-test.html.textile.liquid
- External dependencies:
- install/install-postgresql.html.textile.liquid
+++ /dev/null
-user-management.html.textile.liquid
\ No newline at end of file
Similar settings should be added to @clsr2@ & @clsr3@ hosts, so that all clusters in the federation can talk to each other.
-The @ActivateUsers@ setting indicates whether users from a given cluster are automatically activated or they require manual activation. User activation is covered in more detail in the "user activation section":{{site.baseurl}}/admin/activation.html. In the current example, users from @clsr2@ would be automatically, activated, but users from @clsr3@ would require an admin to activate the account.
+The @ActivateUsers@ setting indicates whether users from a given cluster are automatically activated or they require manual activation. User activation is covered in more detail in the "user activation section":{{site.baseurl}}/admin/user-management.html. In the current example, users from @clsr2@ would be automatically, activated, but users from @clsr3@ would require an admin to activate the account.
+
+Note: The @Proxy:@ variable is intended for future use, and should always be set to @true@.
h2(#LoginCluster). User management
+++ /dev/null
-logging.html.textile.liquid
\ No newline at end of file
In these examples, @zzzzz-tpzed-3kz0nwtjehhl0u4@ is the sample user account. Replace with the uuid of the user you wish to manipulate.
-See "user management":{{site.baseurl}}/admin/activation.html for an overview of how to use these commands.
+See "user management":{{site.baseurl}}/admin/user-management.html for an overview of how to use these commands.
h3. Setup a user
h2. Adding Permissions
-h3. VM login
+h3(#vm-login). VM login
-Give @$user_uuid@ permission to log in to @$vm_uuid@ as @$target_username@
+Give @$user_uuid@ permission to log in to @$vm_uuid@ as @$target_username@ and make sure that @$target_username@ is a member of the @docker@ group
<pre>
user_uuid=xxxxxxxchangeme
"head_uuid":"$vm_uuid",
"link_class":"permission",
"name":"can_login",
-"properties":{"username":"$target_username"}
+"properties":{"username":"$target_username", "groups": [ "docker" ]}
}
EOF
</pre>
notextile. <div class="spaced-out">
# A new user record is not set up, and not active. An inactive user cannot create or update any object, but can read Arvados objects that the user account has permission to read (such as publicly available items readable by the "anonymous" user).
-# Using Workbench or the "command line":{{site.baseurl}}/install/cheat_sheet.html , the admin invokes @setup@ on the user. The setup method adds the user to the "All users" group.
+# Using Workbench or the "command line":{{site.baseurl}}/admin/user-management-cli.html , the admin invokes @setup@ on the user. The setup method adds the user to the "All users" group.
- If "Users.AutoSetupNewUsers":config.html is true, this happens automatically during user creation, so in that case new users start at step (3).
- If "Users.AutoSetupNewUsersWithRepository":config.html is true, a new git repo is created for the user.
- If "Users.AutoSetupNewUsersWithVmUUID":config.html is set, the user is given login permission to the specified shell node
notextile. </div>
-User management can be performed through the web using Workbench or the command line. See "user management at the CLI":{{site.baseurl}}/install/cheat_sheet.html for specific examples.
+User management can be performed through the web using Workbench or the command line. See "user management at the CLI":{{site.baseurl}}/admin/user-management-cli.html for specific examples.
h2(#user_agreements). User agreements and self-activation
h3. setup
-Set up a user. Adds the user to the "All users" group. Enables the user to invoke @activate@. See "user management":{{site.baseurl}}/admin/activation.html for details.
+Set up a user. Adds the user to the "All users" group. Enables the user to invoke @activate@. See "user management":{{site.baseurl}}/admin/user-management.html for details.
Arguments:
h3. activate
-Check that a user has is set up and has signed all the user agreements. If so, activate the user. Users can invoke this for themselves. See "user agreements":{{site.baseurl}}/admin/activation.html#user_agreements for details.
+Check that a user has is set up and has signed all the user agreements. If so, activate the user. Users can invoke this for themselves. See "user agreements":{{site.baseurl}}/admin/user-management.html#user_agreements for details.
Arguments:
h3. unsetup
-Remove the user from the "All users" group and deactivate the user. See "user management":{{site.baseurl}}/admin/activation.html for details.
+Remove the user from the "All users" group and deactivate the user. See "user management":{{site.baseurl}}/admin/user-management.html for details.
Arguments:
* @owner_uuid@ of the system user.
* @link_class@ "permission"
-* @name@ one of *can_read*, *can_write* or *can_manage*
+* @name@ one of *can_read*, *can_write*, *can_manage* or *can_login*
* @head_uuid@ of some Arvados object
* @tail_uuid@ of a User or Group. For Group, the @group_class@ must be a "role".
If a User has *can_manage* permission on some object, the user has the ability to read, create, update and delete permission links with @head_uuid@ of the managed object. In other words, the user has the ability to modify the permission grants on the object.
+The *can_login* @name@ is only meaningful on a permission link with with @tail_uuid@ a user UUID and @head_uuid@ a Virtual Machine UUID. A permission link of this type gives the user UUID permission to log into the Virtual Machine UUID. The username for the VM is specified in the @properties@ field. Group membership can be specified that way as well, optionally. See the "VM login section on the CLI cheat sheet":/install/cheat_sheet.html#vm-login for an example.
+
h3. Transitive permissions
Permissions can be obtained indirectly through nested ownership (*can_manage*) or by following multiple permission links.
h2. User activation
-"Creation and activation of new users is described here.":{{site.baseurl}}/admin/activation.html
+"Creation and activation of new users is described here.":{{site.baseurl}}/admin/user-management.html
h2. Creating tokens via the API
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'notebox_begin' %}
+This installation method is not fully implemented, which is why this page is not yet listed in the "table of installation options":{{site.baseurl}}/install/index.html or in the left nav.
+{% include 'notebox_end' %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example@gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# apt-get update
+# apt-get install arvados-server-easy
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example@gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
+++ /dev/null
-../admin/user-management-cli.html.textile.liquid
\ No newline at end of file
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install client libraries
-
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The "SDK Reference":{{site.baseurl}}/sdk/index.html page has installation instructions for each of the SDKs.
-
---
layout: default
navsection: installguide
-title: Set up Slurm
+title: Configure Slurm
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin_warning' %}
+crunch-dispatch-slurm is only relevant for on premises clusters that will spool jobs to Slurm. Skip this section if you are installing a cloud cluster.
+{% include 'notebox_end' %}
+
Containers can be dispatched to a Slurm cluster. The dispatcher sends work to the cluster using Slurm's @sbatch@ command, so it works in a variety of SLURM configurations.
In order to run containers, you must run the dispatcher as a user that has permission to set up FUSE mounts and run Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster.
+We will assume that you have Slurm and munge running.
-On the API server, install Slurm and munge, and generate a munge key.
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo /usr/bin/apt-get install slurm-llnl munge</span>
-~$ <span class="userinput">sudo /usr/sbin/create-munge-key</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install slurm munge slurm-munge</span>
-</code></pre>
-</notextile>
+h3. Sample Slurm configuration file
-Now we need to give Slurm a configuration file. On Debian-based systems, this is installed at @/etc/slurm-llnl/slurm.conf@. On Red Hat-based systems, this is installed at @/etc/slurm/slurm.conf@. Here's an example @slurm.conf@:
+Here's an example @slurm.conf@ for use with Arvados:
<notextile>
<pre><code>
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Containers API Slurm prerequisites
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Overview
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% comment %}
-Obsolete page, no longer in nav.
-{% endcomment %}
-
-<notextile><script>
-window.location = "install-manual-prerequisites.html";
-</script></notextile>
-
-Please proceed to "Prerequisites":install-manual-prerequisites.html.
|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
|\3=. *Crunch (running containers)*|
|"arvados-dispatch-cloud":crunch2-cloud/install-dispatch-cloud.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static Slurm cluster such as on-premises HPC.|
-|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a Slurm cluster.|Optional, not needed for a Cloud installation, or if you wish to use Arvados for data management only.|
+|"crunch-dispatch-slurm":crunch2-slurm/install-dispatch.html |Run analysis workflows using Docker containers distributed across a Slurm cluster.|Optional, not needed for a Cloud installation, or if you wish to use Arvados for data management only.|
h2(#identity). Identity provider
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install pre-built Docker images
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This method is intended for evaluation and development on a local workstation. It is not suitable for production use in a cluster deployment.
-
-{% include 'notebox_begin' %}
-* The automatic network configuration allows you to log in to Workbench from a browser _running on the same host as Docker_. Connecting from other hosts requires additional configuration (not covered here).
-* Your data will be stored inside the Docker containers. You may stop and restart the containers without loss, but if you delete the container, your data will be gone.
-* Updating the Arvados software inside the Docker containers is not supported. You may download updated Docker images, but migrating data to updated containers is not yet supported.
-{% include 'notebox_end' %}
-
-h2. Prerequisites
-
-# A GNU/Linux x64 (virtual) machine
-# A working Docker installation (see "Installing Docker":https://docs.docker.com/installation/)
-# curl
-
-h2. Verify prerequisites
-
-Make sure that @curl@ and @docker@ are installed on your system, and that you are in the docker group (see "Installing Docker":https://docs.docker.com/installation/).
-
-<notextile><pre><code>~$ <span class="userinput">which curl</span>
-/usr/bin/curl
-~$ <span class="userinput">docker.io --version</span>
-Docker version 1.2.0-dev, build dc243c8
-~$ <span class="userinput">groups</span>
-yourusername sudo fuse docker
-</code></pre></notextile>
-
-h2. Download and install Arvados.
-
-<notextile>
-<pre><code>~$ <span class="userinput">\curl -sSL get.arvados.org | bash</span>
-</code></pre></notextile>
-
-This command will download the latest build of the Arvados docker images. It also gets the @arvdock@ command and saves it in the current working directory. It then uses @arvdock@ to spin up Arvados. Note that the Arvados Docker images are large and may take a while to download.
-
-If you prefer, you can also download and inspect the installation script before running it. @get.arvados.org@ redirects to "https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh":https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh, which is the installation script.
-
-The @arvdock@ command usage is listed here:
-
-<pre>
-usage: ./arvdock (start|stop|restart|reset|test) [options]
-
-start run new or restart stopped arvados containers
-stop stop arvados containers
-restart stop and then start arvados containers
-reset stop and delete containers WARNING: this will delete the data inside Arvados!
-test run tests
-
-./arvdock start/stop/restart options:
- -d[port], --doc[=port] Documentation server (default port 9898)
- -w[port], --workbench[=port] Workbench server (default port 9899)
- -s[port], --sso[=port] SSO server (default port 9901)
- -a[port], --api[=port] API server (default port 9900)
- -c, --compute Compute nodes (starts 2)
- -v, --vm Shell server
- -n, --nameserver Nameserver
- -k, --keep Keep servers
- -p, --keepproxy Keepproxy server
- -h, --help Display this help and exit
-
- If no options are given, the action is applied to all servers.
-
-./arvdock test [testname] [testname] ...
- By default, all tests are run.
-</pre>
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Create a Workflow with Composer
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The Arvados Workflow Composer is a graphical interface for building Common Workflow Language (CWL) workflows to run on Arvados.
-
-This tutorial will demonstrate:
-
-# Creating a new git repository through Arvados to store the workflow
-# Creating CommandLineTools for "sort" and "uniq"
-# Creating a Workflow which uses "sort" and "uniq" to remove duplicate lines from a text file
-# Submitting the Workflow to run on Arvados
-
-h3. 1. Access from workbench
-
-!(screenshot)c1.png!
-
-h3. 2. Composer starting page
-
-!(screenshot)c2.png!
-
-h3. 3. Manage git repositories (opens Workbench in new tab)
-
-!(screenshot)c2b.png!
-
-h3. 4. Add a new repository
-
-!(screenshot)c4.png!
-
-!(screenshot)c3.png!
-
-h3. 5. Return to Composer. Use refresh button to discover new repository (may take a few moments to show up).
-
-!(screenshot)c2c.png!
-
-h3. 6. Create a new Command Line Tool
-
-!(screenshot)c5.png!
-
-!(screenshot)c20.png!
-
-h3. 7. Set Docker image, base command, and input port for "sort" tool
-
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:10@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
-
-!(screenshot)c6.png!
-
-h3. 8. Redirect stdout to a file
-
-!(screenshot)c7.png!
-
-h3. 9. Capture output file
-
-!(screenshot)c8.png!
-
-h3. 10. Save Command Line Tool
-
-!(screenshot)c22.png!
-
-h3. 11. Repeat steps 6-10 for "uniq" tool
-
-Create a new tool with a "base command" of "uniq".
-
-h3. 12. Switch back to "Home" tab and create workflow
-
-!(screenshot)c24.png!
-
-!(screenshot)c9.png!
-
-!(screenshot)c10.png!
-
-h3. 13. Drag and drop tools into Workflow
-
-!(screenshot)c11.png!
-
-h3. 14. Drag from input port of "sort" to empty space to create workflow input
-
-!(screenshot)c21.png!
-
-h3. 15. Drag from output port of "sort" to input port of "uniq"
-
-!(screenshot)c13.png!
-
-h3. 16. Drag from output port of "uniq" to empty space to create workflow output
-
-!(screenshot)c14.png!
-
-h3. 17. Save Workflow
-
-!(screenshot)c23.png!
-
-h3. 18. Click on "Test" tab then click "Run"
-
-!(screenshot)c15.png!
-
-h3. 19. Choose input file
-
-You may need to "upload an input file":{{site.baseurl}}/user/tutorials/tutorial-keep.html
-
-!(screenshot)c16.png!
-
-h3. 20. Run the workflow
-
-!(screenshot)c17.png!
-
-h3. 21. Monitor progress (may take several minutes)
-
-!(screenshot)c18.png!
-
-h3. 22. Get workflow output
-
-!(screenshot)c19.png!
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Introduction to Crunch
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The Arvados "Crunch" framework is designed to support processing very large data batches (gigabytes to terabytes) efficiently, and provides the following benefits:
-* Increase concurrency by running tasks asynchronously, using many CPUs and network interfaces at once (especially beneficial for CPU-bound and I/O-bound tasks respectively).
-* Track inputs, outputs, and settings so you can verify that the inputs, settings, and sequence of programs you used to arrive at an output is really what you think it was.
-* Ensure that your programs and workflows are repeatable with different versions of your code, OS updates, etc.
-* Interrupt and resume long-running jobs consisting of many short tasks.
-* Maintain timing statistics automatically, so they're there when you want them.
-
-h2. Prerequisites
-
-To get the most value out of this section, you should be comfortable with the following:
-
-# Using a secure shell client such as SSH or PuTTY to log on to a remote server
-# Using the Unix command line shell, Bash
-# Viewing and editing files using a unix text editor such as vi, Emacs, or nano
-# Revision control using Git
-
-We also recommend you read the "Arvados Platform Overview":https://dev.arvados.org/projects/arvados/wiki#Platform-Overview for an introduction and background information about Arvados.
"fmt"
"io/ioutil"
"net"
+ "os"
"path/filepath"
)
func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
var san string
if net.ParseIP(super.ListenHost) != nil {
- san = fmt.Sprintf("IP:%s", super.ListenHost)
+ san += fmt.Sprintf(",IP:%s", super.ListenHost)
} else {
- san = fmt.Sprintf("DNS:%s", super.ListenHost)
+ san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+ }
+ if hostname, err := os.Hostname(); err != nil {
+ return fmt.Errorf("hostname: %w", err)
+ } else {
+ san += ",DNS:" + hostname
}
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
// Generate a self-signed root certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-x509", "-new", "-nodes", "-key", "rootCA.key", "-sha256", "-days", "3650", "-out", "rootCA.crt", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost")
if err != nil {
return err
}
// Generate server key
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "genrsa", "-out", "server.key", "2048")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "server.key", "2048")
if err != nil {
return err
}
if err != nil {
return err
}
- err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+ err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
if err != nil {
return err
}
// Generate signing request
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "req", "-new", "-sha256", "-key", "server.key", "-subj", "/C=US/ST=MA/O=Example Org/CN=localhost", "-reqexts", "SAN", "-config", "server.cfg", "-out", "server.csr")
if err != nil {
return err
}
// Sign certificate
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "x509", "-req", "-in", "server.csr", "-CA", "rootCA.crt", "-CAkey", "rootCA.key", "-CAcreateserial", "-out", "server.crt", "-extfile", "server.cfg", "-extensions", "SAN", "-days", "3650", "-sha256")
if err != nil {
return err
}
fmt.Fprintln(stdout, url)
if *shutdown {
super.Stop()
+ // Wait for children to exit. Don't report the
+ // ensuing "context cancelled" error, though:
+ // return nil to indicate successful startup.
+ _ = super.Wait()
+ fmt.Fprintln(stderr, "PASS - all services booted successfully")
+ return nil
}
}
// Wait for signal/crash + orderly shutdown
"fmt"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"regexp"
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
- "TMPDIR": super.tempdir,
+ "TMPDIR": super.wwwtempdir,
}
for _, cmpt := range []struct {
varname string
} {
port, err := internalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
port, err = externalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
}
vars[cmpt.varname+"SSLPORT"] = port
}
- tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ var conftemplate string
+ if super.ClusterType == "production" {
+ conftemplate = "/var/lib/arvados/share/nginx.conf"
+ } else {
+ conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+ }
+ tmpl, err := ioutil.ReadFile(conftemplate)
if err != nil {
return err
}
}
}
}
+
+ args := []string{
+ "-g", "error_log stderr info;",
+ "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
+ "-c", conffile,
+ }
+ // Nginx ignores "user www-data;" when running as a non-root
+ // user... except that it causes it to ignore our other -g
+ // options. So we still have to decide for ourselves whether
+ // it's needed.
+ if u, err := user.Current(); err != nil {
+ return fmt.Errorf("user.Current(): %w", err)
+ } else if u.Uid == "0" {
+ args = append([]string{"-g", "user www-data;"}, args...)
+ }
+
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", nil, nil, nginx,
- "-g", "error_log stderr info;",
- "-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
- "-c", conffile))
+ fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
}()
- return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+ // Choose one of the ports where Nginx should listen, and wait
+ // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+ testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+ if testurl.Port() == "" {
+ testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+ }
+ return waitForConnect(ctx, testurl.Host)
}
}
func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if super.ClusterType == "production" {
+ // passenger has already been installed via package
+ return nil
+ }
err := super.wait(ctx, runner.depends...)
if err != nil {
return err
defer passengerInstallMutex.Unlock()
var buf bytes.Buffer
- err = super.RunProgram(ctx, runner.src, &buf, nil, "gem", "list", "--details", "bundler")
+ err = super.RunProgram(ctx, runner.src, runOptions{output: &buf}, "gem", "list", "--details", "bundler")
if err != nil {
return err
}
- for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+ for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
break
}
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "install", "--jobs", "4", "--path", filepath.Join(os.Getenv("HOME"), ".gem"))
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "build-native-support")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "build-native-support")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "install-standalone-runtime")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "install-standalone-runtime")
if err != nil {
return err
}
- err = super.RunProgram(ctx, runner.src, nil, nil, "bundle", "exec", "passenger-config", "validate-install")
+ err = super.RunProgram(ctx, runner.src, runOptions{}, "bundle", "exec", "passenger-config", "validate-install")
if err != nil && !strings.Contains(err.Error(), "exit status 2") {
// Exit code 2 indicates there were warnings (like
// "other passenger installations have been detected",
}
type runPassenger struct {
- src string
- svc arvados.Service
- depends []supervisedTask
+ src string // path to app in source tree
+ varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+ svc arvados.Service
+ depends []supervisedTask
}
func (runner runPassenger) String() string {
if err != nil {
return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
}
+ var appdir string
+ if super.ClusterType == "production" {
+ appdir = "/var/lib/arvados/" + runner.varlibdir
+ } else {
+ appdir = runner.src
+ }
loglevel := "4"
if lvl, ok := map[string]string{
"debug": "5",
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+ cmdline := []string{
+ "bundle", "exec",
"passenger", "start",
"-p", port,
- "--log-file", "/dev/stderr",
"--log-level", loglevel,
"--no-friendly-error-pages",
- "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+ "--disable-anonymous-telemetry",
+ "--disable-security-update-check",
+ "--no-compile-runtime",
+ "--no-install-runtime",
+ "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+ }
+ opts := runOptions{
+ env: append([]string{
+ "TMPDIR=" + super.wwwtempdir,
+ }, railsEnv...),
+ }
+ if super.ClusterType == "production" {
+ opts.user = "www-data"
+ opts.env = append(opts.env, "HOME=/var/www")
+ } else {
+ // This would be desirable when changing uid
+ // too, but it fails because /dev/stderr is a
+ // symlink to a pty owned by root: "nginx:
+ // [emerg] open() "/dev/stderr" failed (13:
+ // Permission denied)"
+ cmdline = append(cmdline, "--log-file", "/dev/stderr")
+ }
+ err = super.RunProgram(ctx, appdir, opts, cmdline[0], cmdline[1:]...)
fail(err)
}()
return nil
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
+
iamroot := false
if u, err := user.Current(); err != nil {
- return fmt.Errorf("user.Current(): %s", err)
+ return fmt.Errorf("user.Current(): %w", err)
} else if u.Uid == "0" {
iamroot = true
}
buf := bytes.NewBuffer(nil)
- err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{output: buf}, "pg_config", "--bindir")
if err != nil {
return err
}
return err
}
prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+ opts := runOptions{}
if iamroot {
postgresUser, err := user.Lookup("postgres")
if err != nil {
if err != nil {
return err
}
- // We can't use "sudo -u" here because it creates an
- // intermediate process that interferes with our
- // ability to reliably kill postgres. The setuidgid
- // program just calls exec without forking, so it
- // doesn't have this problem.
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
- }
- err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
+ opts.user = "postgres"
+ }
+ err = super.RunProgram(ctx, super.tempdir, opts, prog, args...)
if err != nil {
return err
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "cp", "server.crt", "server.key", datadir)
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "cp", "server.crt", "server.key", datadir)
if err != nil {
return err
}
if iamroot {
- err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
if err != nil {
return err
}
"-k", datadir, // socket dir
"-p", super.cluster.PostgreSQL.Connection["port"],
}
+ opts := runOptions{}
if iamroot {
- args = append([]string{"postgres", prog}, args...)
- prog = "setuidgid"
+ opts.user = "postgres"
}
- fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
+ fail(super.RunProgram(ctx, super.tempdir, opts, prog, args...))
}()
for {
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
+ if super.ClusterType == "production" {
+ return nil
+ }
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb")
+ err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb")
if err != nil {
return err
}
}
func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
- binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
- err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ binfile := filepath.Join(super.bindir, "arvados-server")
+ err := super.RunProgram(ctx, super.bindir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile, runner.name, "-config", super.configfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
}()
}
return nil
return ctx.Err()
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, binfile, "-version")
if err != nil {
return err
}
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}, binfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile))
}()
}
return nil
"io"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"os/signal"
"os/user"
"path/filepath"
"reflect"
+ "strconv"
"strings"
"sync"
"syscall"
tasksReady map[string]chan bool
waitShutdown sync.WaitGroup
+ bindir string
tempdir string
+ wwwtempdir string
configfile string
environ []string // for child processes
}
return err
}
- super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
- if err != nil {
- return err
- }
- defer os.RemoveAll(super.tempdir)
- if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
- return err
+ // Choose bin and temp dirs: /var/lib/arvados/... in
+ // production, transient tempdir otherwise.
+ if super.ClusterType == "production" {
+ // These dirs have already been created by
+ // "arvados-server install" (or by extracting a
+ // package).
+ super.tempdir = "/var/lib/arvados/tmp"
+ super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+ super.bindir = "/var/lib/arvados/bin"
+ } else {
+ super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(super.tempdir)
+ super.wwwtempdir = super.tempdir
+ super.bindir = filepath.Join(super.tempdir, "bin")
+ if err := os.Mkdir(super.bindir, 0755); err != nil {
+ return err
+ }
}
// Fill in any missing config keys, and write the resulting
if err != nil {
return err
}
- conffile, err := os.OpenFile(filepath.Join(super.tempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
+ conffile, err := os.OpenFile(filepath.Join(super.wwwtempdir, "config.yml"), os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+ super.prependEnv("PATH", "/var/lib/arvados/bin:")
+ if super.ClusterType != "production" {
+ super.prependEnv("PATH", super.tempdir+"/bin:")
+ }
super.cluster, err = cfg.GetCluster("")
if err != nil {
"PID": os.Getpid(),
})
- if super.SourceVersion == "" {
+ if super.SourceVersion == "" && super.ClusterType == "production" {
+ // don't need SourceVersion
+ } else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "diff", "--shortstat")
if err != nil {
return err
}
dirty := buf.Len() > 0
buf.Reset()
- err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "log", "-n1", "--format=%H")
+ err = super.RunProgram(super.ctx, ".", runOptions{output: &buf}, "git", "log", "-n1", "--format=%H")
if err != nil {
return err
}
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
- runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
if super.ClusterType != "test" {
tasks = append(tasks,
- runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
- runGoProgram{src: "services/keep-balance"},
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
+ runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
super.tasksReady = map[string]chan bool{}
func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
_, basename := filepath.Split(srcpath)
- bindir := filepath.Join(super.tempdir, "bin")
- binfile := filepath.Join(bindir, basename)
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ binfile := filepath.Join(super.bindir, basename)
+ if super.ClusterType == "production" {
+ return binfile, nil
+ }
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), runOptions{env: []string{"GOBIN=" + super.bindir}}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
"GEM_PATH=",
})
gem := "gem"
- if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
gem = "/var/lib/arvados/bin/gem"
}
cmd := exec.Command(gem, "env", "gempath")
+ if super.ClusterType == "production" {
+ cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+ path, err := exec.LookPath("sudo")
+ if err != nil {
+ return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+ }
+ cmd.Path = path
+ }
+ cmd.Stderr = super.Stderr
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
- return fmt.Errorf("gem env gempath: %v", err)
+ return fmt.Errorf("gem env gempath: %w", err)
}
gempath := string(bytes.Split(buf, []byte{':'})[0])
super.prependEnv("PATH", gempath+"/bin:")
return prog
}
+type runOptions struct {
+ output io.Writer // attach stdout
+ env []string // add/replace environment variables
+ user string // run as specified user
+}
+
// RunProgram runs prog with args, using dir as working directory. If ctx is
// cancelled while the child is running, RunProgram terminates the child, waits
// for it to exit, then returns.
//
// Child's stdout will be written to output if non-nil, otherwise the
// boot command's stderr.
-func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.Writer, env []string, prog string, args ...string) error {
+func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOptions, prog string, args ...string) error {
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "setuidgid" && len(args) >= 2 {
- logprefix = args[1]
- }
- logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
- if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
- logprefix = args[1]
- } else if logprefix == "arvados-server" && len(args) > 1 {
- logprefix = args[0]
- }
- if !strings.HasPrefix(dir, "/") {
- logprefix = dir + ": " + logprefix
+ {
+ innerargs := args
+ if logprefix == "sudo" {
+ for i := 0; i < len(args); i++ {
+ if args[i] == "-u" {
+ i++
+ } else if args[i] == "-E" || strings.Contains(args[i], "=") {
+ } else {
+ logprefix = args[i]
+ innerargs = args[i+1:]
+ break
+ }
+ }
+ }
+ logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+ if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+ _, dirbase := filepath.Split(dir)
+ logprefix = innerargs[1] + "@" + dirbase
+ } else if logprefix == "arvados-server" && len(args) > 1 {
+ logprefix = args[0]
+ }
+ if !strings.HasPrefix(dir, "/") {
+ logprefix = dir + ": " + logprefix
+ }
}
cmd := exec.Command(super.lookPath(prog), args...)
}()
copiers.Add(1)
go func() {
- if output == nil {
+ if opts.output == nil {
io.Copy(logwriter, stdout)
} else {
- io.Copy(output, stdout)
+ io.Copy(opts.output, stdout)
}
copiers.Done()
}()
} else {
cmd.Dir = filepath.Join(super.SourcePath, dir)
}
- env = append([]string(nil), env...)
+ env := append([]string(nil), opts.env...)
env = append(env, super.environ...)
cmd.Env = dedupEnv(env)
+ if opts.user != "" {
+ // Note: We use this approach instead of "sudo"
+ // because in certain circumstances (we are pid 1 in a
+ // docker container, and our passenger child process
+ // changes to pgid 1) the intermediate sudo process
+ // notices we have the same pgid as our child and
+ // refuses to propagate signals from us to our child,
+ // so we can't signal/shutdown our passenger/rails
+ // apps. "chpst" or "setuidgid" would work, but these
+ // few lines avoid depending on runit/daemontools.
+ u, err := user.Lookup(opts.user)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(%q): %w", opts.user, err)
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ cmd.SysProcAttr = &syscall.SysProcAttr{
+ Credential: &syscall.Credential{
+ Uid: uint32(uid),
+ Gid: uint32(gid),
+ },
+ }
+ }
+
exited := false
defer func() { exited = true }()
go func() {
}
}
}
- if cluster.SystemRootToken == "" {
- cluster.SystemRootToken = randomHexString(64)
- }
- if cluster.ManagementToken == "" {
- cluster.ManagementToken = randomHexString(64)
- }
- if cluster.Collections.BlobSigningKey == "" {
- cluster.Collections.BlobSigningKey = randomHexString(64)
- }
- if cluster.Users.AnonymousUserToken == "" {
- cluster.Users.AnonymousUserToken = randomHexString(64)
- }
-
- if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
- buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
- if err != nil {
- return err
- }
- cluster.Containers.DispatchPrivateKey = string(buf)
- }
if super.ClusterType != "production" {
+ if cluster.SystemRootToken == "" {
+ cluster.SystemRootToken = randomHexString(64)
+ }
+ if cluster.ManagementToken == "" {
+ cluster.ManagementToken = randomHexString(64)
+ }
+ if cluster.Collections.BlobSigningKey == "" {
+ cluster.Collections.BlobSigningKey = randomHexString(64)
+ }
+ if cluster.Users.AnonymousUserToken == "" {
+ cluster.Users.AnonymousUserToken = randomHexString(64)
+ }
+ if cluster.Containers.DispatchPrivateKey == "" {
+ buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
+ if err != nil {
+ return err
+ }
+ cluster.Containers.DispatchPrivateKey = string(buf)
+ }
cluster.TLS.Insecure = true
}
if super.ClusterType == "test" {
return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
}
for u := range svc.InternalURLs {
- if _, p, err := net.SplitHostPort(u.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(u)
+ if p := u.Port(); p != "" {
return p, nil
- } else if u.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "ws" {
return "443", nil
} else {
return "80", nil
}
func externalPort(svc arvados.Service) (string, error) {
- if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(svc.ExternalURL)
+ if p := u.Port(); p != "" {
return p, nil
- } else if svc.ExternalURL.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "wss" {
return "443", nil
} else {
return "80", nil
// Already finalized.
return
}
- mt, err := runner.LogCollection.MarshalManifest(".")
- if err != nil {
- err = fmt.Errorf("error creating log manifest: %v", err)
- return
- }
updates := arvadosclient.Dict{
- "name": "logs for " + runner.Container.UUID,
- "manifest_text": mt,
+ "name": "logs for " + runner.Container.UUID,
}
+ mt, err1 := runner.LogCollection.MarshalManifest(".")
+ if err1 == nil {
+ // Only send updated manifest text if there was no
+ // error.
+ updates["manifest_text"] = mt
+ }
+
+ // Even if flushing the manifest had an error, we still want
+ // to update the log record, if possible, to push the trash_at
+ // and delete_at times into the future. Details on bug
+ // #17293.
if final {
updates["is_trashed"] = true
} else {
updates["delete_at"] = exp
}
reqBody := arvadosclient.Dict{"collection": updates}
+ var err2 error
if runner.logUUID == "" {
reqBody["ensure_unique_name"] = true
- err = runner.DispatcherArvClient.Create("collections", reqBody, &response)
+ err2 = runner.DispatcherArvClient.Create("collections", reqBody, &response)
} else {
- err = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
+ err2 = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
}
- if err != nil {
- return
+ if err2 == nil {
+ runner.logUUID = response.UUID
+ }
+
+ if err1 != nil || err2 != nil {
+ err = fmt.Errorf("error recording logs: %q, %q", err1, err2)
}
- runner.logUUID = response.UUID
return
}
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
})
mux.Handler("GET", "/metrics", metricsH)
mux.Handler("GET", "/metrics.json", metricsH)
+ mux.Handler("GET", "/_health/:check", &health.Handler{
+ Token: disp.Cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": disp.CheckHealth},
+ })
disp.httpHandler = auth.RequireLiteralToken(disp.Cluster.ManagementToken, mux)
}
}
"io"
"os"
"os/exec"
+ "os/user"
+ "path/filepath"
"strconv"
"strings"
"syscall"
"github.com/lib/pq"
)
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
const devtestDatabasePassword = "insecure_arvados_test"
-type installCommand struct{}
+type installCommand struct {
+ ClusterType string
+ SourcePath string
+ PackageVersion string
+ EatMyData bool
+}
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.New(stderr, "text", "info")
ctx := ctxlog.Context(context.Background(), logger)
ctx, cancel := context.WithCancel(ctx)
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.SetOutput(stderr)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
- clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+ flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+ flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
+ flags.BoolVar(&inst.EatMyData, "eatmydata", false, "use eatmydata to speed up install")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
return 2
} else if *versionFlag {
return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
}
- var dev, test, prod bool
- switch *clusterType {
+ var dev, test, prod, pkg bool
+ switch inst.ClusterType {
case "development":
dev = true
case "test":
test = true
case "production":
prod = true
+ case "package":
+ pkg = true
default:
- err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+ err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
return 2
}
}
}
- if dev || test {
- debs := []string{
+ if inst.EatMyData {
+ cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends", "eatmydata")
+ cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ pkgs := prodpkgs(osv)
+
+ if pkg {
+ pkgs = append(pkgs,
+ "dpkg-dev",
+ "eatmydata", // install it for later steps, even if we're not using it now
+ "rsync",
+ )
+ }
+
+ if dev || test || pkg {
+ pkgs = append(pkgs,
+ "automake",
"bison",
"bsdmainutils",
"build-essential",
- "ca-certificates",
"cadaver",
"curl",
"cython3",
- "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
- "fuse",
"gettext",
- "git",
- "gitolite3",
- "graphviz",
- "haveged",
"iceweasel",
"libattr1-dev",
"libcrypt-ssleay-perl",
- "libcrypt-ssleay-perl",
- "libcurl3-gnutls",
- "libcurl4-openssl-dev",
"libfuse-dev",
"libgnutls28-dev",
"libjson-perl",
- "libjson-perl",
"libpam-dev",
"libpcre3-dev",
"libpq-dev",
"libssl-dev",
"libwww-perl",
"libxml2-dev",
- "libxslt1.1",
+ "libxslt1-dev",
"linkchecker",
"lsof",
+ "make",
"net-tools",
- "nginx",
"pandoc",
"perl-modules",
"pkg-config",
"sudo",
"wget",
"xvfb",
- "zlib1g-dev",
- }
+ )
switch {
case osv.Debian && osv.Major >= 10:
- debs = append(debs, "libcurl4")
+ pkgs = append(pkgs, "libcurl4")
default:
- debs = append(debs, "libcurl3")
+ pkgs = append(pkgs, "libcurl3")
}
- cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
- cmd.Args = append(cmd.Args, debs...)
+ cmd := exec.CommandContext(ctx, "apt-get")
+ if inst.EatMyData {
+ cmd = exec.CommandContext(ctx, "eatmydata", "apt-get")
+ }
+ cmd.Args = append(cmd.Args, "install", "--yes", "--no-install-recommends")
+ cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
cmd.Stderr = stderr
}
os.Mkdir("/var/lib/arvados", 0755)
- rubyversion := "2.5.7"
+ os.Mkdir("/var/lib/arvados/tmp", 0700)
+ if prod || pkg {
+ os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+ u, er := user.Lookup("www-data")
+ if er != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+ return 1
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+ if err != nil {
+ return 1
+ }
+ }
+ rubyversion := "2.7.2"
+ rubymajorversion := rubyversion[:strings.LastIndex(rubyversion, ".")]
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
} else {
- err = runBash(`
-mkdir -p /var/lib/arvados/tmp
-tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
-trap "rm -r ${tmp}" ERR
-wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+ err = inst.runBash(`
+tmp="$(mktemp -d)"
+trap 'rm -r "${tmp}"' ERR EXIT
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/`+rubymajorversion+`/ruby-`+rubyversion+`.tar.gz | tar -C "${tmp}" -xzf -
+cd "${tmp}/ruby-`+rubyversion+`"
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
+make -j8
make install
-/var/lib/arvados/bin/gem install bundler
-rm -r ${tmp}
+/var/lib/arvados/bin/gem install bundler --no-document
`, stdout, stderr)
if err != nil {
return 1
if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
logger.Print("go " + goversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
cd /tmp
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
return 1
}
}
+ }
+ if !prod && !pkg {
pjsversion := "1.9.8"
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
logger.Print("geckodriver " + geckoversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
GD=v`+geckoversion+`
wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
NJS=`+nodejsversion+`
wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
logger.Print("gradle " + gradleversion + " already installed")
} else {
- err = runBash(`
+ err = inst.runBash(`
G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
logger.Print("locale " + wantlocale + " already installed")
} else {
- err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+ err = inst.runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
if err != nil {
return 1
}
// locales. Otherwise, it might need a
// restart, so we attempt to restart it with
// systemd.
- if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+ if err = inst.runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
} else if err = waitPostgreSQLReady(); err != nil {
return 1
}
}
+ if prod || pkg {
+ // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+ for dstdir, srcdir := range map[string]string{
+ "railsapi": "services/api",
+ "workbench1": "apps/workbench",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("rsync",
+ "-a", "--no-owner", "--no-group", "--delete-after", "--delete-excluded",
+ "--exclude", "/coverage",
+ "--exclude", "/log",
+ "--exclude", "/tmp",
+ "--exclude", "/vendor",
+ "--exclude", "/config/environments",
+ "./", "/var/lib/arvados/"+dstdir+"/")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ for _, cmdline := range [][]string{
+ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
+ {"touch", "log/production.log"},
+ {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+ } {
+ cmd = exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+ // Exit code 2 indicates there were warnings (like
+ // "other passenger installations have been detected",
+ // which we can't expect to avoid) but no errors.
+ // Other non-zero exit codes (1, 9) indicate errors.
+ return 1
+ }
+ }
+
+ // Install Go programs to /var/lib/arvados/bin/
+ for _, srcdir := range []string{
+ "cmd/arvados-client",
+ "cmd/arvados-server",
+ "services/arv-git-httpd",
+ "services/crunch-dispatch-local",
+ "services/crunch-dispatch-slurm",
+ "services/health",
+ "services/keep-balance",
+ "services/keep-web",
+ "services/keepproxy",
+ "services/keepstore",
+ "services/ws",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+ cmd.Env = append(cmd.Env, os.Environ()...)
+ cmd.Env = append(cmd.Env, "GOBIN=/var/lib/arvados/bin")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ // Copy assets from source tree to /var/lib/arvados/share
+ cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
return 0
}
type osversion struct {
Debian bool
Ubuntu bool
+ Centos bool
Major int
}
osv.Ubuntu = true
case "debian":
osv.Debian = true
+ case "centos":
+ osv.Centos = true
default:
return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
}
}
}
-func runBash(script string, stdout, stderr io.Writer) error {
+func (inst *installCommand) runBash(script string, stdout, stderr io.Writer) error {
cmd := exec.Command("bash", "-")
+ if inst.EatMyData {
+ cmd = exec.Command("eatmydata", "bash", "-")
+ }
cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
cmd.Stdout = stdout
cmd.Stderr = stderr
return cmd.Run()
}
+
+func prodpkgs(osv osversion) []string {
+ pkgs := []string{
+ "ca-certificates",
+ "curl",
+ "fuse",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "libcurl3-gnutls",
+ "libxslt1.1",
+ "nginx",
+ "python",
+ "sudo",
+ }
+ if osv.Debian || osv.Ubuntu {
+ if osv.Debian && osv.Major == 8 {
+ pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+ } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+ pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+ }
+ return append(pkgs,
+ "g++",
+ "libcurl4-openssl-dev", // services/api
+ "libpq-dev",
+ "libpython2.7", // services/fuse
+ "mime-support", // keep-web
+ "zlib1g-dev", // services/api
+ )
+ } else if osv.Centos {
+ return append(pkgs,
+ "fuse-libs", // services/fuse
+ "gcc",
+ "gcc-c++",
+ "libcurl-devel", // services/api
+ "mailcap", // keep-web
+ "postgresql-devel", // services/api
+ )
+ } else {
+ panic("os version not supported")
+ }
+}
+
+func ProductionDependencies() ([]string, error) {
+ osv, err := identifyOS()
+ if err != nil {
+ return nil, err
+ }
+ return prodpkgs(osv), nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/user"
+ "regexp"
+ "strconv"
+ "text/template"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+ ClusterID string
+ Domain string
+ PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("Hostname(): %w", err)
+ return 1
+ }
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+ flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
+ } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+ err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+ return 1
+ }
+
+ wwwuser, err := user.Lookup("www-data")
+ if err != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+ return 1
+ }
+ wwwgid, err := strconv.Atoi(wwwuser.Gid)
+ if err != nil {
+ return 1
+ }
+ initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+ err = os.Mkdir("/var/lib/arvados/keep", 0600)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+ err = os.Mkdir("/etc/arvados", 0750)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ return 1
+ }
+ err = os.Chown("/etc/arvados", 0, wwwgid)
+ f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ tmpl, err := template.New("config").Parse(`Clusters:
+ {{.ClusterID}}:
+ Services:
+ Controller:
+ InternalURLs:
+ "http://0.0.0.0:8003/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ Websocket:
+ InternalURLs:
+ "http://0.0.0.0:8005/": {}
+ ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ Keepbalance:
+ InternalURLs:
+ "http://0.0.0.0:9005/": {}
+ GitHTTP:
+ InternalURLs:
+ "http://0.0.0.0:9001/": {}
+ ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ DispatchCloud:
+ InternalURLs:
+ "http://0.0.0.0:9006/": {}
+ Keepproxy:
+ InternalURLs:
+ "http://0.0.0.0:25108/": {}
+ ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ WebDAV:
+ InternalURLs:
+ "http://0.0.0.0:9002/": {}
+ ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ WebDAVDownload:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ Keepstore:
+ InternalURLs:
+ "http://0.0.0.0:25107/": {}
+ Composer:
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ Workbench1:
+ InternalURLs:
+ "http://0.0.0.0:8001/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+ #Workbench2:
+ # InternalURLs:
+ # "http://0.0.0.0:8002/": {}
+ # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ Health:
+ InternalURLs:
+ "http://0.0.0.0:9007/": {}
+ Collections:
+ BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+ Containers:
+ DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+ ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+ PostgreSQL:
+ Connection:
+ dbname: arvados_production
+ host: localhost
+ user: arvados
+ password: {{printf "%q" .PostgreSQLPassword}}
+ SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ TLS:
+ Insecure: true
+ Volumes:
+ {{.ClusterID}}-nyw5e-000000000000000:
+ Driver: Directory
+ DriverParameters:
+ Root: /var/lib/arvados/keep
+ Replication: 2
+ Workbench:
+ SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+`)
+ if err != nil {
+ return 1
+ }
+ err = tmpl.Execute(f, initcmd)
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ return 1
+ }
+ err = f.Close()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+ ldr := config.NewLoader(nil, logger)
+ ldr.SkipLegacy = true
+ cfg, err := ldr.Load()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+
+ err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+ if err != nil {
+ return 1
+ }
+
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ cmd.Dir = "/var/lib/arvados/railsapi"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("rake db:setup: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "initialized database")
+
+ return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+ privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+ if err != nil {
+ return "", err
+ }
+ err = privkey.Validate()
+ if err != nil {
+ return "", err
+ }
+ return string(pem.EncodeToMemory(&pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+ })), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+ b := make([]byte, chars/2)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+ for _, sql := range []string{
+ `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+ `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+ } {
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
+ }
+ }
+ return nil
+}
// RuntimeConstraints specify a container's compute resources (RAM,
// CPU) and network connectivity.
type RuntimeConstraints struct {
- API bool `json:"api"`
+ API bool `json:"API"`
RAM int64 `json:"ram"`
VCPUs int `json:"vcpus"`
KeepCacheRAM int64 `json:"keep_cache_ram"`
}
req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
+ // Avoid workbench1's redirect-http-to-https feature
+ req.Header.Set("X-Forwarded-Proto", "https")
+
ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout))
defer cancel()
req = req.WithContext(ctx)
'[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
- client_body_temp_path "{{TMPDIR}}";
- proxy_temp_path "{{TMPDIR}}";
- fastcgi_temp_path "{{TMPDIR}}";
- uwsgi_temp_path "{{TMPDIR}}";
- scgi_temp_path "{{TMPDIR}}";
+ client_body_temp_path "{{TMPDIR}}/nginx";
+ proxy_temp_path "{{TMPDIR}}/nginx";
+ fastcgi_temp_path "{{TMPDIR}}/nginx";
+ uwsgi_temp_path "{{TMPDIR}}/nginx";
+ scgi_temp_path "{{TMPDIR}}/nginx";
+ upstream controller {
+ server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+ server_name controller ~.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream arv-git-http {
server {{LISTENHOST}}:{{GITPORT}};
}
server {
- listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
- server_name arv-git-http;
+ listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+ server_name arv-git-http git.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{KEEPPROXYPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name keepproxy;
+ listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+ server_name keepproxy keep.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{KEEPWEBPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name keep-web;
+ listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+ server_name keep-web collections.* ~\.collections\.;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{HEALTHPORT}};
}
server {
- listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
- server_name health;
+ listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+ server_name health health.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
}
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name keep-web-dl ~.*;
+ listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+ server_name keep-web-dl download.* ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{WSPORT}};
}
server {
- listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
- server_name websocket;
+ listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+ server_name websocket ws.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
server {{LISTENHOST}}:{{WORKBENCH1PORT}};
}
server {
- listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
- server_name workbench1;
+ listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+ server_name workbench1 workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
proxy_redirect off;
}
}
- upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
- }
- server {
- listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
- server_name controller;
- ssl_certificate "{{SSLCERT}}";
- ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://controller;
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
}
# request.
def fill_container_defaults
self.runtime_constraints = {
- 'api' => false,
+ 'API' => false,
'keep_cache_ram' => 0,
'ram' => 0,
'vcpus' => 0,
command: ["echo", "hello"],
output_path: "test",
runtime_constraints: {
- "api" => false,
+ "API" => false,
"keep_cache_ram" => 0,
"ram" => 12000000000,
"vcpus" => 4,
set_user_from_auth :active
env = {"C" => "3", "B" => "2", "A" => "1"}
m = {"F" => {"kind" => "3"}, "E" => {"kind" => "2"}, "D" => {"kind" => "1"}}
- rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "api" => true}
+ rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1, "API" => true}
c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
c.reload
assert_equal Container.deep_sort_hash(env).to_json, c.environment.to_json
"flag"
"fmt"
"io"
- "net/http"
"os"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
}
srv := &Server{
- Handler: http.NotFoundHandler(),
Cluster: cluster,
ArvClient: ac,
RunOptions: options,
Logger: options.Logger,
Dumper: options.Dumper,
}
+ srv.Handler = &health.Handler{
+ Token: cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": srv.CheckHealth},
+ }
go srv.run()
return srv
log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
if notFoundBlocks > 0 {
- return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
+ return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix", notFoundBlocks, totalBlocks)
}
return nil
allLocators = append(allLocators, TestHash2)
err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true)
c.Check(err, NotNil)
// Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
setupKeepBlockCheck(c, true, "")
setupTestData(c)
err := performKeepBlockCheck(kc, blobSignatureTTL, "badblobsigningkey", []string{TestHash, TestHash2}, false)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403")
// verbose logging not requested
c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false)
func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
setupKeepBlockCheck(c, false, testKeepServicesJSON)
err := performKeepBlockCheck(kc, blobSignatureTTL, "", []string{TestHash, TestHash2}, true)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "")
}
args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
err := doMain(args)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
}
projects / arvados.git / commitdiff
