From 60801a7c485c50e3c2f518f9b3ad6f86c8783fca Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 23 Jul 2020 17:31:39 -0400 Subject: [PATCH] 16306: Packaging dev cycle, type=production support in lib/boot. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- cmd/arvados-dev/buildpackage.go | 4 +- cmd/arvados-dev/docker-boot.sh | 42 ++++ cmd/arvados-dev/docker-build-install.sh | 123 +++++++++++ cmd/arvados-dev/example.sh | 35 ---- cmd/arvados-server/cmd.go | 1 + lib/boot/nginx.go | 4 +- lib/boot/passenger.go | 42 +++- lib/boot/postgresql.go | 4 + lib/boot/seed.go | 3 + lib/boot/service.go | 4 +- lib/boot/supervisor.go | 78 ++++--- lib/install/deps.go | 27 ++- lib/install/init.go | 265 ++++++++++++++++++++++++ 13 files changed, 556 insertions(+), 76 deletions(-) create mode 100755 cmd/arvados-dev/docker-boot.sh create mode 100755 cmd/arvados-dev/docker-build-install.sh delete mode 100755 cmd/arvados-dev/example.sh create mode 100644 lib/install/init.go diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go index 4b395c4baa..aee955ea2e 100644 --- a/cmd/arvados-dev/buildpackage.go +++ b/cmd/arvados-dev/buildpackage.go @@ -85,12 +85,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin cmd.Args = append(cmd.Args, "--depends", pkg) } cmd.Args = append(cmd.Args, + "--deb-use-file-permissions", + "--rpm-use-file-permissions", "--exclude", "/var/lib/arvados/go", "/var/lib/arvados", "/var/www/.gem", "/var/www/.passenger", ) - fmt.Fprintf(stderr, "%s...\n", cmd.Args) + fmt.Fprintf(stderr, "... %s\n", cmd.Args) cmd.Dir = bldr.OutputDir cmd.Stdout = stdout cmd.Stderr = stderr diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh new file mode 100755 index 0000000000..e8703e41f0 --- /dev/null +++ b/cmd/arvados-dev/docker-boot.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Bring up a docker container with some locally-built commands (e.g., +# cmd/arvados-server) replacing the ones that came with +# arvados-server-easy when the arvados-installpackage-* image was +# built. +# +# Assumes docker-build-install.sh has already succeeded. +# +# Example: +# +# docker-boot.sh cmd/arvados-server services/keep-balance + +set -e -o pipefail + +cleanup() { + if [[ -n "${tmpdir}" ]]; then + rm -rf "${tmpdir}" + fi +} +trap cleanup ERR EXIT + +tmpdir=$(mktemp -d) +version=$(git describe --tag --dirty) + +declare -a volargs=() +for srcdir in "$@"; do + echo >&2 "building $srcdir..." + (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}") + cmd="$(basename "$srcdir")" + volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro") +done + +osbase=debian:10 +installimage=arvados-installpackage-${osbase} +docker run -it --rm \ + "${volargs[@]}" \ + "${installimage}" \ + bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot' diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh new file mode 100755 index 0000000000..3c6e177cf8 --- /dev/null +++ b/cmd/arvados-dev/docker-build-install.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Build an arvados-server-easy package, then install and run it on a +# base OS image. +# +# Examples: +# +# docker-build-install.sh --force-buildimage --force-installimage # always build fresh docker images +# +# docker-build-install.sh # reuse cached docker images if possible + +set -e -o pipefail + +declare -A opts=() +while [[ $# -gt 0 ]]; do + arg="$1" + shift + case "$arg" in + --force-buildimage) + opts[force-buildimage]=1 + ;; + --force-installimage) + opts[force-installimage]=1 + ;; + *) + echo >&2 "invalid argument '$arg'" + exit 1 + esac +done + +cleanup() { + if [[ -n "${buildctr}" ]]; then + docker rm "${buildctr}" || true + fi + if [[ -n "${installctr}" ]]; then + docker rm "${installctr}" || true + fi +} +trap cleanup ERR EXIT + +version=$(git describe --tag --dirty) +osbase=debian:10 + +mkdir -p /tmp/pkg + +buildimage=arvados-buildpackage-${osbase} +if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then + ( + echo >&2 building arvados-server... + cd cmd/arvados-server + go install + ) + echo >&2 building ${buildimage}... + buildctr=${buildimage/:/-} + docker rm "${buildctr}" || true + docker run \ + --name "${buildctr}" \ + -v /tmp/pkg:/pkg \ + -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \ + -v "$(pwd)":/arvados:ro \ + "${osbase}" \ + /arvados-server install \ + -type package \ + -source /arvados \ + -package-version "${version}" + docker commit "${buildctr}" "${buildimage}" + docker rm "${buildctr}" + buildctr= +fi + +pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb +rm -v -f "${pkgfile}" + +( + echo >&2 building arvados-dev... + cd cmd/arvados-dev + go install +) +echo >&2 building ${pkgfile}... +docker run --rm \ + -v /tmp/pkg:/pkg \ + -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \ + -v "$(pwd)":/arvados:ro \ + "${buildimage}" \ + /arvados-dev buildpackage \ + -source /arvados \ + -package-version "${version}" \ + -output-directory /pkg + +ls -l ${pkgfile} +( + echo >&2 dpkg-scanpackages... + cd /tmp/pkg + dpkg-scanpackages . | gzip > Packages.gz +) +sourcesfile=/tmp/sources.conf.d-arvados +echo >$sourcesfile "deb [trusted=yes] file:///pkg ./" + +installimage="arvados-installpackage-${osbase}" +if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then + echo >&2 building ${installimage}... + installctr=${installimage/:/-} + docker rm "${installctr}" || true + docker run -it \ + --name "${installctr}" \ + -v /tmp/pkg:/pkg:ro \ + -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \ + "${osbase}" \ + bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql' + docker commit "${installctr}" "${installimage}" + docker rm "${installctr}" + installctr= +fi + +echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados... +docker run -it --rm \ + -v /tmp/pkg:/pkg:ro \ + -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \ + "${installimage}" \ + bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot' diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh deleted file mode 100755 index 072dfcf9c4..0000000000 --- a/cmd/arvados-dev/example.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -set -e -o pipefail - -version="${PACKAGE_VERSION:-0.9.99}" - -# mkdir -p /tmp/pkg -# ( -# cd cmd/arvados-dev -# go install -# ) -# docker run --rm \ -# -v /tmp/pkg:/pkg \ -# -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \ -# -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \ -# /arvados-dev buildpackage \ -# -source /arvados \ -# -package-version "${version}" \ -# -output-directory /pkg -pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb -# ls -l ${pkgfile} -# ( -# cd /tmp/pkg -# dpkg-scanpackages . | gzip > Packages.gz -# ) -sourcesfile=/tmp/sources.conf.d-arvados -echo >$sourcesfile "deb [trusted=yes] file:///pkg ./" -docker run -it --rm \ - -v /tmp/pkg:/pkg:ro \ - -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \ - ${INSTALLIMAGE:-debian:10} \ - bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login' diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index ff99de75c4..d0aa9da94d 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -34,6 +34,7 @@ var ( "crunch-run": crunchrun.Command, "dispatch-cloud": dispatchcloud.Command, "install": install.Command, + "init": install.InitCommand, "recover-collection": recovercollection.Command, "ws": ws.Command, }) diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go index 0f105d6b6c..c1da7d18d1 100644 --- a/lib/boot/nginx.go +++ b/lib/boot/nginx.go @@ -53,7 +53,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er } { port, err := internalPort(cmpt.svc) if err != nil { - return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc) + return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc) } if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil { return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err) @@ -62,7 +62,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er port, err = externalPort(cmpt.svc) if err != nil { - return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc) + return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc) } if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil { return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err) diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go index f18300c4cc..481300b45e 100644 --- a/lib/boot/passenger.go +++ b/lib/boot/passenger.go @@ -37,6 +37,10 @@ func (runner installPassenger) String() string { } func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error { + if super.ClusterType == "production" { + // passenger has already been installed via package + return nil + } err := super.wait(ctx, runner.depends...) if err != nil { return err @@ -52,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super } for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} { if !strings.Contains(buf.String(), "("+version+")") { - err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2") + err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2") if err != nil { return err } @@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super } type runPassenger struct { - src string - svc arvados.Service - depends []supervisedTask + src string // path to app in source tree + varlibdir string // path to app (relative to /var/lib/arvados) in OS package + svc arvados.Service + depends []supervisedTask } func (runner runPassenger) String() string { @@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup if err != nil { return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc) } + var appdir string + if super.ClusterType == "production" { + appdir = "/var/lib/arvados/" + runner.varlibdir + } else { + appdir = runner.src + } loglevel := "4" if lvl, ok := map[string]string{ "debug": "5", @@ -116,13 +127,30 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup super.waitShutdown.Add(1) go func() { defer super.waitShutdown.Done() - err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec", + cmdline := []string{ + "bundle", "exec", "passenger", "start", "-p", port, - "--log-file", "/dev/stderr", "--log-level", loglevel, "--no-friendly-error-pages", - "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid")) + "--disable-anonymous-telemetry", + "--disable-security-update-check", + "--no-compile-runtime", + "--no-install-runtime", + "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"), + } + if super.ClusterType == "production" { + cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...) + } else { + // This would be desirable in the production + // case too, but it fails with sudo because + // /dev/stderr is a symlink to a pty owned by + // root: "nginx: [emerg] open() "/dev/stderr" + // failed (13: Permission denied)" + cmdline = append(cmdline, "--log-file", "/dev/stderr") + } + env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...) + err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...) fail(err) }() return nil diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go index 34ccf04a88..199a93a9d5 100644 --- a/lib/boot/postgresql.go +++ b/lib/boot/postgresql.go @@ -36,6 +36,10 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso return err } + if super.ClusterType == "production" { + return nil + } + iamroot := false if u, err := user.Current(); err != nil { return fmt.Errorf("user.Current(): %s", err) diff --git a/lib/boot/seed.go b/lib/boot/seed.go index d1cf2a8709..1f07601a09 100644 --- a/lib/boot/seed.go +++ b/lib/boot/seed.go @@ -20,6 +20,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor if err != nil { return err } + if super.ClusterType == "production" { + return nil + } err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup") if err != nil { return err diff --git a/lib/boot/service.go b/lib/boot/service.go index 5afacfe716..77fdc98be0 100644 --- a/lib/boot/service.go +++ b/lib/boot/service.go @@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string { } func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error { - binfile := filepath.Join(super.tempdir, "bin", "arvados-server") - err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version") + binfile := filepath.Join(super.bindir, "arvados-server") + err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version") if err != nil { return err } diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index e38a4775e8..2d76972336 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -14,6 +14,7 @@ import ( "io" "io/ioutil" "net" + "net/url" "os" "os/exec" "os/signal" @@ -54,7 +55,9 @@ type Supervisor struct { tasksReady map[string]chan bool waitShutdown sync.WaitGroup + bindir string tempdir string + wwwtempdir string configfile string environ []string // for child processes } @@ -131,13 +134,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error { return err } - super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") - if err != nil { - return err - } - defer os.RemoveAll(super.tempdir) - if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil { - return err + // Choose bin and temp dirs: /var/lib/arvados/... in + // production, transient tempdir otherwise. + if super.ClusterType == "production" { + // These dirs have already been created by + // "arvados-server install" (or by extracting a + // package). + super.tempdir = "/var/lib/arvados/tmp" + super.wwwtempdir = "/var/lib/arvados/wwwtmp" + super.bindir = "/var/lib/arvados/bin" + } else { + super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") + if err != nil { + return err + } + defer os.RemoveAll(super.tempdir) + super.wwwtempdir = super.tempdir + super.bindir = filepath.Join(super.tempdir, "bin") + if err := os.Mkdir(super.bindir, 0755); err != nil { + return err + } } // Fill in any missing config keys, and write the resulting @@ -166,7 +182,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error { super.setEnv("ARVADOS_CONFIG", super.configfile) super.setEnv("RAILS_ENV", super.ClusterType) super.setEnv("TMPDIR", super.tempdir) - super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:") + super.prependEnv("PATH", "/var/lib/arvados/bin:") + if super.ClusterType != "production" { + super.prependEnv("PATH", super.tempdir+"/bin:") + } super.cluster, err = cfg.GetCluster("") if err != nil { @@ -182,7 +201,9 @@ func (super *Supervisor) run(cfg *arvados.Config) error { "PID": os.Getpid(), }) - if super.SourceVersion == "" { + if super.SourceVersion == "" && super.ClusterType == "production" { + // don't need SourceVersion + } else if super.SourceVersion == "" { // Find current source tree version. var buf bytes.Buffer err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat") @@ -224,15 +245,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error { runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}}, installPassenger{src: "services/api"}, - runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, + runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup - runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, seedDatabase{}, } if super.ClusterType != "test" { tasks = append(tasks, runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller}, - runGoProgram{src: "services/keep-balance"}, + runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance}, ) } super.tasksReady = map[string]chan bool{} @@ -382,9 +403,11 @@ func dedupEnv(in []string) []string { func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) { _, basename := filepath.Split(srcpath) - bindir := filepath.Join(super.tempdir, "bin") - binfile := filepath.Join(bindir, basename) - err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) + binfile := filepath.Join(super.bindir, basename) + if super.ClusterType == "production" { + return binfile, nil + } + err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) return binfile, err } @@ -401,10 +424,19 @@ func (super *Supervisor) setupRubyEnv() error { "GEM_PATH=", }) gem := "gem" - if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil { + if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" { gem = "/var/lib/arvados/bin/gem" } cmd := exec.Command(gem, "env", "gempath") + if super.ClusterType == "production" { + cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...) + path, err := exec.LookPath("sudo") + if err != nil { + return fmt.Errorf("LookPath(\"sudo\"): %w", err) + } + cmd.Path = path + } + cmd.Stderr = super.Stderr cmd.Env = super.environ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:... if err != nil || len(buf) == 0 { @@ -694,11 +726,10 @@ func internalPort(svc arvados.Service) (string, error) { return "", errors.New("internalPort() doesn't work with multiple InternalURLs") } for u := range svc.InternalURLs { - if _, p, err := net.SplitHostPort(u.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(u) + if p := u.Port(); p != "" { return p, nil - } else if u.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "ws" { return "443", nil } else { return "80", nil @@ -708,11 +739,10 @@ func internalPort(svc arvados.Service) (string, error) { } func externalPort(svc arvados.Service) (string, error) { - if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(svc.ExternalURL) + if p := u.Port(); p != "" { return p, nil - } else if svc.ExternalURL.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "wss" { return "443", nil } else { return "80", nil diff --git a/lib/install/deps.go b/lib/install/deps.go index f9b962fdd8..3f19aa1a82 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -14,6 +14,7 @@ import ( "io" "os" "os/exec" + "os/user" "path/filepath" "strconv" "strings" @@ -176,12 +177,26 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } os.Mkdir("/var/lib/arvados", 0755) + os.Mkdir("/var/lib/arvados/tmp", 0700) + if prod { + os.Mkdir("/var/lib/arvados/wwwtmp", 0700) + u, er := user.Lookup("www-data") + if er != nil { + err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er) + return 1 + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid) + if err != nil { + return 1 + } + } rubyversion := "2.5.7" if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) { logger.Print("ruby " + rubyversion + " already installed") } else { err = runBash(` -mkdir -p /var/lib/arvados/tmp tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+` trap "rm -r ${tmp}" ERR wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf - @@ -189,7 +204,9 @@ cd ${tmp} ./configure --disable-install-doc --prefix /var/lib/arvados make -j8 make install -/var/lib/arvados/bin/gem install bundler +/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc +# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html +/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc rm -r ${tmp} `, stdout, stderr) if err != nil { @@ -262,7 +279,6 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ } else { err = runBash(` G=`+gradleversion+` -mkdir -p /var/lib/arvados/tmp zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip trap "rm ${zip}" ERR wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip @@ -414,8 +430,7 @@ rm ${zip} for _, cmdline := range [][]string{ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"}, {"touch", "log/production.log"}, - // {"chown", "-R", "root:root", "."}, - {"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, + {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"}, @@ -426,6 +441,7 @@ rm ${zip} cmd.Dir = "/var/lib/arvados/" + dstdir cmd.Stdout = stdout cmd.Stderr = stderr + fmt.Fprintf(stderr, "... %s\n", cmd.Args) err = cmd.Run() if err != nil { return 1 @@ -569,6 +585,7 @@ func prodpkgs(osv osversion) []string { "make", "nginx", "python", + "sudo", } if osv.Debian || osv.Ubuntu { if osv.Debian && osv.Major == 8 { diff --git a/lib/install/init.go b/lib/install/init.go new file mode 100644 index 0000000000..6d4f197576 --- /dev/null +++ b/lib/install/init.go @@ -0,0 +1,265 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package install + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "flag" + "fmt" + "io" + "os" + "os/exec" + "os/user" + "regexp" + "strconv" + "text/template" + + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/lib/pq" +) + +var InitCommand cmd.Handler = &initCommand{} + +type initCommand struct { + ClusterID string + Domain string + PostgreSQLPassword string +} + +func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + logger := ctxlog.New(stderr, "text", "info") + ctx := ctxlog.Context(context.Background(), logger) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var err error + defer func() { + if err != nil { + logger.WithError(err).Info("exiting") + } + }() + + hostname, err := os.Hostname() + if err != nil { + err = fmt.Errorf("Hostname(): %w", err) + return 1 + } + + flags := flag.NewFlagSet(prog, flag.ContinueOnError) + flags.SetOutput(stderr) + versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") + flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster") + flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com") + err = flags.Parse(args) + if err == flag.ErrHelp { + err = nil + return 0 + } else if err != nil { + return 2 + } else if *versionFlag { + return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr) + } else if len(flags.Args()) > 0 { + err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) + return 2 + } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) { + err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID) + return 1 + } + + wwwuser, err := user.Lookup("www-data") + if err != nil { + err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err) + return 1 + } + wwwgid, err := strconv.Atoi(wwwuser.Gid) + if err != nil { + return 1 + } + initcmd.PostgreSQLPassword = initcmd.RandomHex(32) + + err = os.Mkdir("/var/lib/arvados/keep", 0600) + if err != nil && !os.IsExist(err) { + err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err) + return 1 + } + fmt.Fprintln(stderr, "created /var/lib/arvados/keep") + + err = os.Mkdir("/etc/arvados", 0750) + if err != nil && !os.IsExist(err) { + err = fmt.Errorf("mkdir /etc/arvados: %w", err) + return 1 + } + err = os.Chown("/etc/arvados", 0, wwwgid) + f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + if err != nil { + err = fmt.Errorf("open /etc/arvados/config.yml: %w", err) + return 1 + } + tmpl, err := template.New("config").Parse(`Clusters: + {{.ClusterID}}: + Services: + Controller: + InternalURLs: + "http://0.0.0.0:8003/": {} + ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }} + RailsAPI: + InternalURLs: + "http://0.0.0.0:8004/": {} + Websocket: + InternalURLs: + "http://0.0.0.0:8005/": {} + ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }} + Keepbalance: + InternalURLs: + "http://0.0.0.0:9005/": {} + GitHTTP: + InternalURLs: + "http://0.0.0.0:9001/": {} + ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }} + DispatchCloud: + InternalURLs: + "http://0.0.0.0:9006/": {} + Keepproxy: + InternalURLs: + "http://0.0.0.0:25108/": {} + ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }} + WebDAV: + InternalURLs: + "http://0.0.0.0:9002/": {} + ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }} + WebDAVDownload: + InternalURLs: + "http://0.0.0.0:8004/": {} + ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }} + Keepstore: + InternalURLs: + "http://0.0.0.0:25107/": {} + Composer: + ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }} + Workbench1: + InternalURLs: + "http://0.0.0.0:8001/": {} + ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }} + Workbench2: + InternalURLs: + "http://0.0.0.0:8002/": {} + ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }} + Health: + InternalURLs: + "http://0.0.0.0:9007/": {} + API: + RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}} + Collections: + BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}} + Containers: + DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}} + ManagementToken: {{printf "%q" ( .RandomHex 50 )}} + PostgreSQL: + Connection: + dbname: arvados_production + host: localhost + user: arvados + password: {{printf "%q" .PostgreSQLPassword}} + SystemRootToken: {{printf "%q" ( .RandomHex 50 )}} + Volumes: + {{.ClusterID}}-nyw5e-000000000000000: + Driver: Directory + DriverParameters: + Root: /var/lib/arvados/keep + Replication: 2 +`) + if err != nil { + return 1 + } + err = tmpl.Execute(f, initcmd) + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err) + return 1 + } + err = f.Close() + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err) + return 1 + } + fmt.Fprintln(stderr, "created /etc/arvados/config.yml") + + ldr := config.NewLoader(nil, logger) + ldr.SkipLegacy = true + cfg, err := ldr.Load() + if err != nil { + err = fmt.Errorf("/etc/arvados/config.yml: %w", err) + return 1 + } + cluster, err := cfg.GetCluster("") + if err != nil { + return 1 + } + + err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr) + if err != nil { + return 1 + } + + cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup") + cmd.Dir = "/var/lib/arvados/railsapi" + cmd.Stdout = stderr + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + err = fmt.Errorf("rake db:setup: %w", err) + return 1 + } + fmt.Fprintln(stderr, "initialized database") + + return 0 +} + +func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) { + privkey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + err = privkey.Validate() + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privkey), + })), nil +} + +func (initcmd *initCommand) RandomHex(chars int) string { + b := make([]byte, chars/2) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", b) +} + +func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error { + for _, sql := range []string{ + `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]), + `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`, + `CREATE EXTENSION IF NOT EXISTS pg_trgm`, + } { + cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql) + cmd.Stdout = stderr + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("error setting up arvados user/database: %w", err) + } + } + return nil +} -- 2.30.2