X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b0766c77a7bfce908fdf9e02cd12a0cf35c0ee4c..3aaefcb3c76ff470b475d950398d01255e87712a:/lib/install/deps.go

diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4146042bd2..cc9595db64 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -8,11 +8,14 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
+	"os/user"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -23,13 +26,17 @@ import (
 	"github.com/lib/pq"
 )
 
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
 
 const devtestDatabasePassword = "insecure_arvados_test"
 
-type installCommand struct{}
+type installCommand struct {
+	ClusterType    string
+	SourcePath     string
+	PackageVersion string
+}
 
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	logger := ctxlog.New(stderr, "text", "info")
 	ctx := ctxlog.Context(context.Background(), logger)
 	ctx, cancel := context.WithCancel(ctx)
@@ -45,7 +52,9 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
 	flags.SetOutput(stderr)
 	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-	clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+	flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+	flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+	flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -54,21 +63,31 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		return 2
 	} else if *versionFlag {
 		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+	} else if len(flags.Args()) > 0 {
+		err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+		return 2
 	}
 
-	var dev, test, prod bool
-	switch *clusterType {
+	var dev, test, prod, pkg bool
+	switch inst.ClusterType {
 	case "development":
 		dev = true
 	case "test":
 		test = true
 	case "production":
 		prod = true
+	case "package":
+		pkg = true
 	default:
-		err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
+		err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
 		return 2
 	}
 
+	if prod {
+		err = errors.New("production install is not yet implemented")
+		return 1
+	}
+
 	osv, err := identifyOS()
 	if err != nil {
 		return 1
@@ -90,68 +109,73 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 		}
 	}
 
-	if dev || test {
-		debs := []string{
+	pkgs := prodpkgs(osv)
+
+	if pkg {
+		pkgs = append(pkgs,
+			"dpkg-dev",
+			"rsync",
+		)
+	}
+
+	if dev || test || pkg {
+		pkgs = append(pkgs,
+			"automake",
 			"bison",
 			"bsdmainutils",
 			"build-essential",
 			"cadaver",
 			"curl",
-			"cython",
+			"cython3",
 			"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
-			"fuse",
+			"default-jdk-headless",
+			"default-jre-headless",
 			"gettext",
-			"git",
-			"gitolite3",
-			"graphviz",
-			"haveged",
 			"iceweasel",
 			"libattr1-dev",
 			"libcrypt-ssleay-perl",
-			"libcrypt-ssleay-perl",
-			"libcurl3-gnutls",
-			"libcurl4-openssl-dev",
 			"libfuse-dev",
 			"libgnutls28-dev",
 			"libjson-perl",
-			"libjson-perl",
 			"libpam-dev",
 			"libpcre3-dev",
 			"libpq-dev",
-			"libpython2.7-dev",
 			"libreadline-dev",
 			"libssl-dev",
 			"libwww-perl",
 			"libxml2-dev",
-			"libxslt1.1",
+			"libxslt1-dev",
 			"linkchecker",
 			"lsof",
+			"make",
 			"net-tools",
-			"nginx",
 			"pandoc",
 			"perl-modules",
 			"pkg-config",
 			"postgresql",
 			"postgresql-contrib",
-			"python",
 			"python3-dev",
-			"python-epydoc",
+			"python3-venv",
+			"python3-virtualenv",
 			"r-base",
 			"r-cran-testthat",
+			"r-cran-devtools",
+			"r-cran-knitr",
+			"r-cran-markdown",
+			"r-cran-roxygen2",
+			"r-cran-xml",
 			"sudo",
-			"virtualenv",
 			"wget",
 			"xvfb",
-			"zlib1g-dev",
-		}
+		)
 		switch {
 		case osv.Debian && osv.Major >= 10:
-			debs = append(debs, "libcurl4")
+			pkgs = append(pkgs, "libcurl4")
 		default:
-			debs = append(debs, "libcurl3")
+			pkgs = append(pkgs, "libcurl3")
 		}
 		cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
-		cmd.Args = append(cmd.Args, debs...)
+		cmd.Args = append(cmd.Args, pkgs...)
 		cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr
@@ -162,20 +186,37 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
 	}
 
 	os.Mkdir("/var/lib/arvados", 0755)
+	os.Mkdir("/var/lib/arvados/tmp", 0700)
+	if prod || pkg {
+		os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+		u, er := user.Lookup("www-data")
+		if er != nil {
+			err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+			return 1
+		}
+		uid, _ := strconv.Atoi(u.Uid)
+		gid, _ := strconv.Atoi(u.Gid)
+		err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+		if err != nil {
+			return 1
+		}
+	}
 	rubyversion := "2.5.7"
 	if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
 		logger.Print("ruby " + rubyversion + " already installed")
 	} else {
 		err = runBash(`
-mkdir -p /var/lib/arvados/tmp
+tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
+trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd /var/lib/arvados/tmp/ruby-`+rubyversion+`
-./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+cd ${tmp}
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
+make -j8
 make install
-/var/lib/arvados/bin/gem install bundler
-cd /var/lib/arvados/tmp
-rm -r ruby-`+rubyversion+`
+/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
+# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
+rm -r ${tmp}
 `, stdout, stderr)
 		if err != nil {
 			return 1
@@ -196,7 +237,9 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
 				return 1
 			}
 		}
+	}
 
+	if !prod && !pkg {
 		pjsversion := "1.9.8"
 		if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
 			logger.Print("phantomjs " + pjsversion + " already installed")
@@ -239,8 +282,29 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 			}
 		}
 
+		gradleversion := "5.3.1"
+		if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
+			logger.Print("gradle " + gradleversion + " already installed")
+		} else {
+			err = runBash(`
+G=`+gradleversion+`
+zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
+trap "rm ${zip}" ERR
+wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
+unzip -o -d /var/lib/arvados ${zip}
+ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
+rm ${zip}
+`, stdout, stderr)
+			if err != nil {
+				return 1
+			}
+		}
+
+		// The entry in /etc/locale.gen is "en_US.UTF-8"; once
+		// it's installed, locale -a reports it as
+		// "en_US.utf8".
 		wantlocale := "en_US.UTF-8"
-		if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
+		if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
 			logger.Print("locale " + wantlocale + " already installed")
 		} else {
 			err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
@@ -258,10 +322,10 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 			DataDirectory string
 			LogFile       string
 		}
-		if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+		if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
 			err = fmt.Errorf("pg_lsclusters: %s", err2)
 			return 1
-		} else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+		} else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
 			logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
 		} else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
 			err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
@@ -279,19 +343,12 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 			}
 			defer func() {
 				cmd.Process.Signal(syscall.SIGTERM)
-				logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+				logger.Info("sent SIGTERM; waiting for postgres to shut down")
 				cmd.Wait()
 			}()
-			for deadline := time.Now().Add(10 * time.Second); ; {
-				output, err2 := exec.Command("pg_isready").CombinedOutput()
-				if err2 == nil {
-					break
-				} else if time.Now().After(deadline) {
-					err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
-					return 1
-				} else {
-					time.Sleep(time.Second)
-				}
+			err = waitPostgreSQLReady()
+			if err != nil {
+				return 1
 			}
 		}
 
@@ -302,11 +359,59 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 			// might never have been run.
 		}
 
-		withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
-		if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+		var needcoll []string
+		// If the en_US.UTF-8 locale wasn't installed when
+		// postgresql initdb ran, it needs to be added
+		// explicitly before we can use it in our test suite.
+		for _, collname := range []string{"en_US", "en_US.UTF-8"} {
+			cmd := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_collation WHERE collname='"+collname+"' AND collcollate IN ('en_US.UTF-8', 'en_US.utf8')")
+			cmd.Dir = "/"
+			out, err2 := cmd.CombinedOutput()
+			if err != nil {
+				err = fmt.Errorf("error while checking postgresql collations: %s", err2)
+				return 1
+			}
+			if strings.Contains(string(out), "1") {
+				logger.Infof("postgresql supports collation %s", collname)
+			} else {
+				needcoll = append(needcoll, collname)
+			}
+		}
+		if len(needcoll) > 0 && os.Getpid() != 1 {
+			// In order for the CREATE COLLATION statement
+			// below to work, the locale must have existed
+			// when PostgreSQL started up. If we're
+			// running as init, we must have started
+			// PostgreSQL ourselves after installing the
+			// locales. Otherwise, it might need a
+			// restart, so we attempt to restart it with
+			// systemd.
+			if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+				logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
+			} else if err = waitPostgreSQLReady(); err != nil {
+				return 1
+			}
+		}
+		for _, collname := range needcoll {
+			cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE COLLATION \""+collname+"\" (LOCALE = \"en_US.UTF-8\")")
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			cmd.Dir = "/"
+			err = cmd.Run()
+			if err != nil {
+				err = fmt.Errorf("error adding postgresql collation %s: %s", collname, err)
+				return 1
+			}
+		}
+
+		withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+		cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff)
+		cmd.Dir = "/"
+		if err := cmd.Run(); err == nil {
 			logger.Print("arvados role exists; superuser privileges added, password updated")
 		} else {
 			cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+			cmd.Dir = "/"
 			cmd.Stdout = stdout
 			cmd.Stderr = stderr
 			err = cmd.Run()
@@ -316,12 +421,104 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
 		}
 	}
 
+	if prod || pkg {
+		// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+		for dstdir, srcdir := range map[string]string{
+			"railsapi":   "services/api",
+			"workbench1": "apps/workbench",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("rsync",
+				"-a", "--no-owner", "--delete-after", "--delete-excluded",
+				"--exclude", "/coverage",
+				"--exclude", "/log",
+				"--exclude", "/tmp",
+				"--exclude", "/vendor",
+				"./", "/var/lib/arvados/"+dstdir+"/")
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+			for _, cmdline := range [][]string{
+				{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
+				{"touch", "log/production.log"},
+				{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+				{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+			} {
+				cmd = exec.Command(cmdline[0], cmdline[1:]...)
+				cmd.Env = append([]string{}, os.Environ()...)
+				cmd.Dir = "/var/lib/arvados/" + dstdir
+				cmd.Stdout = stdout
+				cmd.Stderr = stderr
+				fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+				err = cmd.Run()
+				if err != nil {
+					return 1
+				}
+			}
+			cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+			cmd.Dir = "/var/lib/arvados/" + dstdir
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+				// Exit code 2 indicates there were warnings (like
+				// "other passenger installations have been detected",
+				// which we can't expect to avoid) but no errors.
+				// Other non-zero exit codes (1, 9) indicate errors.
+				return 1
+			}
+		}
+
+		// Install Go programs to /var/lib/arvados/bin/
+		for _, srcdir := range []string{
+			"cmd/arvados-client",
+			"cmd/arvados-server",
+			"services/arv-git-httpd",
+			"services/crunch-dispatch-local",
+			"services/crunch-dispatch-slurm",
+			"services/health",
+			"services/keep-balance",
+			"services/keep-web",
+			"services/keepproxy",
+			"services/keepstore",
+			"services/ws",
+		} {
+			fmt.Fprintf(stderr, "building %s...\n", srcdir)
+			cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+			cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+			cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			err = cmd.Run()
+			if err != nil {
+				return 1
+			}
+		}
+
+		// Copy assets from source tree to /var/lib/arvados/share
+		cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+		err = cmd.Run()
+		if err != nil {
+			return 1
+		}
+	}
+
 	return 0
 }
 
 type osversion struct {
 	Debian bool
 	Ubuntu bool
+	Centos bool
 	Major  int
 }
 
@@ -359,6 +556,8 @@ func identifyOS() (osversion, error) {
 		osv.Ubuntu = true
 	case "debian":
 		osv.Debian = true
+	case "centos":
+		osv.Centos = true
 	default:
 		return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
 	}
@@ -368,11 +567,24 @@ func identifyOS() (osversion, error) {
 	}
 	osv.Major, err = strconv.Atoi(vstr)
 	if err != nil {
-		return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os/release: %q", kv["VERSION_ID"])
+		return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os-release: %q", kv["VERSION_ID"])
 	}
 	return osv, nil
 }
 
+func waitPostgreSQLReady() error {
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		output, err := exec.Command("pg_isready").CombinedOutput()
+		if err == nil {
+			return nil
+		} else if time.Now().After(deadline) {
+			return fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+		} else {
+			time.Sleep(time.Second)
+		}
+	}
+}
+
 func runBash(script string, stdout, stderr io.Writer) error {
 	cmd := exec.Command("bash", "-")
 	cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
@@ -380,3 +592,54 @@ func runBash(script string, stdout, stderr io.Writer) error {
 	cmd.Stderr = stderr
 	return cmd.Run()
 }
+
+func prodpkgs(osv osversion) []string {
+	pkgs := []string{
+		"ca-certificates",
+		"curl",
+		"fuse",
+		"git",
+		"gitolite3",
+		"graphviz",
+		"haveged",
+		"libcurl3-gnutls",
+		"libxslt1.1",
+		"nginx",
+		"python",
+		"sudo",
+	}
+	if osv.Debian || osv.Ubuntu {
+		if osv.Debian && osv.Major == 8 {
+			pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+		} else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+			pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+		}
+		return append(pkgs,
+			"g++",
+			"libcurl4-openssl-dev", // services/api
+			"libpq-dev",
+			"libpython2.7", // services/fuse
+			"mime-support", // keep-web
+			"zlib1g-dev",   // services/api
+		)
+	} else if osv.Centos {
+		return append(pkgs,
+			"fuse-libs", // services/fuse
+			"gcc",
+			"gcc-c++",
+			"libcurl-devel",    // services/api
+			"mailcap",          // keep-web
+			"postgresql-devel", // services/api
+		)
+	} else {
+		panic("os version not supported")
+	}
+}
+
+func ProductionDependencies() ([]string, error) {
+	osv, err := identifyOS()
+	if err != nil {
+		return nil, err
+	}
+	return prodpkgs(osv), nil
+}