X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/af3fe384bb15c4fc5b915cdee5cd1f2a87b6301f..2c8874c0625a2126813d4df7170076f95cdc7327:/lib/install/deps.go diff --git a/lib/install/deps.go b/lib/install/deps.go index 3b904c458c..342ef03a7f 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "context" + "errors" "flag" "fmt" "io" @@ -15,13 +16,18 @@ import ( "os/exec" "strconv" "strings" + "syscall" + "time" "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/lib/pq" ) var Command cmd.Handler = installCommand{} +const devtestDatabasePassword = "insecure_arvados_test" + type installCommand struct{} func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { @@ -60,10 +66,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st case "production": prod = true default: - err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'") + err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType) return 2 } + if prod { + err = errors.New("production install is not yet implemented") + return 1 + } + osv, err := identifyOS() if err != nil { return 1 @@ -90,10 +101,13 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st "bison", "bsdmainutils", "build-essential", + "ca-certificates", "cadaver", "curl", - "cython", + "cython3", "daemontools", // lib/boot uses setuidgid to drop privileges when running as root + "default-jdk-headless", + "default-jre-headless", "fuse", "gettext", "git", @@ -113,7 +127,6 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st "libpam-dev", "libpcre3-dev", "libpq-dev", - "libpython2.7-dev", "libreadline-dev", "libssl-dev", "libwww-perl", @@ -128,13 +141,17 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st "pkg-config", "postgresql", "postgresql-contrib", - "python", "python3-dev", - "python-epydoc", + "python3-venv", + "python3-virtualenv", "r-base", "r-cran-testthat", + "r-cran-devtools", + "r-cran-knitr", + "r-cran-markdown", + "r-cran-roxygen2", + "r-cran-xml", "sudo", - "virtualenv", "wget", "xvfb", "zlib1g-dev", @@ -163,14 +180,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st } else { err = runBash(` mkdir -p /var/lib/arvados/tmp +tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+` +trap "rm -r ${tmp}" ERR wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf - -cd /var/lib/arvados/tmp/ruby-`+rubyversion+` +cd ${tmp} ./configure --disable-install-doc --prefix /var/lib/arvados make -j4 make install /var/lib/arvados/bin/gem install bundler -cd /var/lib/arvados/tmp -rm -r ruby-`+rubyversion+` +rm -r ${tmp} `, stdout, stderr) if err != nil { return 1 @@ -233,6 +251,145 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ return 1 } } + + gradleversion := "5.3.1" + if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") { + logger.Print("gradle " + gradleversion + " already installed") + } else { + err = runBash(` +G=`+gradleversion+` +mkdir -p /var/lib/arvados/tmp +zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip +trap "rm ${zip}" ERR +wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip +unzip -o -d /var/lib/arvados ${zip} +ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/ +rm ${zip} +`, stdout, stderr) + if err != nil { + return 1 + } + } + + // The entry in /etc/locale.gen is "en_US.UTF-8"; once + // it's installed, locale -a reports it as + // "en_US.utf8". + wantlocale := "en_US.UTF-8" + if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) { + logger.Print("locale " + wantlocale + " already installed") + } else { + err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr) + if err != nil { + return 1 + } + } + + var pgc struct { + Version string + Cluster string + Port int + Status string + Owner string + DataDirectory string + LogFile string + } + if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil { + err = fmt.Errorf("pg_lsclusters: %s", err2) + return 1 + } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 { + logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters)) + } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil { + err = fmt.Errorf("error parsing pg_lsclusters output: %s", err) + return 1 + } else if pgc.Status == "online" { + logger.Infof("postgresql cluster %s-%s is online", pgc.Version, pgc.Cluster) + } else { + logger.Infof("postgresql cluster %s-%s is %s; trying to start", pgc.Version, pgc.Cluster, pgc.Status) + cmd := exec.Command("pg_ctlcluster", "--foreground", pgc.Version, pgc.Cluster, "start") + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Start() + if err != nil { + return 1 + } + defer func() { + cmd.Process.Signal(syscall.SIGTERM) + logger.Info("sent SIGTERM; waiting for postgres to shut down") + cmd.Wait() + }() + err = waitPostgreSQLReady() + if err != nil { + return 1 + } + } + + if os.Getpid() == 1 { + // We are the init process (presumably in a + // docker container) so although postgresql is + // installed, it's not running, and initdb + // might never have been run. + } + + var needcoll []string + // If the en_US.UTF-8 locale wasn't installed when + // postgresql initdb ran, it needs to be added + // explicitly before we can use it in our test suite. + for _, collname := range []string{"en_US", "en_US.UTF-8"} { + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_collation WHERE collname='"+collname+"' AND collcollate IN ('en_US.UTF-8', 'en_US.utf8')") + cmd.Dir = "/" + out, err2 := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("error while checking postgresql collations: %s", err2) + return 1 + } + if strings.Contains(string(out), "1") { + logger.Infof("postgresql supports collation %s", collname) + } else { + needcoll = append(needcoll, collname) + } + } + if len(needcoll) > 0 && os.Getpid() != 1 { + // In order for the CREATE COLLATION statement + // below to work, the locale must have existed + // when PostgreSQL started up. If we're + // running as init, we must have started + // PostgreSQL ourselves after installing the + // locales. Otherwise, it might need a + // restart, so we attempt to restart it with + // systemd. + if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil { + logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted") + } else if err = waitPostgreSQLReady(); err != nil { + return 1 + } + } + for _, collname := range needcoll { + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE COLLATION \""+collname+"\" (LOCALE = \"en_US.UTF-8\")") + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Dir = "/" + err = cmd.Run() + if err != nil { + err = fmt.Errorf("error adding postgresql collation %s: %s", collname, err) + return 1 + } + } + + withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword) + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff) + cmd.Dir = "/" + if err := cmd.Run(); err == nil { + logger.Print("arvados role exists; superuser privileges added, password updated") + } else { + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff) + cmd.Dir = "/" + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } } return 0 @@ -287,11 +444,24 @@ func identifyOS() (osversion, error) { } osv.Major, err = strconv.Atoi(vstr) if err != nil { - return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os/release: %q", kv["VERSION_ID"]) + return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os-release: %q", kv["VERSION_ID"]) } return osv, nil } +func waitPostgreSQLReady() error { + for deadline := time.Now().Add(10 * time.Second); ; { + output, err := exec.Command("pg_isready").CombinedOutput() + if err == nil { + return nil + } else if time.Now().After(deadline) { + return fmt.Errorf("timed out waiting for pg_isready (%q)", output) + } else { + time.Sleep(time.Second) + } + } +} + func runBash(script string, stdout, stderr io.Writer) error { cmd := exec.Command("bash", "-") cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)