16306: Merge branch 'master'

[arvados.git] / lib / install / deps.go

diff --git a/lib/install/deps.go b/lib/install/deps.go
index 7b22f45fdcbd884208940cd4a574a7344f6be1d7..cc9595db64f5562b641c665d9463480430364199 100644 (file)
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -120,10 +120,13 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 
        if dev || test || pkg {
                pkgs = append(pkgs,
+                       "automake",
+                       "bison",
                        "bsdmainutils",
                        "build-essential",
                        "cadaver",
-                       "cython",
+                       "curl",
+                       "cython3",
                        "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
                        "default-jdk-headless",
                        "default-jre-headless",
@@ -136,7 +139,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
                        "libjson-perl",
                        "libpam-dev",
                        "libpcre3-dev",
-                       "libpython2.7-dev",
+                       "libpq-dev",
                        "libreadline-dev",
                        "libssl-dev",
                        "libwww-perl",
@@ -144,6 +147,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
                        "libxslt1-dev",
                        "linkchecker",
                        "lsof",
+                       "make",
                        "net-tools",
                        "pandoc",
                        "perl-modules",
@@ -151,11 +155,16 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
                        "postgresql",
                        "postgresql-contrib",
                        "python3-dev",
-                       "python-epydoc",
+                       "python3-venv",
+                       "python3-virtualenv",
                        "r-base",
                        "r-cran-testthat",
+                       "r-cran-devtools",
+                       "r-cran-knitr",
+                       "r-cran-markdown",
+                       "r-cran-roxygen2",
+                       "r-cran-xml",
                        "sudo",
-                       "virtualenv",
                        "wget",
                        "xvfb",
                )
@@ -201,7 +210,7 @@ tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
 trap "rm -r ${tmp}" ERR
 wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
 cd ${tmp}
-./configure --disable-install-doc --prefix /var/lib/arvados
+./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados
 make -j8
 make install
 /var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
@@ -313,10 +322,10 @@ rm ${zip}
                        DataDirectory string
                        LogFile       string
                }
-               if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+               if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
                        err = fmt.Errorf("pg_lsclusters: %s", err2)
                        return 1
-               } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+               } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
                        logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
                } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
                        err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
@@ -412,14 +421,20 @@ rm ${zip}
                }
        }
 
-       if pkg {
+       if prod || pkg {
                // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
                for dstdir, srcdir := range map[string]string{
                        "railsapi":   "services/api",
                        "workbench1": "apps/workbench",
                } {
                        fmt.Fprintf(stderr, "building %s...\n", srcdir)
-                       cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+                       cmd := exec.Command("rsync",
+                               "-a", "--no-owner", "--delete-after", "--delete-excluded",
+                               "--exclude", "/coverage",
+                               "--exclude", "/log",
+                               "--exclude", "/tmp",
+                               "--exclude", "/vendor",
+                               "./", "/var/lib/arvados/"+dstdir+"/")
                        cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
                        cmd.Stdout = stdout
                        cmd.Stderr = stderr
@@ -428,10 +443,10 @@ rm ${zip}
                                return 1
                        }
                        for _, cmdline := range [][]string{
-                               {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+                               {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"},
                                {"touch", "log/production.log"},
-                               {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
-                               {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+                               {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+                               {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
                                {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
                                {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
                                {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
@@ -580,8 +595,6 @@ func runBash(script string, stdout, stderr io.Writer) error {
 
 func prodpkgs(osv osversion) []string {
        pkgs := []string{
-               "automake",
-               "bison",
                "ca-certificates",
                "curl",
                "fuse",
@@ -591,7 +604,6 @@ func prodpkgs(osv osversion) []string {
                "haveged",
                "libcurl3-gnutls",
                "libxslt1.1",
-               "make",
                "nginx",
                "python",
                "sudo",