X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/86660414472d4ff0d8267f9845a753497bd41692..09cbdc3074b3f1e69c9c537875146f6da0a6ed8f:/lib/install/deps.go diff --git a/lib/install/deps.go b/lib/install/deps.go index a7c32c5052..1b4bf7266d 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -17,6 +17,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "strconv" "strings" "syscall" @@ -29,18 +30,18 @@ import ( var Command cmd.Handler = &installCommand{} -const goversion = "1.17.7" +const goversion = "1.18.8" const ( - rubyversion = "2.7.5" + rubyversion = "2.7.6" bundlerversion = "2.2.19" singularityversion = "3.9.9" pjsversion = "1.9.8" geckoversion = "0.24.0" gradleversion = "5.3.1" - nodejsversion = "v12.22.11" + nodejsversion = "v12.22.12" devtestDatabasePassword = "insecure_arvados_test" - workbench2version = "5e020488f67b5bc919796e0dc8b0b9f3b3ff23b0" + workbench2version = "e30e54d674c95ee15e296c71e471c1555bdc5a38" // 2.4.3 ) //go:embed arvados.service @@ -154,16 +155,14 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "default-jre-headless", "gettext", "libattr1-dev", - "libcrypt-ssleay-perl", "libfuse-dev", + "libgbm1", // cypress / workbench2 tests "libgnutls28-dev", - "libjson-perl", "libpam-dev", "libpcre3-dev", "libpq-dev", "libreadline-dev", "libssl-dev", - "libwww-perl", "libxml2-dev", "libxslt1-dev", "linkchecker", @@ -189,6 +188,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "uuid-dev", "wget", "xvfb", + "zlib1g-dev", // services/api ) if test { if osv.Debian && osv.Major <= 10 { @@ -203,11 +203,13 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } switch { case osv.Debian && osv.Major >= 11: - pkgs = append(pkgs, "libcurl4", "perl-modules-5.32") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") case osv.Debian && osv.Major >= 10: - pkgs = append(pkgs, "libcurl4", "perl-modules") - default: - pkgs = append(pkgs, "libcurl3", "perl-modules") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") + case osv.Debian || osv.Ubuntu: + pkgs = append(pkgs, "g++", "libcurl3", "libcurl3-openssl-dev") + case osv.Centos: + pkgs = append(pkgs, "gcc", "gcc-c++", "libcurl-devel", "postgresql-devel") } cmd := exec.CommandContext(ctx, "apt-get") if inst.EatMyData { @@ -505,6 +507,7 @@ setcap "cap_sys_admin+pei cap_sys_chroot+pei" /var/lib/arvados/bin/nsenter } else { err = inst.runBash(` NJS=`+nodejsversion+` +rm -rf /var/lib/arvados/node-*-linux-x64 wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ `, stdout, stderr) @@ -563,8 +566,6 @@ yarn install for _, srcdir := range []string{ "cmd/arvados-client", "cmd/arvados-server", - "services/crunch-dispatch-local", - "services/crunch-dispatch-slurm", } { fmt.Fprintf(stderr, "building %s...\n", srcdir) cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion+" -s -w") @@ -579,19 +580,6 @@ yarn install } } - // Symlink user-facing Go programs /usr/bin/x -> - // /var/lib/arvados/bin/x - for _, prog := range []string{"arvados-client", "arvados-server"} { - err = os.Remove("/usr/bin/" + prog) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return 1 - } - err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog) - if err != nil { - return 1 - } - } - // Copy assets from source tree to /var/lib/arvados/share cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) cmd.Stdout = stdout @@ -601,6 +589,33 @@ yarn install return 1 } + // Install python SDK and arv-mount in + // /var/lib/arvados/lib/python. + // + // setup.py writes a file in the source directory in + // order to include the version number in the package + // itself. We don't want to write to the source tree + // (in "arvados-package" context it's mounted + // readonly) so we run setup.py in a temporary copy of + // the source dir. + if err = inst.runBash(` +v=/var/lib/arvados/lib/python +tmp=/var/lib/arvados/tmp/python +python3 -m venv "$v" +. "$v/bin/activate" +pip3 install --no-cache-dir 'setuptools>=18.5' 'pip>=7' +export ARVADOS_BUILDING_VERSION="`+inst.PackageVersion+`" +for src in "`+inst.SourcePath+`/sdk/python" "`+inst.SourcePath+`/services/fuse"; do + rsync -a --delete-after "$src/" "$tmp/" + cd "$tmp" + python3 setup.py install + cd .. + rm -rf "$tmp" +done +`, stdout, stderr); err != nil { + return 1 + } + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ for dstdir, srcdir := range map[string]string{ "railsapi": "services/api", @@ -629,12 +644,15 @@ yarn install {"touch", "log/production.log"}, {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion}, - {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem", "--without", "development test diagnostics performance"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "deployment", "true"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "path", "/var/www/.gem"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "without", "development test diagnostics performance"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", fmt.Sprintf("%d", runtime.NumCPU())}, {"chown", "www-data:www-data", ".", "public/assets"}, // {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"}, - {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"}, - {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"}, {"chown", "root:root", "."}, {"chown", "-R", "root:root", "public/assets", "vendor"}, @@ -677,21 +695,71 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/ return 1 } + // Install arvados-cli gem (binaries go in + // /var/lib/arvados/bin) + if err = inst.runBash(` +/var/lib/arvados/bin/gem install --conservative --no-document arvados-cli +`, stdout, stderr); err != nil { + return 1 + } + err = os.WriteFile("/lib/systemd/system/arvados.service", arvadosServiceFile, 0777) if err != nil { return 1 } - // This is equivalent to "systemd enable", but does - // not depend on the systemctl program being - // available. - symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" - err = os.Remove(symlink) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return 1 + if prod { + // (fpm will do this for us in the pkg case) + // This is equivalent to "systemd enable", but + // does not depend on the systemctl program + // being available: + symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" + err = os.Remove(symlink) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink("/lib/systemd/system/arvados.service", symlink) + if err != nil { + return 1 + } } - err = os.Symlink("/lib/systemd/system/arvados.service", symlink) - if err != nil { - return 1 + + // Add symlinks in /usr/bin for user-facing programs + for _, srcdst := range [][]string{ + // go + {"bin/arvados-client"}, + {"bin/arvados-client", "arv"}, + {"bin/arvados-server"}, + // sdk/cli + {"bin/arv", "arv-ruby"}, + {"bin/arv-tag"}, + // sdk/python + {"lib/python/bin/arv-copy"}, + {"lib/python/bin/arv-federation-migrate"}, + {"lib/python/bin/arv-get"}, + {"lib/python/bin/arv-keepdocker"}, + {"lib/python/bin/arv-ls"}, + {"lib/python/bin/arv-migrate-docker19"}, + {"lib/python/bin/arv-normalize"}, + {"lib/python/bin/arv-put"}, + {"lib/python/bin/arv-ws"}, + // services/fuse + {"lib/python/bin/arv-mount"}, + } { + src := "/var/lib/arvados/" + srcdst[0] + if _, err = os.Stat(src); err != nil { + return 1 + } + dst := srcdst[len(srcdst)-1] + _, dst = filepath.Split(dst) + dst = "/usr/bin/" + dst + err = os.Remove(dst) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink(src, dst) + if err != nil { + return 1 + } } } @@ -791,7 +859,7 @@ func prodpkgs(osv osversion) []string { "libcurl3-gnutls", "libxslt1.1", "nginx", - "python", + "python3", "sudo", } if osv.Debian || osv.Ubuntu { @@ -801,21 +869,12 @@ func prodpkgs(osv osversion) []string { pkgs = append(pkgs, "python3-distutils") // sdk/cwl } return append(pkgs, - "g++", - "libcurl4-openssl-dev", // services/api - "libpq-dev", - "libpython2.7", // services/fuse "mime-support", // keep-web - "zlib1g-dev", // services/api ) } else if osv.Centos { return append(pkgs, "fuse-libs", // services/fuse - "gcc", - "gcc-c++", - "libcurl-devel", // services/api - "mailcap", // keep-web - "postgresql-devel", // services/api + "mailcap", // keep-web ) } else { panic("os version not supported")