X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ba1098916779eb5da4a47463dd7f38cfcea133fc..09cbdc3074b3f1e69c9c537875146f6da0a6ed8f:/lib/install/deps.go diff --git a/lib/install/deps.go b/lib/install/deps.go index af3cc656c1..1b4bf7266d 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -17,6 +17,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "strconv" "strings" "syscall" @@ -29,18 +30,18 @@ import ( var Command cmd.Handler = &installCommand{} -const goversion = "1.17.7" +const goversion = "1.18.8" const ( - rubyversion = "2.7.5" + rubyversion = "2.7.6" bundlerversion = "2.2.19" - singularityversion = "3.7.4" + singularityversion = "3.9.9" pjsversion = "1.9.8" geckoversion = "0.24.0" gradleversion = "5.3.1" - nodejsversion = "v12.22.11" + nodejsversion = "v12.22.12" devtestDatabasePassword = "insecure_arvados_test" - workbench2version = "5e020488f67b5bc919796e0dc8b0b9f3b3ff23b0" + workbench2version = "e30e54d674c95ee15e296c71e471c1555bdc5a38" // 2.4.3 ) //go:embed arvados.service @@ -138,7 +139,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read pkgs = append(pkgs, "dpkg-dev", "eatmydata", // install it for later steps, even if we're not using it now - "rsync", ) } @@ -155,16 +155,14 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "default-jre-headless", "gettext", "libattr1-dev", - "libcrypt-ssleay-perl", "libfuse-dev", + "libgbm1", // cypress / workbench2 tests "libgnutls28-dev", - "libjson-perl", "libpam-dev", "libpcre3-dev", "libpq-dev", "libreadline-dev", "libssl-dev", - "libwww-perl", "libxml2-dev", "libxslt1-dev", "linkchecker", @@ -185,10 +183,12 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "r-cran-markdown", "r-cran-roxygen2", "r-cran-xml", + "rsync", "sudo", "uuid-dev", "wget", "xvfb", + "zlib1g-dev", // services/api ) if test { if osv.Debian && osv.Major <= 10 { @@ -199,14 +199,17 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } if dev || test { pkgs = append(pkgs, "squashfs-tools") // for singularity + pkgs = append(pkgs, "gnupg") // for docker install recipe } switch { case osv.Debian && osv.Major >= 11: - pkgs = append(pkgs, "libcurl4", "perl-modules-5.32") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") case osv.Debian && osv.Major >= 10: - pkgs = append(pkgs, "libcurl4", "perl-modules") - default: - pkgs = append(pkgs, "libcurl3", "perl-modules") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") + case osv.Debian || osv.Ubuntu: + pkgs = append(pkgs, "g++", "libcurl3", "libcurl3-openssl-dev") + case osv.Centos: + pkgs = append(pkgs, "gcc", "gcc-c++", "libcurl-devel", "postgresql-devel") } cmd := exec.CommandContext(ctx, "apt-get") if inst.EatMyData { @@ -223,6 +226,37 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } } + if dev || test { + if havedockerversion, err := exec.Command("docker", "--version").CombinedOutput(); err == nil { + logger.Printf("%s installed, assuming that version is ok", bytes.TrimSuffix(havedockerversion, []byte("\n"))) + } else if osv.Debian { + var codename string + switch osv.Major { + case 10: + codename = "buster" + case 11: + codename = "bullseye" + default: + err = fmt.Errorf("don't know how to install docker-ce for debian %d", osv.Major) + return 1 + } + err = inst.runBash(` +rm -f /usr/share/keyrings/docker-archive-keyring.gpg +curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian/ `+codename+` stable' | \ + tee /etc/apt/sources.list.d/docker.list +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get --yes --no-install-recommends install docker-ce +`, stdout, stderr) + if err != nil { + return 1 + } + } else { + err = fmt.Errorf("don't know how to install docker for osversion %v", osv) + return 1 + } + } + os.Mkdir("/var/lib/arvados", 0755) os.Mkdir("/var/lib/arvados/tmp", 0700) if prod || pkg { @@ -338,6 +372,14 @@ make -C ./builddir install } } + err = inst.runBash(` +install /usr/bin/nsenter /var/lib/arvados/bin/nsenter +setcap "cap_sys_admin+pei cap_sys_chroot+pei" /var/lib/arvados/bin/nsenter +`, stdout, stderr) + if err != nil { + return 1 + } + // The entry in /etc/locale.gen is "en_US.UTF-8"; once // it's installed, locale -a reports it as // "en_US.utf8". @@ -465,6 +507,7 @@ make -C ./builddir install } else { err = inst.runBash(` NJS=`+nodejsversion+` +rm -rf /var/lib/arvados/node-*-linux-x64 wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ `, stdout, stderr) @@ -523,14 +566,6 @@ yarn install for _, srcdir := range []string{ "cmd/arvados-client", "cmd/arvados-server", - "services/arv-git-httpd", - "services/crunch-dispatch-local", - "services/crunch-dispatch-slurm", - "services/health", - "services/keep-balance", - "services/keep-web", - "services/keepstore", - "services/ws", } { fmt.Fprintf(stderr, "building %s...\n", srcdir) cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion+" -s -w") @@ -545,19 +580,6 @@ yarn install } } - // Symlink user-facing Go programs /usr/bin/x -> - // /var/lib/arvados/bin/x - for _, prog := range []string{"arvados-client", "arvados-server"} { - err = os.Remove("/usr/bin/" + prog) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return 1 - } - err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog) - if err != nil { - return 1 - } - } - // Copy assets from source tree to /var/lib/arvados/share cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) cmd.Stdout = stdout @@ -567,6 +589,33 @@ yarn install return 1 } + // Install python SDK and arv-mount in + // /var/lib/arvados/lib/python. + // + // setup.py writes a file in the source directory in + // order to include the version number in the package + // itself. We don't want to write to the source tree + // (in "arvados-package" context it's mounted + // readonly) so we run setup.py in a temporary copy of + // the source dir. + if err = inst.runBash(` +v=/var/lib/arvados/lib/python +tmp=/var/lib/arvados/tmp/python +python3 -m venv "$v" +. "$v/bin/activate" +pip3 install --no-cache-dir 'setuptools>=18.5' 'pip>=7' +export ARVADOS_BUILDING_VERSION="`+inst.PackageVersion+`" +for src in "`+inst.SourcePath+`/sdk/python" "`+inst.SourcePath+`/services/fuse"; do + rsync -a --delete-after "$src/" "$tmp/" + cd "$tmp" + python3 setup.py install + cd .. + rm -rf "$tmp" +done +`, stdout, stderr); err != nil { + return 1 + } + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ for dstdir, srcdir := range map[string]string{ "railsapi": "services/api", @@ -595,12 +644,15 @@ yarn install {"touch", "log/production.log"}, {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion}, - {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem", "--without", "development test diagnostics performance"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "deployment", "true"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "path", "/var/www/.gem"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "without", "development test diagnostics performance"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", fmt.Sprintf("%d", runtime.NumCPU())}, {"chown", "www-data:www-data", ".", "public/assets"}, // {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"}, - {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"}, - {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"}, + {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"}, {"chown", "root:root", "."}, {"chown", "-R", "root:root", "public/assets", "vendor"}, @@ -643,21 +695,71 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/ return 1 } + // Install arvados-cli gem (binaries go in + // /var/lib/arvados/bin) + if err = inst.runBash(` +/var/lib/arvados/bin/gem install --conservative --no-document arvados-cli +`, stdout, stderr); err != nil { + return 1 + } + err = os.WriteFile("/lib/systemd/system/arvados.service", arvadosServiceFile, 0777) if err != nil { return 1 } - // This is equivalent to "systemd enable", but does - // not depend on the systemctl program being - // available. - symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" - err = os.Remove(symlink) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return 1 + if prod { + // (fpm will do this for us in the pkg case) + // This is equivalent to "systemd enable", but + // does not depend on the systemctl program + // being available: + symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" + err = os.Remove(symlink) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink("/lib/systemd/system/arvados.service", symlink) + if err != nil { + return 1 + } } - err = os.Symlink("/lib/systemd/system/arvados.service", symlink) - if err != nil { - return 1 + + // Add symlinks in /usr/bin for user-facing programs + for _, srcdst := range [][]string{ + // go + {"bin/arvados-client"}, + {"bin/arvados-client", "arv"}, + {"bin/arvados-server"}, + // sdk/cli + {"bin/arv", "arv-ruby"}, + {"bin/arv-tag"}, + // sdk/python + {"lib/python/bin/arv-copy"}, + {"lib/python/bin/arv-federation-migrate"}, + {"lib/python/bin/arv-get"}, + {"lib/python/bin/arv-keepdocker"}, + {"lib/python/bin/arv-ls"}, + {"lib/python/bin/arv-migrate-docker19"}, + {"lib/python/bin/arv-normalize"}, + {"lib/python/bin/arv-put"}, + {"lib/python/bin/arv-ws"}, + // services/fuse + {"lib/python/bin/arv-mount"}, + } { + src := "/var/lib/arvados/" + srcdst[0] + if _, err = os.Stat(src); err != nil { + return 1 + } + dst := srcdst[len(srcdst)-1] + _, dst = filepath.Split(dst) + dst = "/usr/bin/" + dst + err = os.Remove(dst) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink(src, dst) + if err != nil { + return 1 + } } } @@ -757,7 +859,7 @@ func prodpkgs(osv osversion) []string { "libcurl3-gnutls", "libxslt1.1", "nginx", - "python", + "python3", "sudo", } if osv.Debian || osv.Ubuntu { @@ -767,21 +869,12 @@ func prodpkgs(osv osversion) []string { pkgs = append(pkgs, "python3-distutils") // sdk/cwl } return append(pkgs, - "g++", - "libcurl4-openssl-dev", // services/api - "libpq-dev", - "libpython2.7", // services/fuse "mime-support", // keep-web - "zlib1g-dev", // services/api ) } else if osv.Centos { return append(pkgs, "fuse-libs", // services/fuse - "gcc", - "gcc-c++", - "libcurl-devel", // services/api - "mailcap", // keep-web - "postgresql-devel", // services/api + "mailcap", // keep-web ) } else { panic("os version not supported")