X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f159610ed02196b33fa30e1c3bba45fa947d48a9..a402ce84c38d03c4364b29cc28acf1bd691d49b2:/lib/install/deps.go diff --git a/lib/install/deps.go b/lib/install/deps.go index fb2de8a72d..dc8ec39f6f 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -17,6 +17,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "strconv" "strings" "syscall" @@ -29,18 +30,18 @@ import ( var Command cmd.Handler = &installCommand{} -const goversion = "1.17.7" +const goversion = "1.20.6" const ( - rubyversion = "2.7.5" + rubyversion = "2.7.7" bundlerversion = "2.2.19" - singularityversion = "3.9.9" + singularityversion = "3.10.4" pjsversion = "1.9.8" geckoversion = "0.24.0" gradleversion = "5.3.1" - nodejsversion = "v12.22.11" + nodejsversion = "v12.22.12" devtestDatabasePassword = "insecure_arvados_test" - workbench2version = "2454ac35292a79594c32a80430740317ed5005cf" + workbench2version = "9a62117dbe56bdfa42489415eb6696638c2bb336" // 2.6.3 ) //go:embed arvados.service @@ -154,16 +155,16 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "default-jre-headless", "gettext", "libattr1-dev", - "libcrypt-ssleay-perl", "libfuse-dev", + "libgbm1", // cypress / workbench2 tests + "libglib2.0-dev", // common, used by singularity "libgnutls28-dev", - "libjson-perl", "libpam-dev", "libpcre3-dev", "libpq-dev", "libreadline-dev", + "libseccomp-dev", // singularity "libssl-dev", - "libwww-perl", "libxml2-dev", "libxslt1-dev", "linkchecker", @@ -189,6 +190,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "uuid-dev", "wget", "xvfb", + "zlib1g-dev", // services/api ) if test { if osv.Debian && osv.Major <= 10 { @@ -203,11 +205,13 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read } switch { case osv.Debian && osv.Major >= 11: - pkgs = append(pkgs, "libcurl4", "perl-modules-5.32") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") case osv.Debian && osv.Major >= 10: - pkgs = append(pkgs, "libcurl4", "perl-modules") - default: - pkgs = append(pkgs, "libcurl3", "perl-modules") + pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev") + case osv.Debian || osv.Ubuntu: + pkgs = append(pkgs, "g++", "libcurl3", "libcurl3-openssl-dev") + case osv.Centos: + pkgs = append(pkgs, "gcc", "gcc-c++", "libcurl-devel", "postgresql-devel") } cmd := exec.CommandContext(ctx, "apt-get") if inst.EatMyData { @@ -358,7 +362,7 @@ S=`+singularityversion+` tmp=/var/lib/arvados/tmp/singularity trap "rm -r ${tmp}" ERR EXIT cd /var/lib/arvados/tmp -git clone https://github.com/sylabs/singularity +git clone --recurse-submodules https://github.com/sylabs/singularity cd singularity git checkout v${S} ./mconfig --prefix=/var/lib/arvados @@ -505,6 +509,7 @@ setcap "cap_sys_admin+pei cap_sys_chroot+pei" /var/lib/arvados/bin/nsenter } else { err = inst.runBash(` NJS=`+nodejsversion+` +rm -rf /var/lib/arvados/node-*-linux-x64 wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf - ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/ `, stdout, stderr) @@ -586,6 +591,33 @@ yarn install return 1 } + // Install python SDK and arv-mount in + // /var/lib/arvados/lib/python. + // + // setup.py writes a file in the source directory in + // order to include the version number in the package + // itself. We don't want to write to the source tree + // (in "arvados-package" context it's mounted + // readonly) so we run setup.py in a temporary copy of + // the source dir. + if err = inst.runBash(` +v=/var/lib/arvados/lib/python +tmp=/var/lib/arvados/tmp/python +python3 -m venv "$v" +. "$v/bin/activate" +pip3 install --no-cache-dir 'setuptools>=18.5' 'pip>=7' +export ARVADOS_BUILDING_VERSION="`+inst.PackageVersion+`" +for src in "`+inst.SourcePath+`/sdk/python" "`+inst.SourcePath+`/services/fuse"; do + rsync -a --delete-after "$src/" "$tmp/" + cd "$tmp" + python3 setup.py install + cd .. + rm -rf "$tmp" +done +`, stdout, stderr); err != nil { + return 1 + } + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ for dstdir, srcdir := range map[string]string{ "railsapi": "services/api", @@ -617,7 +649,7 @@ yarn install {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "deployment", "true"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "path", "/var/www/.gem"}, {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "without", "development test diagnostics performance"}, - {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", "8"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", fmt.Sprintf("%d", runtime.NumCPU())}, {"chown", "www-data:www-data", ".", "public/assets"}, // {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"}, @@ -677,35 +709,56 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/ if err != nil { return 1 } - // This is equivalent to "systemd enable", but does - // not depend on the systemctl program being - // available. - symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" - err = os.Remove(symlink) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return 1 - } - err = os.Symlink("/lib/systemd/system/arvados.service", symlink) - if err != nil { - return 1 + if prod { + // (fpm will do this for us in the pkg case) + // This is equivalent to "systemd enable", but + // does not depend on the systemctl program + // being available: + symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service" + err = os.Remove(symlink) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 1 + } + err = os.Symlink("/lib/systemd/system/arvados.service", symlink) + if err != nil { + return 1 + } } - // Symlink user-facing programs /usr/bin/x -> - // /var/lib/arvados/bin/x + // Add symlinks in /usr/bin for user-facing programs for _, srcdst := range [][]string{ - {"arvados-client", "arvados-client"}, - {"arvados-client", "arv"}, - {"arvados-server", "arvados-server"}, - {"arv", "arv-ruby"}, - {"arv-tag", "arv-tag"}, + // go + {"bin/arvados-client"}, + {"bin/arvados-client", "arv"}, + {"bin/arvados-server"}, + // sdk/cli + {"bin/arv", "arv-ruby"}, + {"bin/arv-tag"}, + // sdk/python + {"lib/python/bin/arv-copy"}, + {"lib/python/bin/arv-federation-migrate"}, + {"lib/python/bin/arv-get"}, + {"lib/python/bin/arv-keepdocker"}, + {"lib/python/bin/arv-ls"}, + {"lib/python/bin/arv-migrate-docker19"}, + {"lib/python/bin/arv-normalize"}, + {"lib/python/bin/arv-put"}, + {"lib/python/bin/arv-ws"}, + // services/fuse + {"lib/python/bin/arv-mount"}, } { - src := srcdst[0] - dst := srcdst[1] - err = os.Remove("/usr/bin/" + dst) + src := "/var/lib/arvados/" + srcdst[0] + if _, err = os.Stat(src); err != nil { + return 1 + } + dst := srcdst[len(srcdst)-1] + _, dst = filepath.Split(dst) + dst = "/usr/bin/" + dst + err = os.Remove(dst) if err != nil && !errors.Is(err, os.ErrNotExist) { return 1 } - err = os.Symlink("/var/lib/arvados/bin/"+src, "/usr/bin/"+dst) + err = os.Symlink(src, dst) if err != nil { return 1 } @@ -808,7 +861,7 @@ func prodpkgs(osv osversion) []string { "libcurl3-gnutls", "libxslt1.1", "nginx", - "python", + "python3", "sudo", } if osv.Debian || osv.Ubuntu { @@ -818,21 +871,12 @@ func prodpkgs(osv osversion) []string { pkgs = append(pkgs, "python3-distutils") // sdk/cwl } return append(pkgs, - "g++", - "libcurl4-openssl-dev", // services/api - "libpq-dev", - "libpython2.7", // services/fuse "mime-support", // keep-web - "zlib1g-dev", // services/api ) } else if osv.Centos { return append(pkgs, "fuse-libs", // services/fuse - "gcc", - "gcc-c++", - "libcurl-devel", // services/api - "mailcap", // keep-web - "postgresql-devel", // services/api + "mailcap", // keep-web ) } else { panic("os version not supported")