X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6b3cfe60beb46ba087de5093926363d8b03ab889..8afc85aabb9563da4de17b0b5f7d4fe574e9ad8d:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index a02f732cb3..8a8f5b6d24 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -74,6 +74,7 @@ doc lib/cli lib/cmd lib/crunchstat +lib/dispatchcloud services/api services/arv-git-httpd services/crunchstat @@ -108,6 +109,7 @@ sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner sdk/cwl +sdk/R tools/sync-groups tools/crunchstat-summary tools/keep-exercise @@ -132,6 +134,7 @@ VENV3DIR= PYTHONPATH= GEMHOME= PERLINSTALLBASE= +R_LIBS= short= only_install= @@ -179,8 +182,8 @@ sanity_checks() { echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \ - || fatal "Go >= 1.8 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \ + || fatal "Go >= 1.10 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -239,6 +242,18 @@ sanity_checks() { which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" + + if [[ "$NEED_SDK_R" = true ]]; then + # R SDK stuff + echo -n 'R: ' + which Rscript || fatal "No Rscript. Try: apt-get install r-base" + echo -n 'testthat: ' + Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat" + # needed for roxygen2, needed for devtools, needed for R sdk + pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev" + # needed for pkgdown, builds R SDK doc pages + which pandoc || fatal "No pandoc. Try: apt-get install pandoc" + fi } rotate_logfile() { @@ -308,6 +323,18 @@ do esac done +# R SDK installation is very slow (~360s in a clean environment) and only +# required when testing it. Skip that step if it is not needed. +NEED_SDK_R=true + +if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then + NEED_SDK_R=false +fi + +if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then + NEED_SDK_R=false +fi + start_services() { echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then @@ -367,7 +394,7 @@ if [[ -z "$temp" ]]; then fi # Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE +for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -462,80 +489,71 @@ setup_virtualenv() { local venvdest="$1"; shift if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + elif [[ -n "$short" ]]; then + return fi if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' else - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' + "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' } export PERLINSTALLBASE export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" +export R_LIBS export GOPATH -mkdir -p "$GOPATH/src/git.curoverse.com" -rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" -for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -d "$d" ]] && rmdir "$d" - [[ -h "$d" ]] && rm "$d" -done -ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ - || fatal "symlink failed" -go get -v github.com/kardianos/govendor \ - || fatal "govendor install failed" -cd "$GOPATH/src/git.curoverse.com/arvados.git" \ - || fatal -# Remove cached source dirs in workdir. Otherwise, they won't qualify -# as +missing or +external below, and we won't be able to detect that -# they're missing from vendor/vendor.json. -rm -r vendor/*/ -go get -v -d ... -"$GOPATH/bin/govendor" sync \ - || fatal "govendor sync failed" -[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ - || fatal "vendor/vendor.json has unused or missing dependencies -- try: -* govendor remove +unused -* govendor add +missing +external -" -cd "$WORKSPACE" - +( + set -e + mkdir -p "$GOPATH/src/git.curoverse.com" + rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH" + for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -d "$d" ]] && rmdir "$d" + [[ -h "$d" ]] && rm "$d" + done + ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" + go get -v github.com/kardianos/govendor + cd "$GOPATH/src/git.curoverse.com/arvados.git" + if [[ -n "$short" ]]; then + go get -v -d ... + "$GOPATH/bin/govendor" sync + else + # Remove cached source dirs in workdir. Otherwise, they will + # not qualify as +missing or +external below, and we won't be + # able to detect that they're missing from vendor/vendor.json. + rm -rf vendor/*/ + go get -v -d ... + "$GOPATH/bin/govendor" sync + [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ + || fatal "vendor/vendor.json has unused or missing dependencies -- try: + +(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) + +"; + fi +) || fatal "Go setup failed" setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. -pip freeze 2>/dev/null | egrep ^PyYAML= \ - || pip install PyYAML >/dev/null \ +pip install --no-cache-dir PyYAML \ || fatal "pip install PyYAML failed" -# Preinstall forked version of libcloud, because nodemanager "pip install" -# won't pick it up by default. -pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ - || fatal "pip install apache-libcloud failed" - -# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks. -pip freeze | grep -x llfuse==1.2.0 || ( - set -e - yes | pip uninstall llfuse || true - cython --version || fatal "no cython; try sudo apt-get install cython" - cd "$temp" - (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse) - cd python-llfuse - git checkout 620722fd990ea642ddb8e7412676af482c090c0c - git checkout setup.py - sed -i -e "s:'1\\.2':'1.2.0':" setup.py - python setup.py build_cython - python setup.py install --force -) || fatal "llfuse fork failed" -pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???" +# Preinstall libcloud if using a fork; otherwise nodemanager "pip +# install" won't pick it up by default. +if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then + pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ + || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ + || fatal "pip install apache-libcloud failed" +fi # Deactivate Python 2 virtualenv deactivate @@ -687,11 +705,17 @@ do_test_once() { } do_install() { - if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then - retry do_install_once ${@} - else - title "Skipping $1 install" - fi + skipit=false + + if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then + retry do_install_once ${@} + else + skipit=true + fi + + if [[ "$skipit" = true ]]; then + title "Skipping $1 install" + fi } do_install_once() { @@ -717,8 +741,8 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -765,6 +789,14 @@ install_ruby_sdk() { } do_install sdk/ruby ruby_sdk +install_R_sdk() { + if [[ "$NEED_SDK_R" = true ]]; then + cd "$WORKSPACE/sdk/R" \ + && Rscript --vanilla install_deps.R + fi +} +do_install sdk/R R_sdk + install_perl_sdk() { cd "$WORKSPACE/sdk/perl" \ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ @@ -821,7 +853,7 @@ install_apiserver() { # is a postgresql superuser. cd "$WORKSPACE/services/api" \ && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \ - && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null + && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -857,6 +889,7 @@ gostuff=( lib/cli lib/cmd lib/crunchstat + lib/dispatchcloud sdk/go/arvados sdk/go/arvadosclient sdk/go/blockdigest @@ -935,6 +968,15 @@ test_ruby_sdk() { } do_test sdk/ruby ruby_sdk +test_R_sdk() { + if [[ "$NEED_SDK_R" = true ]]; then + cd "$WORKSPACE/sdk/R" \ + && Rscript --vanilla run_test.R + fi +} + +do_test sdk/R R_sdk + test_cli() { cd "$WORKSPACE/sdk/cli" \ && mkdir -p /tmp/keep \