X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/289d2cf581b59632369087388f6163f3979c5e86..66d07bb91f91b0cd4c92c7ffa913f7181a3d4942:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index e1e83edbc8..2fb82554a5 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -69,7 +69,14 @@ apps/workbench_functionals (*) apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile +cmd/arvados-client +cmd/arvados-server doc +lib/cli +lib/cmd +lib/controller +lib/crunchstat +lib/dispatchcloud services/api services/arv-git-httpd services/crunchstat @@ -94,17 +101,19 @@ sdk/python:py3 sdk/ruby sdk/go/arvados sdk/go/arvadosclient +sdk/go/auth sdk/go/dispatch sdk/go/keepclient sdk/go/health sdk/go/httpserver sdk/go/manifest sdk/go/blockdigest -sdk/go/streamer +sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner sdk/cwl -tools/arv-sync-groups +sdk/R +tools/sync-groups tools/crunchstat-summary tools/keep-exercise tools/keep-rsync @@ -128,6 +137,7 @@ VENV3DIR= PYTHONPATH= GEMHOME= PERLINSTALLBASE= +R_LIBS= short= only_install= @@ -175,8 +185,8 @@ sanity_checks() { echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \ - || fatal "Go >= 1.8 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \ + || fatal "Go >= 1.10 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -212,6 +222,43 @@ sanity_checks() { echo -n 'cadaver: ' cadaver --version | grep -w cadaver \ || fatal "No cadaver. Try: apt-get install cadaver" + echo -n 'libattr1 xattr.h: ' + find /usr/include -path '*/attr/xattr.h' | egrep --max-count=1 . \ + || fatal "No libattr1 xattr.h. Try: apt-get install libattr1-dev" + echo -n 'libcurl curl.h: ' + find /usr/include -path '*/curl/curl.h' | egrep --max-count=1 . \ + || fatal "No libcurl curl.h. Try: apt-get install libcurl4-gnutls-dev" + echo -n 'libpq libpq-fe.h: ' + find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ + || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" + echo -n 'services/api/config/database.yml: ' + if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then + fatal "Please provide a database.yml file for the test suite" + else + echo "OK" + fi + echo -n 'postgresql: ' + psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" + echo -n 'phantomjs: ' + phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs" + echo -n 'xvfb: ' + which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" + echo -n 'graphviz: ' + dot -V || fatal "No graphviz. Try: apt-get install graphviz" + echo -n 'geckodriver: ' + geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver" + + if [[ "$NEED_SDK_R" = true ]]; then + # R SDK stuff + echo -n 'R: ' + which Rscript || fatal "No Rscript. Try: apt-get install r-base" + echo -n 'testthat: ' + Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat" + # needed for roxygen2, needed for devtools, needed for R sdk + pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev" + # needed for pkgdown, builds R SDK doc pages + which pandoc || fatal "No pandoc. Try: apt-get install pandoc" + fi } rotate_logfile() { @@ -226,8 +273,11 @@ rotate_logfile() { declare -a failures declare -A skip +declare -A only declare -A testargs skip[apps/workbench_profile]=1 +# nodemanager_integration tests are not reliable, see #12061. +skip[services/nodemanager_integration]=1 while [[ -n "$1" ]] do @@ -242,7 +292,7 @@ do skip[$1]=1; shift ;; --only) - only="$1"; skip[$1]=""; shift + only[$1]=1; skip[$1]=""; shift ;; --short) short=1 @@ -281,45 +331,61 @@ do esac done -start_api() { - echo 'Starting API server...' +# R SDK installation is very slow (~360s in a clean environment) and only +# required when testing it. Skip that step if it is not needed. +NEED_SDK_R=true + +if [[ ${#only[@]} -ne 0 ]] && + [[ -z "${only['sdk/R']}" && -z "${only['doc']}" ]]; then + NEED_SDK_R=false +fi + +if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then + NEED_SDK_R=false +fi + +if [[ $NEED_SDK_R == false ]]; then + echo "R SDK not needed, it will not be installed." +fi + +start_services() { + echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' + if [[ ! -d "$WORKSPACE/services/api/log" ]]; then + mkdir -p "$WORKSPACE/services/api/log" + fi + # Remove empty api.pid file if it exists + if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then + rm -f "$WORKSPACE/tmp/api.pid" + fi cd "$WORKSPACE" \ - && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ + && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ - && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && (env | egrep ^ARVADOS) -} - -start_nginx_proxy_services() { - echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' - cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py start_controller \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && export ARVADOS_TEST_PROXY_SERVICES=1 + && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \ + && (env | egrep ^ARVADOS) + if [[ -n "$fail" ]]; then + return 1 + fi } stop_services() { - if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then - unset ARVADOS_TEST_PROXY_SERVICES - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy - fi - if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then - unset ARVADOS_TEST_API_HOST - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop + if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then + return fi + unset ARVADOS_TEST_API_HOST + cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py stop_nginx \ + && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_ws \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ + && python sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python sdk/python/tests/run_test_server.py stop_controller \ + && python sdk/python/tests/run_test_server.py stop } interrupt() { @@ -346,7 +412,7 @@ if [[ -z "$temp" ]]; then fi # Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE +for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -356,6 +422,8 @@ do fi done +rm -vf "${WORKSPACE}/tmp/*.log" + setup_ruby_environment() { if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then source "$HOME/.rvm/scripts/rvm" @@ -441,56 +509,78 @@ setup_virtualenv() { local venvdest="$1"; shift if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + elif [[ -n "$short" ]]; then + return fi if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' else - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' + "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' } export PERLINSTALLBASE export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" +export R_LIBS + export GOPATH -mkdir -p "$GOPATH/src/git.curoverse.com" -ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ - || fatal "symlink failed" -go get -v github.com/kardianos/govendor \ - || fatal "govendor install failed" +( + set -e + mkdir -p "$GOPATH/src/git.curoverse.com" + rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH" + if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then + for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \ + "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -d "$d" ]] && rmdir "$d" + done + fi + for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -h "$d" ]] && rm "$d" + done + ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" + go get -v github.com/kardianos/govendor + cd "$GOPATH/src/git.curoverse.com/arvados.git" + if [[ -n "$short" ]]; then + go get -v -d ... + "$GOPATH/bin/govendor" sync + else + # Remove cached source dirs in workdir. Otherwise, they will + # not qualify as +missing or +external below, and we won't be + # able to detect that they're missing from vendor/vendor.json. + rm -rf vendor/*/ + go get -v -d ... + "$GOPATH/bin/govendor" sync + [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ + || fatal "vendor/vendor.json has unused or missing dependencies -- try: + +(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) + +"; + fi +) || fatal "Go setup failed" setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. -pip freeze 2>/dev/null | egrep ^PyYAML= \ - || pip install PyYAML >/dev/null \ +pip install --no-cache-dir PyYAML \ || fatal "pip install PyYAML failed" -# Preinstall forked version of libcloud, because nodemanager "pip install" -# won't pick it up by default. -pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ - || fatal "pip install apache-libcloud failed" - -# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks. -pip freeze | grep -x llfuse==1.2.0 || ( - set -e - yes | pip uninstall llfuse || true - cython --version || fatal "no cython; try sudo apt-get install cython" - cd "$temp" - (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse) - cd python-llfuse - git checkout 620722fd990ea642ddb8e7412676af482c090c0c - git checkout setup.py - sed -i -e "s:'1\\.2':'1.2.0':" setup.py - python setup.py build_cython - python setup.py install --force -) || fatal "llfuse fork failed" -pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???" +# Preinstall libcloud if using a fork; otherwise nodemanager "pip +# install" won't pick it up by default. +if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then + pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ + || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ + || fatal "pip install apache-libcloud failed" +fi # Deactivate Python 2 virtualenv deactivate @@ -535,6 +625,12 @@ then gem install --user-install bundler || fatal 'Could not install bundler' fi +# Jenkins config requires that glob tmp/*.log match something. Ensure +# that happens even if we don't end up running services that set up +# logging. +mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp" +touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log" + retry() { remain="${repeat}" while : @@ -571,8 +667,9 @@ do_test() { ;; esac if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \ - (-z "${only}" || "${only}" == "${suite}" || \ - "${only}" == "${1}") ]]; then + (${#only[@]} -eq 0 || ${only[$suite]} -eq 1 || \ + ${only[$1]} -eq 1) || + ${only[$2]} -eq 1 ]]; then retry do_test_once ${@} else title "Skipping ${1} tests" @@ -592,7 +689,7 @@ do_test_once() { # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -t "git.curoverse.com/arvados.git/$1" && \ + go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \ if [[ -n "${testargs[$1]}" ]] @@ -642,11 +739,17 @@ do_test_once() { } do_install() { - if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then - retry do_install_once ${@} - else - title "Skipping $1 install" - fi + skipit=false + + if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then + retry do_install_once ${@} + else + skipit=true + fi + + if [[ "$skipit" = true ]]; then + title "Skipping $1 install" + fi } do_install_once() { @@ -654,7 +757,7 @@ do_install_once() { timer_reset if [[ "$2" == "go" ]] then - go get -t "git.curoverse.com/arvados.git/$1" + go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -672,8 +775,8 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -720,6 +823,14 @@ install_ruby_sdk() { } do_install sdk/ruby ruby_sdk +install_R_sdk() { + if [[ "$NEED_SDK_R" = true ]]; then + cd "$WORKSPACE/sdk/R" \ + && Rscript --vanilla install_deps.R + fi +} +do_install sdk/R R_sdk + install_perl_sdk() { cd "$WORKSPACE/sdk/perl" \ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ @@ -776,10 +887,29 @@ install_apiserver() { # is a postgresql superuser. cd "$WORKSPACE/services/api" \ && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \ - && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null + && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" + cert="$WORKSPACE/services/api/tmp/self-signed" + if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then + ( + dir="$WORKSPACE/services/api/tmp" + set -ex + openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"