X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e663d52fd17d989ad4be1a34413c21537cbe957e..be671e455b11f50af965923b980977264a8d7364:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index e1e83edbc8..f567cf41d5 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -69,7 +69,12 @@ apps/workbench_functionals (*) apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile +cmd/arvados-client doc +lib/cli +lib/cmd +lib/crunchstat +lib/dispatchcloud services/api services/arv-git-httpd services/crunchstat @@ -100,11 +105,12 @@ sdk/go/health sdk/go/httpserver sdk/go/manifest sdk/go/blockdigest -sdk/go/streamer +sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner sdk/cwl -tools/arv-sync-groups +sdk/R +tools/sync-groups tools/crunchstat-summary tools/keep-exercise tools/keep-rsync @@ -128,6 +134,7 @@ VENV3DIR= PYTHONPATH= GEMHOME= PERLINSTALLBASE= +R_LIBS= short= only_install= @@ -212,6 +219,41 @@ sanity_checks() { echo -n 'cadaver: ' cadaver --version | grep -w cadaver \ || fatal "No cadaver. Try: apt-get install cadaver" + echo -n 'libattr1 xattr.h: ' + find /usr/include -path '*/attr/xattr.h' | egrep --max-count=1 . \ + || fatal "No libattr1 xattr.h. Try: apt-get install libattr1-dev" + echo -n 'libcurl curl.h: ' + find /usr/include -path '*/curl/curl.h' | egrep --max-count=1 . \ + || fatal "No libcurl curl.h. Try: apt-get install libcurl4-gnutls-dev" + echo -n 'libpq libpq-fe.h: ' + find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ + || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" + echo -n 'services/api/config/database.yml: ' + if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then + fatal "Please provide a database.yml file for the test suite" + else + echo "OK" + fi + echo -n 'postgresql: ' + psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" + echo -n 'phantomjs: ' + phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs" + echo -n 'xvfb: ' + which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" + echo -n 'graphviz: ' + dot -V || fatal "No graphviz. Try: apt-get install graphviz" + + if [[ "$NEED_SDK_R" = true ]]; then + # R SDK stuff + echo -n 'R: ' + which Rscript || fatal "No Rscript. Try: apt-get install r-base" + echo -n 'testthat: ' + Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat" + # needed for roxygen2, needed for devtools, needed for R sdk + pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev" + # needed for pkgdown, builds R SDK doc pages + which pandoc || fatal "No pandoc. Try: apt-get install pandoc" + fi } rotate_logfile() { @@ -281,45 +323,51 @@ do esac done -start_api() { - echo 'Starting API server...' +# R SDK installation is very slow (~360s in a clean environment) and only +# required when testing it. Skip that step if it is not needed. +NEED_SDK_R=true + +if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then + NEED_SDK_R=false +fi + +if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then + NEED_SDK_R=false +fi + +start_services() { + echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' + if [[ ! -d "$WORKSPACE/services/api/log" ]]; then + mkdir -p "$WORKSPACE/services/api/log" + fi + # Remove empty api.pid file if it exists + if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then + rm -f "$WORKSPACE/tmp/api.pid" + fi cd "$WORKSPACE" \ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ - && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && (env | egrep ^ARVADOS) -} - -start_nginx_proxy_services() { - echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' - cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_ws \ && python sdk/python/tests/run_test_server.py start_nginx \ - && export ARVADOS_TEST_PROXY_SERVICES=1 + && (env | egrep ^ARVADOS) } stop_services() { - if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then - unset ARVADOS_TEST_PROXY_SERVICES - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy - fi - if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then - unset ARVADOS_TEST_API_HOST - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop + if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then + return fi + unset ARVADOS_TEST_API_HOST + cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py stop_nginx \ + && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_ws \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ + && python sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python sdk/python/tests/run_test_server.py stop } interrupt() { @@ -346,7 +394,7 @@ if [[ -z "$temp" ]]; then fi # Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE +for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -444,36 +492,61 @@ setup_virtualenv() { fi if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' else - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' + "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' } export PERLINSTALLBASE export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" +export R_LIBS + export GOPATH mkdir -p "$GOPATH/src/git.curoverse.com" -ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ +rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" +for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -d "$d" ]] && rmdir "$d" + [[ -h "$d" ]] && rm "$d" +done +ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ || fatal "symlink failed" go get -v github.com/kardianos/govendor \ || fatal "govendor install failed" +cd "$GOPATH/src/git.curoverse.com/arvados.git" \ + || fatal +# Remove cached source dirs in workdir. Otherwise, they won't qualify +# as +missing or +external below, and we won't be able to detect that +# they're missing from vendor/vendor.json. +rm -r vendor/*/ +go get -v -d ... +"$GOPATH/bin/govendor" sync \ + || fatal "govendor sync failed" +[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ + || fatal "vendor/vendor.json has unused or missing dependencies -- try: +* govendor remove +unused +* govendor add +missing +external +" +cd "$WORKSPACE" + setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. pip freeze 2>/dev/null | egrep ^PyYAML= \ - || pip install PyYAML >/dev/null \ + || pip install --no-cache-dir PyYAML >/dev/null \ || fatal "pip install PyYAML failed" -# Preinstall forked version of libcloud, because nodemanager "pip install" +# Preinstall libcloud, because nodemanager "pip install" # won't pick it up by default. pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ + || pip install --pre --ignore-installed --no-cache-dir apache-libcloud>=$LIBCLOUD_PIN >/dev/null \ || fatal "pip install apache-libcloud failed" # We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks. @@ -642,11 +715,17 @@ do_test_once() { } do_install() { - if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then - retry do_install_once ${@} - else - title "Skipping $1 install" - fi + skipit=false + + if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then + retry do_install_once ${@} + else + skipit=true + fi + + if [[ "$skipit" = true ]]; then + title "Skipping $1 install" + fi } do_install_once() { @@ -672,8 +751,8 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -720,6 +799,14 @@ install_ruby_sdk() { } do_install sdk/ruby ruby_sdk +install_R_sdk() { + if [[ "$NEED_SDK_R" = true ]]; then + cd "$WORKSPACE/sdk/R" \ + && Rscript --vanilla install_deps.R + fi +} +do_install sdk/R R_sdk + install_perl_sdk() { cd "$WORKSPACE/sdk/perl" \ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ @@ -780,6 +867,25 @@ install_apiserver() { mkdir -p "$WORKSPACE/services/api/tmp/pids" + cert="$WORKSPACE/services/api/tmp/self-signed" + if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then + ( + dir="$WORKSPACE/services/api/tmp" + set -ex + openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"