X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e3ac17f8a8aa439e21a8bf56a571f91a671313f7..8e2a634e2b0ef180f30db5322233244b142f93f6:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index afaa834d3b..d18250a95a 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -1,6 +1,9 @@ #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 -. `dirname "$(readlink -f "$0")"`/libcloud-pin +. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh COLUMNS=80 . `dirname "$(readlink -f "$0")"`/run-library.sh @@ -66,18 +69,25 @@ apps/workbench_functionals (*) apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile +cmd/arvados-client doc +lib/cli +lib/cmd +lib/crunchstat +lib/dispatchcloud services/api services/arv-git-httpd services/crunchstat services/dockercleaner services/fuse +services/health services/keep-web services/keepproxy services/keepstore services/keep-balance services/login-sync services/nodemanager +services/nodemanager_integration services/crunch-run services/crunch-dispatch-local services/crunch-dispatch-slurm @@ -85,18 +95,22 @@ services/ws sdk/cli sdk/pam sdk/python +sdk/python:py3 sdk/ruby sdk/go/arvados sdk/go/arvadosclient sdk/go/dispatch sdk/go/keepclient +sdk/go/health sdk/go/httpserver sdk/go/manifest sdk/go/blockdigest -sdk/go/streamer +sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner sdk/cwl +sdk/R +tools/sync-groups tools/crunchstat-summary tools/keep-exercise tools/keep-rsync @@ -120,6 +134,7 @@ VENV3DIR= PYTHONPATH= GEMHOME= PERLINSTALLBASE= +R_LIBS= short= only_install= @@ -164,26 +179,26 @@ sanity_checks() { echo -n 'ruby: ' ruby -v \ || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" - echo -n 'bundler: ' - bundle version \ - || fatal "No bundler. Try: gem install bundler" echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 7 ]] \ - || fatal "Go >= 1.7 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \ + || fatal "Go >= 1.8 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" echo -n 'fuse.h: ' - find /usr/include -wholename '*fuse/fuse.h' \ + find /usr/include -path '*fuse/fuse.h' | egrep --max-count=1 . \ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev" echo -n 'gnutls.h: ' - find /usr/include -wholename '*gnutls/gnutls.h' \ + find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev" - echo -n 'pyconfig.h: ' - find /usr/include -name pyconfig.h | egrep --max-count=1 . \ - || fatal "No pyconfig.h. Try: apt-get install python-dev" + echo -n 'Python2 pyconfig.h: ' + find /usr/include -path '*/python2*/pyconfig.h' | egrep --max-count=1 . \ + || fatal "No Python2 pyconfig.h. Try: apt-get install python2.7-dev" + echo -n 'Python3 pyconfig.h: ' + find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ + || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" echo -n 'nginx: ' PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \ || fatal "No nginx. Try: apt-get install nginx" @@ -198,6 +213,45 @@ sanity_checks() { echo -n 'gitolite: ' which gitolite \ || fatal "No gitolite. Try: apt-get install gitolite3" + echo -n 'npm: ' + npm --version \ + || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/" + echo -n 'cadaver: ' + cadaver --version | grep -w cadaver \ + || fatal "No cadaver. Try: apt-get install cadaver" + echo -n 'libattr1 xattr.h: ' + find /usr/include -path '*/attr/xattr.h' | egrep --max-count=1 . \ + || fatal "No libattr1 xattr.h. Try: apt-get install libattr1-dev" + echo -n 'libcurl curl.h: ' + find /usr/include -path '*/curl/curl.h' | egrep --max-count=1 . \ + || fatal "No libcurl curl.h. Try: apt-get install libcurl4-gnutls-dev" + echo -n 'libpq libpq-fe.h: ' + find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ + || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" + echo -n 'services/api/config/database.yml: ' + if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then + fatal "Please provide a database.yml file for the test suite" + else + echo "OK" + fi + echo -n 'postgresql: ' + psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" + echo -n 'phantomjs: ' + phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs" + echo -n 'xvfb: ' + which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" + echo -n 'graphviz: ' + dot -V || fatal "No graphviz. Try: apt-get install graphviz" + + # R SDK stuff + echo -n 'R: ' + which R || fatal "No R. Try: apt-get install r-base" + echo -n 'testthat: ' + R -q -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat" + # needed for roxygen2, needed for devtools, needed for R sdk + pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev" + # needed for pkgdown, builds R SDK doc pages + which pandoc || fatal "No pandoc. Try: apt-get install pandoc" } rotate_logfile() { @@ -267,45 +321,39 @@ do esac done -start_api() { - echo 'Starting API server...' +start_services() { + echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' + if [[ ! -d "$WORKSPACE/services/api/log" ]]; then + mkdir -p "$WORKSPACE/services/api/log" + fi + # Remove empty api.pid file if it exists + if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then + rm -f "$WORKSPACE/tmp/api.pid" + fi cd "$WORKSPACE" \ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ - && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && (env | egrep ^ARVADOS) -} - -start_nginx_proxy_services() { - echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' - cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_ws \ && python sdk/python/tests/run_test_server.py start_nginx \ - && export ARVADOS_TEST_PROXY_SERVICES=1 + && (env | egrep ^ARVADOS) } stop_services() { - if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then - unset ARVADOS_TEST_PROXY_SERVICES - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy - fi - if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then - unset ARVADOS_TEST_API_HOST - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop + if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then + return fi + unset ARVADOS_TEST_API_HOST + cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py stop_nginx \ + && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_ws \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ + && python sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python sdk/python/tests/run_test_server.py stop } interrupt() { @@ -332,7 +380,7 @@ if [[ -z "$temp" ]]; then fi # Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE +for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -344,13 +392,13 @@ done setup_ruby_environment() { if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then - source "$HOME/.rvm/scripts/rvm" - using_rvm=true + source "$HOME/.rvm/scripts/rvm" + using_rvm=true elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then - source "/usr/local/rvm/scripts/rvm" - using_rvm=true + source "/usr/local/rvm/scripts/rvm" + using_rvm=true else - using_rvm=false + using_rvm=false fi if [[ "$using_rvm" == true ]]; then @@ -399,12 +447,14 @@ setup_ruby_environment() { tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)" PATH="$tmpdir_gem_home/bin:$PATH" - export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)" + export GEM_PATH="$tmpdir_gem_home" echo "Will install dependencies to $(gem env gemdir)" echo "Will install arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" fi + bundle config || gem install bundler \ + || fatal 'install bundler' } with_test_gemset() { @@ -439,10 +489,37 @@ setup_virtualenv() { export PERLINSTALLBASE export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" +export R_LIBS + export GOPATH mkdir -p "$GOPATH/src/git.curoverse.com" -ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ +rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" +for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -d "$d" ]] && rmdir "$d" + [[ -h "$d" ]] && rm "$d" +done +ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ || fatal "symlink failed" +go get -v github.com/kardianos/govendor \ + || fatal "govendor install failed" +cd "$GOPATH/src/git.curoverse.com/arvados.git" \ + || fatal +# Remove cached source dirs in workdir. Otherwise, they won't qualify +# as +missing or +external below, and we won't be able to detect that +# they're missing from vendor/vendor.json. +rm -r vendor/*/ +go get -v -d ... +"$GOPATH/bin/govendor" sync \ + || fatal "govendor sync failed" +[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ + || fatal "vendor/vendor.json has unused or missing dependencies -- try: +* govendor remove +unused +* govendor add +missing +external +" +cd "$WORKSPACE" + setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" @@ -456,29 +533,50 @@ pip freeze 2>/dev/null | egrep ^PyYAML= \ # won't pick it up by default. pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ + || pip install apache-libcloud>=$LIBCLOUD_PIN >/dev/null \ || fatal "pip install apache-libcloud failed" -# Uninstall old llfuse (<1.0), because services/fuse "pip install" -# won't upgrade it by default. -if pip freeze | egrep '^llfuse==0'; then - yes | pip uninstall 'llfuse<1.0' -fi +# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks. +pip freeze | grep -x llfuse==1.2.0 || ( + set -e + yes | pip uninstall llfuse || true + cython --version || fatal "no cython; try sudo apt-get install cython" + cd "$temp" + (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse) + cd python-llfuse + git checkout 620722fd990ea642ddb8e7412676af482c090c0c + git checkout setup.py + sed -i -e "s:'1\\.2':'1.2.0':" setup.py + python setup.py build_cython + python setup.py install --force +) || fatal "llfuse fork failed" +pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???" # Deactivate Python 2 virtualenv deactivate +declare -a pythonstuff +pythonstuff=( + sdk/pam + sdk/python + sdk/python:py3 + sdk/cwl + services/dockercleaner:py3 + services/fuse + services/nodemanager + tools/crunchstat-summary + ) + # If Python 3 is available, set up its virtualenv in $VENV3DIR. # Otherwise, skip dependent tests. PYTHON3=$(which python3) -if [ "0" = "$?" ]; then +if [[ ${?} = 0 ]]; then setup_virtualenv "$VENV3DIR" --python python3 else PYTHON3= - skip[services/dockercleaner]=1 cat >&2 </dev/null + mkdir -p "$WORKSPACE/services/api/tmp/pids" + + cert="$WORKSPACE/services/api/tmp/self-signed" + if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then + ( + dir="$WORKSPACE/services/api/tmp" + set -ex + openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"