X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/123d3d778051c90b6be39b0f0b25da24d025d382..14ef215b72d2d56986ad29f1358cf27f7be75294:/jenkins/run-tests.sh diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index 8e58d67..89e75a8 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -1,5 +1,7 @@ #!/bin/bash +. `dirname "$(readlink -f "$0")"`/libcloud-pin + read -rd "\000" helpmessage <&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}" + echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})" exit 1 } @@ -135,6 +139,7 @@ report_outcomes() { exit_cleanly() { trap - INT + create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/" rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log" stop_services rotate_logfile "$WORKSPACE/services/api/log/" "test.log" @@ -144,47 +149,42 @@ exit_cleanly() { } sanity_checks() { - # Make sure WORKSPACE is set - if ! [[ -n "$WORKSPACE" ]]; then - echo >&2 "$helpmessage" - echo >&2 - echo >&2 "Error: WORKSPACE environment variable not set" - echo >&2 - exit 1 - fi - - # Make sure virtualenv is installed - `virtualenv --help >/dev/null 2>&1` - - if [[ "$?" != "0" ]]; then - echo >&2 - echo >&2 "Error: virtualenv could not be found" - echo >&2 - exit 1 - fi - - # Make sure go is installed - `go env >/dev/null 2>&1` - - if [[ "$?" != "0" ]]; then - echo >&2 - echo >&2 "Error: go could not be found" - echo >&2 - exit 1 - fi - - # Make sure gcc is installed - `gcc --help >/dev/null 2>&1` - - if [[ "$?" != "0" ]]; then - echo >&2 - echo >&2 "Error: gcc could not be found" - echo >&2 - exit 1 - fi + ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ + || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" + echo Checking dependencies: + echo -n 'virtualenv: ' + virtualenv --version \ + || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" + echo -n 'go: ' + go version \ + || fatal "No go binary. See http://golang.org/doc/install" + echo -n 'gcc: ' + gcc --version | egrep ^gcc \ + || fatal "No gcc. Try: apt-get install build-essential" + echo -n 'fuse.h: ' + find /usr/include -wholename '*fuse/fuse.h' \ + || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev" + echo -n 'pyconfig.h: ' + find /usr/include -name pyconfig.h | egrep --max-count=1 . \ + || fatal "No pyconfig.h. Try: apt-get install python-dev" + echo -n 'nginx: ' + PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \ + || fatal "No nginx. Try: apt-get install nginx" + echo -n 'perl: ' + perl -v | grep version \ + || fatal "No perl. Try: apt-get install perl" + for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do + echo -n "perl $mod: " + perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \ + || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl" + done + echo -n 'gitolite: ' + which gitolite \ + || fatal "No gitolite. Try: apt-get install gitolite3" } rotate_logfile() { + # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log" # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run if [[ -f "$1/$2" ]]; then THEDATE=`date +%Y%m%d%H%M%S` @@ -221,11 +221,12 @@ do skip_install=1 only_install="$1"; shift ;; + --temp) + temp="$1"; shift + temp_preserve=1 + ;; --leave-temp) - leave_temp[VENVDIR]=1 - leave_temp[VENV3DIR]=1 - leave_temp[GOPATH]=1 - leave_temp[GEMHOME]=1 + temp_preserve=1 ;; --retry) retry=1 @@ -255,9 +256,10 @@ start_api() { } start_nginx_proxy_services() { - echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...' + echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...' cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_nginx \ && export ARVADOS_TEST_PROXY_SERVICES=1 @@ -269,6 +271,7 @@ stop_services() { cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py stop_nginx \ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ && python sdk/python/tests/run_test_server.py stop_keep_proxy fi if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then @@ -297,13 +300,18 @@ fi cd "$WORKSPACE" find -name '*.pyc' -delete +if [[ -z "$temp" ]]; then + temp="$(mktemp -d)" +fi + # Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME +for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE do - if [[ -n "${!tmpdir}" ]]; then - leave_temp[$tmpdir]=1 - else - eval $tmpdir=$(mktemp -d) + if [[ -z "${!tmpdir}" ]]; then + eval "$tmpdir"="$temp/$tmpdir" + fi + if ! [[ -d "${!tmpdir}" ]]; then + mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)" fi done @@ -373,43 +381,56 @@ with_test_gemset() { if [[ "$using_rvm" == true ]]; then "$@" else - GEM_HOME="$tmpdir_gem_home" "$@" + GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@" fi } +gem_uninstall_if_exists() { + if gem list "$1\$" | egrep '^\w'; then + gem uninstall --force --all --executables "$1" + fi +} + +setup_virtualenv() { + local venvdest=$1; shift + if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then + virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + fi + "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7' + # ubuntu1404 can't seem to install mock via tests_require, but it can do this. + "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' +} + +export PERLINSTALLBASE +export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" + export GOPATH mkdir -p "$GOPATH/src/git.curoverse.com" ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ || fatal "symlink failed" -virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed" +setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" -# When re-using $VENVDIR, upgrade any packages (except arvados) that are -# already installed -pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=` - -# Note: this must be the last time we change PATH, otherwise rvm will -# whine a lot. -setup_ruby_environment - -echo "PATH is $PATH" +# Needed for run_test_server.py which is used by certain (non-Python) tests. +pip freeze 2>/dev/null | egrep ^PyYAML= \ + || pip install PyYAML >/dev/null \ + || fatal "pip install PyYAML failed" -if ! which bundler >/dev/null -then - gem install --user-install bundler || fatal 'Could not install bundler' -fi +# Preinstall forked version of libcloud, because nodemanager "pip install" +# won't pick it up by default. +pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ + || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ + || fatal "pip install apache-libcloud failed" -# Needed for run_test_server.py which is used by certain (non-Python) tests. -echo "pip install -q PyYAML" -pip install --quiet PyYAML || fatal "pip install PyYAML failed" +# Deactivate Python 2 virtualenv +deactivate # If Python 3 is available, set up its virtualenv in $VENV3DIR. # Otherwise, skip dependent tests. PYTHON3=$(which python3) if [ "0" = "$?" ]; then - virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \ - || fatal "python3 virtualenv $VENV3DIR failed" + setup_virtualenv "$VENV3DIR" --python python3 else PYTHON3= skip[services/dockercleaner]=1 @@ -421,6 +442,20 @@ services/dockercleaner install and tests will be skipped EOF fi +# Reactivate Python 2 virtualenv +. "$VENVDIR/bin/activate" + +# Note: this must be the last time we change PATH, otherwise rvm will +# whine a lot. +setup_ruby_environment + +echo "PATH is $PATH" + +if ! which bundler >/dev/null +then + gem install --user-install bundler || fatal 'Could not install bundler' +fi + checkexit() { if [[ "$1" != "0" ]]; then title "!!!!!! $2 FAILED !!!!!!" @@ -450,22 +485,35 @@ do_test() { } do_test_once() { + unset result if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] ) then title "Running $1 tests" timer_reset if [[ "$2" == "go" ]] then + covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" + coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") + # We do "go get -t" here to catch compilation errors + # before trying "go test". Otherwise, coverage-reporting + # mode makes Go show the wrong line numbers when reporting + # compilation errors. if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this # does: - cd "$WORKSPACE/$1" && go test ${testargs[$1]} + cd "$WORKSPACE/$1" && \ + go get -t "git.curoverse.com/arvados.git/$1" && \ + go test ${coverflags[@]} ${testargs[$1]} else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test "git.curoverse.com/arvados.git/$1" + go get -t "git.curoverse.com/arvados.git/$1" && \ + go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1" fi + result="$?" + go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html" + rm "$WORKSPACE/tmp/.$covername.tmp" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -478,7 +526,7 @@ do_test_once() { else "test_$1" fi - result="$?" + result=${result:-$?} checkexit $result "$1 tests" title "End of $1 tests (`timer`)" return $result @@ -504,10 +552,10 @@ do_install() { # the source dist package to avoid a pip bug. # see https://arvados.org/issues/5766 for details. - # Also need to install twice, because if it belives the package is + # Also need to install twice, because if it believes the package is # already installed, pip it won't install it. So the first "pip # install" ensures that the dependencies are met, the second "pip - # install" ensures that we've actually install the local package + # install" ensures that we've actually installed the local package # we just built. cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ @@ -551,24 +599,38 @@ install_doc() { } do_install doc -install_ruby_sdk() { - with_test_gemset gem uninstall --force --all --executables arvados \ - && cd "$WORKSPACE/sdk/ruby" \ +install_gem() { + gemname=$1 + srcpath=$2 + with_test_gemset gem_uninstall_if_exists "$gemname" \ + && cd "$WORKSPACE/$srcpath" \ && bundle_install_trylocal \ - && gem build arvados.gemspec \ - && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1` + && gem build "$gemname.gemspec" \ + && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1) +} + +install_ruby_sdk() { + install_gem arvados sdk/ruby } do_install sdk/ruby ruby_sdk +install_perl_sdk() { + cd "$WORKSPACE/sdk/perl" \ + && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ + && make install INSTALLDIRS=perl +} +do_install sdk/perl perl_sdk + install_cli() { - with_test_gemset gem uninstall --force --all --executables arvados-cli \ - && cd "$WORKSPACE/sdk/cli" \ - && bundle_install_trylocal \ - && gem build arvados-cli.gemspec \ - && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1` + install_gem arvados-cli sdk/cli } do_install sdk/cli cli +install_login-sync() { + install_gem arvados-login-sync services/login-sync +} +do_install services/login-sync login-sync + # Install the Python SDK early. Various other test suites (like # keepproxy) bring up run_test_server.py, which imports the arvados # module. We can't actually *test* the Python SDK yet though, because @@ -576,6 +638,7 @@ do_install sdk/cli cli # keepproxy). declare -a pythonstuff pythonstuff=( + sdk/pam sdk/python services/fuse services/nodemanager @@ -627,8 +690,9 @@ install_apiserver() { # Clear out any lingering postgresql connections to the test # database, so that we can drop it. This assumes the current user # is a postgresql superuser. - test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") - psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null + cd "$WORKSPACE/services/api" \ + && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \ + && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null cd "$WORKSPACE/services/api" \ && RAILS_ENV=test bundle exec rake db:drop \ @@ -639,13 +703,22 @@ do_install services/api apiserver declare -a gostuff gostuff=( + sdk/go/arvadosclient + sdk/go/blockdigest + sdk/go/manifest + sdk/go/streamer + sdk/go/crunchrunner services/arv-git-httpd services/crunchstat + services/keep-web services/keepstore - services/keepproxy - sdk/go/arvadosclient sdk/go/keepclient - sdk/go/streamer + services/keepproxy + services/datamanager/summary + services/datamanager/collection + services/datamanager/keep + services/datamanager + tools/keep-rsync ) for g in "${gostuff[@]}" do @@ -666,7 +739,7 @@ test_doclinkchecker() { ARVADOS_API_HOST=qr1hi.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! - PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST + PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST ) } do_test doc doclinkchecker @@ -701,6 +774,12 @@ test_cli() { } do_test sdk/cli cli +test_login-sync() { + cd "$WORKSPACE/services/login-sync" \ + && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]} +} +do_test services/login-sync login-sync + for p in "${pythonstuff[@]}" do do_test "$p" pip