X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/d99331757367dbbb3607a4644a5a6572667cb262..b4ca11159ff0d38af3750c3a109e731f295a8460:/jenkins/run-tests.sh diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index 62786dc..89e75a8 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -59,21 +59,25 @@ apps/workbench_benchmark apps/workbench_profile doc services/api +services/arv-git-httpd services/crunchstat services/dockercleaner services/fuse +services/keep-web services/keepproxy services/keepstore services/login-sync services/nodemanager -services/arv-git-httpd sdk/cli sdk/pam sdk/python sdk/ruby sdk/go/arvadosclient sdk/go/keepclient +sdk/go/manifest +sdk/go/blockdigest sdk/go/streamer +sdk/go/crunchrunner tools/keep-rsync EOF @@ -150,7 +154,7 @@ sanity_checks() { echo Checking dependencies: echo -n 'virtualenv: ' virtualenv --version \ - || fatal "No virtualenv. Try: apt-get install virtualenv" + || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" @@ -172,7 +176,7 @@ sanity_checks() { for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do echo -n "perl $mod: " perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \ - || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl" + || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl" done echo -n 'gitolite: ' which gitolite \ @@ -252,9 +256,10 @@ start_api() { } start_nginx_proxy_services() { - echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...' + echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...' cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_nginx \ && export ARVADOS_TEST_PROXY_SERVICES=1 @@ -266,6 +271,7 @@ stop_services() { cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py stop_nginx \ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ && python sdk/python/tests/run_test_server.py stop_keep_proxy fi if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then @@ -385,6 +391,16 @@ gem_uninstall_if_exists() { fi } +setup_virtualenv() { + local venvdest=$1; shift + if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then + virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + fi + "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7' + # ubuntu1404 can't seem to install mock via tests_require, but it can do this. + "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' +} + export PERLINSTALLBASE export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" @@ -393,15 +409,9 @@ mkdir -p "$GOPATH/src/git.curoverse.com" ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ || fatal "symlink failed" -if ! [[ -e "$VENVDIR/bin/activate" ]] || ! [[ -e "$VENVDIR/bin/pip" ]]; then - virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed" -fi +setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" -if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\ -f2 | cut -d. -f1)" -lt 18 ]; then - pip install --upgrade setuptools pip -fi - # Needed for run_test_server.py which is used by certain (non-Python) tests. pip freeze 2>/dev/null | egrep ^PyYAML= \ || pip install PyYAML >/dev/null \ @@ -420,17 +430,7 @@ deactivate # Otherwise, skip dependent tests. PYTHON3=$(which python3) if [ "0" = "$?" ]; then - virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \ - || fatal "python3 virtualenv $VENV3DIR failed" - - . "$VENV3DIR/bin/activate" - - if (pip install setuptools | grep setuptools-0) || [ "$($VENV3DIR/bin/easy_install --version | cut -d\ -f2 | cut -d. -f1)" -lt 18 ]; then - pip install --upgrade setuptools pip - fi - - # Deactivate Python 3 virtualenv - deactivate + setup_virtualenv "$VENV3DIR" --python python3 else PYTHON3= skip[services/dockercleaner]=1 @@ -494,15 +494,22 @@ do_test_once() { then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") + # We do "go get -t" here to catch compilation errors + # before trying "go test". Otherwise, coverage-reporting + # mode makes Go show the wrong line numbers when reporting + # compilation errors. if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this # does: - cd "$WORKSPACE/$1" && go test ${coverflags[@]} ${testargs[$1]} + cd "$WORKSPACE/$1" && \ + go get -t "git.curoverse.com/arvados.git/$1" && \ + go test ${coverflags[@]} ${testargs[$1]} else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1" + go get -t "git.curoverse.com/arvados.git/$1" && \ + go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1" fi result="$?" go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html" @@ -697,14 +704,19 @@ do_install services/api apiserver declare -a gostuff gostuff=( sdk/go/arvadosclient + sdk/go/blockdigest + sdk/go/manifest sdk/go/streamer + sdk/go/crunchrunner services/arv-git-httpd services/crunchstat + services/keep-web services/keepstore sdk/go/keepclient services/keepproxy services/datamanager/summary services/datamanager/collection + services/datamanager/keep services/datamanager tools/keep-rsync )