X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/71fd4da18b22100682ae7e2079aadfd66360d310..2f66d4cc05e9442a9bb69969744d0750a02a1ed4:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 749075d815..7a3302de9b 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -19,6 +19,10 @@ Syntax: Options: --skip FOO Do not test the FOO component. +--skip sanity Skip initial dev environment sanity checks. +--skip install Do not run any install steps. Just run tests. + You should provide GOPATH, GEMHOME, and VENVDIR options + from a previous invocation if you use this option. --only FOO Do not test anything except the FOO component. --temp DIR Install components and dependencies under DIR instead of making a new temporary directory. Implies --leave-temp. @@ -27,14 +31,11 @@ Options: subsequent invocations. --repeat N Repeat each install/test step until it succeeds N times. --retry Prompt to retry if an install or test suite fails. ---skip-install Do not run any install steps. Just run tests. - You should provide GOPATH, GEMHOME, and VENVDIR options - from a previous invocation if you use this option. --only-install Run specific install step --short Skip (or scale down) some slow tests. +--interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. -CONFIGSRC=path Dir with api server config files to copy into source tree. - (If none given, leave config files alone in source tree.) +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required) services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file sdk/python_test="--test-suite tests.test_keep_locator" @@ -49,7 +50,7 @@ ARVADOS_DEBUG=1 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE, *_test, and other examples shown above. -Assuming --skip-install is not given, all components are installed +Assuming "--skip install" is not given, all components are installed into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many test suites depend on other components being installed, and installing everything tends to be quicker than debugging dependencies. @@ -75,18 +76,28 @@ doc lib/cli lib/cmd lib/controller +lib/controller/federation +lib/controller/railsproxy +lib/controller/router +lib/controller/rpc lib/crunchstat +lib/crunch-run lib/cloud +lib/cloud/azure +lib/cloud/cloudtest lib/dispatchcloud lib/dispatchcloud/container lib/dispatchcloud/scheduler lib/dispatchcloud/ssh_executor lib/dispatchcloud/worker +lib/mount +lib/service services/api services/arv-git-httpd services/crunchstat services/dockercleaner services/fuse +services/fuse:py3 services/health services/keep-web services/keepproxy @@ -95,12 +106,12 @@ services/keep-balance services/login-sync services/nodemanager services/nodemanager_integration -services/crunch-run services/crunch-dispatch-local services/crunch-dispatch-slurm services/ws sdk/cli sdk/pam +sdk/pam:py3 sdk/python sdk/python:py3 sdk/ruby @@ -116,10 +127,12 @@ sdk/go/blockdigest sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner -sdk/cwl +sdk/cwl:py3 sdk/R +sdk/java-v2 tools/sync-groups tools/crunchstat-summary +tools/crunchstat-summary:py3 tools/keep-exercise tools/keep-rsync tools/keep-block-check @@ -143,6 +156,7 @@ PYTHONPATH= GEMHOME= PERLINSTALLBASE= R_LIBS= +export LANG=en_US.UTF-8 short= only_install= @@ -168,7 +182,9 @@ fatal() { exit_cleanly() { trap - INT - create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/" + if which create-plot-data-from-log.sh >/dev/null; then + create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/" + fi rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log" stop_services rotate_logfile "$WORKSPACE/services/api/log/" "test.log" @@ -178,9 +194,17 @@ exit_cleanly() { } sanity_checks() { + [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" + [[ -n "$CONFIGSRC" ]] \ + || fatal "CONFIGSRC environment not set (see: $0 --help)" + [[ -s "$CONFIGSRC/config.yml" ]] \ + || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: + echo "locale: ${LANG}" + [[ "$(locale charmap)" = "UTF-8" ]] \ + || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen" echo -n 'virtualenv: ' virtualenv --version \ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" @@ -190,8 +214,8 @@ sanity_checks() { echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \ - || fatal "Go >= 1.10 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \ + || fatal "Go >= 1.12 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -207,6 +231,8 @@ sanity_checks() { echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" + which netstat \ + || fatal "No netstat. Try: apt-get install net-tools" echo -n 'nginx: ' PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \ || fatal "No nginx. Try: apt-get install nginx" @@ -236,12 +262,6 @@ sanity_checks() { echo -n 'libpq libpq-fe.h: ' find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" - echo -n 'services/api/config/database.yml: ' - if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then - fatal "Please provide a database.yml file for the test suite" - else - echo "OK" - fi echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" echo -n 'phantomjs: ' @@ -264,6 +284,10 @@ sanity_checks() { # needed for pkgdown, builds R SDK doc pages which pandoc || fatal "No pandoc. Try: apt-get install pandoc" fi + echo 'procs with /dev/fuse open:' + find /proc/*/fd -lname /dev/fuse 2>/dev/null | cut -d/ -f3 | xargs --no-run-if-empty ps -lywww + echo 'grep fuse /proc/self/mountinfo:' + grep fuse /proc/self/mountinfo } rotate_logfile() { @@ -302,8 +326,11 @@ do --short) short=1 ;; + --interactive) + interactive=1 + ;; --skip-install) - only_install=nothing + skip[install]=1 ;; --only-install) only_install="$1"; shift @@ -353,8 +380,43 @@ if [[ $NEED_SDK_R == false ]]; then echo "R SDK not needed, it will not be installed." fi +checkpidfile() { + svc="$1" + pid="$(cat "$WORKSPACE/tmp/${svc}.pid")" + if [[ -z "$pid" ]] || ! kill -0 "$pid"; then + tail $WORKSPACE/tmp/${1}*.log + echo "${svc} pid ${pid} not running" + return 1 + fi + echo "${svc} pid ${pid} ok" +} + +checkhealth() { + svc="$1" + base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + url="$base/_health/ping" + if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then + echo "${url} failed" + return 1 + fi +} + +checkdiscoverydoc() { + dd="https://${1}/discovery/v1/apis/arvados/v1/rest" + if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then + echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd" + tail -v $WORKSPACE/tmp/railsapi.log + return 1 + fi + echo "${dd} ok" +} + start_services() { - echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' + if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then + return 0 + fi + . "$VENVDIR/bin/activate" + echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -362,27 +424,47 @@ start_services() { if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then rm -f "$WORKSPACE/tmp/api.pid" fi + all_services_stopped= + fail=1 + cd "$WORKSPACE" \ - && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \ + && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ + && checkpidfile api \ + && checkdiscoverydoc $ARVADOS_API_HOST \ + && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ + && checkpidfile nginx \ && python sdk/python/tests/run_test_server.py start_controller \ + && checkpidfile controller \ + && checkhealth Controller \ + && checkdiscoverydoc $ARVADOS_API_HOST \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && checkpidfile keepproxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ + && checkpidfile keep-web \ + && checkhealth WebDAV \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ + && checkpidfile arv-git-httpd \ + && checkhealth GitHTTP \ && python sdk/python/tests/run_test_server.py start_ws \ - && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \ - && (env | egrep ^ARVADOS) - if [[ -n "$fail" ]]; then - return 1 + && checkpidfile ws \ + && export ARVADOS_TEST_PROXY_SERVICES=1 \ + && (env | egrep ^ARVADOS) \ + && fail=0 + deactivate + if [[ $fail != 0 ]]; then + unset ARVADOS_TEST_API_HOST fi + return $fail } stop_services() { - if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then + if [[ -n "$all_services_stopped" ]]; then return fi - unset ARVADOS_TEST_API_HOST + unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES + . "$VENVDIR/bin/activate" || return cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py stop_nginx \ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ @@ -390,7 +472,10 @@ stop_services() { && python sdk/python/tests/run_test_server.py stop_keep-web \ && python sdk/python/tests/run_test_server.py stop_keep_proxy \ && python sdk/python/tests/run_test_server.py stop_controller \ - && python sdk/python/tests/run_test_server.py stop + && python sdk/python/tests/run_test_server.py stop \ + && all_services_stopped=1 + deactivate + unset ARVADOS_CONFIG } interrupt() { @@ -399,36 +484,6 @@ interrupt() { } trap interrupt INT -sanity_checks - -echo "WORKSPACE=$WORKSPACE" - -if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then - # Jenkins expects us to use this by default. - CONFIGSRC="$HOME/arvados-api-server" -fi - -# Clean up .pyc files that may exist in the workspace -cd "$WORKSPACE" -find -name '*.pyc' -delete - -if [[ -z "$temp" ]]; then - temp="$(mktemp -d)" -fi - -# Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS -do - if [[ -z "${!tmpdir}" ]]; then - eval "$tmpdir"="$temp/$tmpdir" - fi - if ! [[ -d "${!tmpdir}" ]]; then - mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)" - fi -done - -rm -vf "${WORKSPACE}/tmp/*.log" - setup_ruby_environment() { if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then source "$HOME/.rvm/scripts/rvm" @@ -466,6 +521,10 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env + (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2 + bundle="$(which bundle)" + echo "$bundle" + "$bundle" version | grep 2.0.2 || fatal 'install bundler' else # When our "bundle install"s need to install new gems to # satisfy dependencies, we want them to go where "gem install @@ -491,9 +550,14 @@ setup_ruby_environment() { echo "Will install dependencies to $(gem env gemdir)" echo "Will install arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" + bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" + ( + export HOME=$GEMHOME + ("$bundle" version | grep -q 2.0.2) \ + || gem install --user bundler -v 2.0.2 + "$bundle" version | tee /dev/stderr | grep -q 'version 2' + ) || fatal 'install bundler' fi - bundle config || gem install bundler \ - || fatal 'install bundler' } with_test_gemset() { @@ -523,118 +587,100 @@ setup_virtualenv() { else "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi - # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' } -export PERLINSTALLBASE -export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" - -export R_LIBS - -export GOPATH -( - set -e - mkdir -p "$GOPATH/src/git.curoverse.com" - rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH" - if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -d "$d" ]] && rmdir "$d" - done +initialize() { + sanity_checks + + echo "WORKSPACE=$WORKSPACE" + + # Clean up .pyc files that may exist in the workspace + cd "$WORKSPACE" + find -name '*.pyc' -delete + + if [[ -z "$temp" ]]; then + temp="$(mktemp -d)" fi - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -h "$d" ]] && rm "$d" + + # Set up temporary install dirs (unless existing dirs were supplied) + for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS + do + if [[ -z "${!tmpdir}" ]]; then + eval "$tmpdir"="$temp/$tmpdir" + fi + if ! [[ -d "${!tmpdir}" ]]; then + mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)" + fi done - ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" - go get -v github.com/kardianos/govendor - cd "$GOPATH/src/git.curoverse.com/arvados.git" - if [[ -n "$short" ]]; then - go get -v -d ... - "$GOPATH/bin/govendor" sync - else - # Remove cached source dirs in workdir. Otherwise, they will - # not qualify as +missing or +external below, and we won't be - # able to detect that they're missing from vendor/vendor.json. - rm -rf vendor/*/ - go get -v -d ... - "$GOPATH/bin/govendor" sync - [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ - || fatal "vendor/vendor.json has unused or missing dependencies -- try: - -(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) - -"; - fi -) || fatal "Go setup failed" -setup_virtualenv "$VENVDIR" --python python2.7 -. "$VENVDIR/bin/activate" + rm -vf "${WORKSPACE}/tmp/*.log" -# Needed for run_test_server.py which is used by certain (non-Python) tests. -pip install --no-cache-dir PyYAML \ - || fatal "pip install PyYAML failed" + export PERLINSTALLBASE + export PERL5LIB="$PERLINSTALLBASE/lib/perl5${PERL5LIB:+:$PERL5LIB}" -# Preinstall libcloud if using a fork; otherwise nodemanager "pip -# install" won't pick it up by default. -if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then - pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ - || fatal "pip install apache-libcloud failed" -fi + export R_LIBS -# Deactivate Python 2 virtualenv -deactivate + export GOPATH + # Make sure our compiled binaries under test override anything + # else that might be in the environment. + export PATH=$GOPATH/bin:$PATH -declare -a pythonstuff -pythonstuff=( - sdk/pam - sdk/python - sdk/python:py3 - sdk/cwl - services/dockercleaner:py3 - services/fuse - services/nodemanager - tools/crunchstat-summary - ) + # Jenkins config requires that glob tmp/*.log match something. Ensure + # that happens even if we don't end up running services that set up + # logging. + mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp" + touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log" -# If Python 3 is available, set up its virtualenv in $VENV3DIR. -# Otherwise, skip dependent tests. -PYTHON3=$(which python3) -if [[ ${?} = 0 ]]; then - setup_virtualenv "$VENV3DIR" --python python3 -else - PYTHON3= - cat >&2 </dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" -# Note: this must be the last time we change PATH, otherwise rvm will -# whine a lot. -setup_ruby_environment + setup_virtualenv "$VENVDIR" --python python2.7 + . "$VENVDIR/bin/activate" -echo "PATH is $PATH" + # Needed for run_test_server.py which is used by certain (non-Python) tests. + ( + set -e + "${VENVDIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip" install PyYAML + cd "$WORKSPACE/sdk/python" + python setup.py install + ) || fatal "installing PyYAML and sdk/python failed" + + # Preinstall libcloud if using a fork; otherwise nodemanager "pip + # install" won't pick it up by default. + if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then + pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ + || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ + || fatal "pip install apache-libcloud failed" + fi -if ! which bundler >/dev/null -then - gem install --user-install bundler || fatal 'Could not install bundler' -fi + # Deactivate Python 2 virtualenv + deactivate -# Jenkins config requires that glob tmp/*.log match something. Ensure -# that happens even if we don't end up running services that set up -# logging. -mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp" -touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log" + # If Python 3 is available, set up its virtualenv in $VENV3DIR. + # Otherwise, skip dependent tests. + PYTHON3=$(which python3) + if [[ ${?} = 0 ]]; then + setup_virtualenv "$VENV3DIR" --python python3 + else + PYTHON3= + cat >&2 </dev/null; then deactivate; fi + if ! . "$VENVDIR/bin/activate" + then + result=1 + elif [[ "$2" == "go" ]] then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") - # We do "go get -t" here to catch compilation errors + # We do "go install" here to catch compilation errors # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ - cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ - [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \ + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" && \ + cd "$WORKSPACE/$1" && \ if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this @@ -705,7 +777,7 @@ do_test_once() { else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1" + go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1" fi result=${result:-$?} if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]] @@ -713,6 +785,7 @@ do_test_once() { go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html" rm "$WORKSPACE/tmp/.$covername.tmp" fi + [[ $result = 0 ]] && gofmt -e -d *.go elif [[ "$2" == "pip" ]] then tries=0 @@ -721,7 +794,10 @@ do_test_once() { tries=$((${tries}+1)) # $3 can name a path directory for us to use, including trailing # slash; e.g., the bin/ subdirectory of a virtualenv. - "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]} + if [[ -e "${3}activate" ]]; then + . "${3}activate" + fi + python setup.py ${short:+--short-tests-only} test ${testargs[$1]} result=$? if [[ ${tries} < 3 && ${result} == 137 ]] then @@ -739,30 +815,47 @@ do_test_once() { fi result=${result:-$?} checkexit $result "$1 tests" - title "End of $1 tests (`timer`)" + title "test $1 -- `timer`" return $result } -do_install() { - skipit=false - - if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then - retry do_install_once ${@} - else - skipit=true - fi +check_arvados_config() { + if [[ "$1" = "env" ]] ; then + return + fi + if [[ -z "$ARVADOS_CONFIG" ]] ; then + # Create config file. The run_test_server script requires PyYAML, + # so virtualenv needs to be active. Downstream steps like + # workbench install which require a valid config.yml. + if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + install_env + fi + . "$VENVDIR/bin/activate" + cd "$WORKSPACE" + eval $(python sdk/python/tests/run_test_server.py setup_config) + deactivate + fi +} - if [[ "$skipit" = true ]]; then - title "Skipping $1 install" - fi +do_install() { + if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then + return 0 + fi + check_arvados_config "$1" + retry do_install_once ${@} } do_install_once() { - title "Running $1 install" + title "install $1" timer_reset - if [[ "$2" == "go" ]] + + result= + if which deactivate >/dev/null; then deactivate; fi + if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then + result=1 + elif [[ "$2" == "go" ]] then - go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -780,17 +873,17 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" else "install_$1" fi - result=$? + result=${result:-$?} checkexit $result "$1 install" - title "End of $1 install (`timer`)" + title "install $1 -- `timer`" return $result } @@ -798,11 +891,11 @@ bundle_install_trylocal() { ( set -e echo "(Running bundle install --local. 'could not find package' messages are OK.)" - if ! bundle install --local --no-deployment; then + if ! "$bundle" install --local --no-deployment; then echo "(Running bundle install again, without --local.)" - bundle install --no-deployment + "$bundle" install --no-deployment fi - bundle package --all + "$bundle" package ) } @@ -811,7 +904,6 @@ install_doc() { && bundle_install_trylocal \ && rm -rf .site } -do_install doc install_gem() { gemname=$1 @@ -820,78 +912,49 @@ install_gem() { && cd "$WORKSPACE/$srcpath" \ && bundle_install_trylocal \ && gem build "$gemname.gemspec" \ - && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1) + && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1) } -install_ruby_sdk() { +install_sdk/ruby() { install_gem arvados sdk/ruby } -do_install sdk/ruby ruby_sdk -install_R_sdk() { +install_sdk/R() { if [[ "$NEED_SDK_R" = true ]]; then cd "$WORKSPACE/sdk/R" \ && Rscript --vanilla install_deps.R fi } -do_install sdk/R R_sdk -install_perl_sdk() { +install_sdk/perl() { cd "$WORKSPACE/sdk/perl" \ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ && make install INSTALLDIRS=perl } -do_install sdk/perl perl_sdk -install_cli() { +install_sdk/cli() { install_gem arvados-cli sdk/cli } -do_install sdk/cli cli -install_login-sync() { +install_services/login-sync() { install_gem arvados-login-sync services/login-sync } -do_install services/login-sync login-sync - -# Install the Python SDK early. Various other test suites (like -# keepproxy) bring up run_test_server.py, which imports the arvados -# module. We can't actually *test* the Python SDK yet though, because -# its own test suite brings up some of those other programs (like -# keepproxy). -for p in "${pythonstuff[@]}" -do - dir=${p%:py3} - if [[ ${dir} = ${p} ]]; then - if [[ -z ${skip[python2]} ]]; then - do_install ${dir} pip - fi - elif [[ -n ${PYTHON3} ]]; then - if [[ -z ${skip[python3]} ]]; then - do_install ${dir} pip "$VENV3DIR/bin/" - fi - fi -done -install_apiserver() { +install_services/api() { + stop_services + check_arvados_config "services/api" cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle_install_trylocal + && RAILS_ENV=test bundle_install_trylocal \ + || return 1 rm -f config/environments/test.rb cp config/environments/test.rb.example config/environments/test.rb - if [ -n "$CONFIGSRC" ] - then - for f in database.yml - do - cp "$CONFIGSRC/$f" config/ || fatal "$f" - done - fi - # Clear out any lingering postgresql connections to the test # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \ + && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -915,182 +978,320 @@ install_apiserver() { && git --git-dir internal.git init \ || return 1 - cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle exec rake db:drop \ - && RAILS_ENV=test bundle exec rake db:setup \ - && RAILS_ENV=test bundle exec rake db:fixtures:load + ( + set -e + cd "$WORKSPACE/services/api" + export RAILS_ENV=test + if "$bundle" exec rails db:environment:set ; then + "$bundle" exec rake db:drop + fi + "$bundle" exec rake db:setup + "$bundle" exec rake db:fixtures:load + ) || return 1 } -do_install services/api apiserver -declare -a gostuff -gostuff=( - cmd/arvados-client - cmd/arvados-server - lib/cli - lib/cmd - lib/controller - lib/crunchstat - lib/cloud - lib/dispatchcloud - lib/dispatchcloud/container - lib/dispatchcloud/scheduler - lib/dispatchcloud/ssh_executor - lib/dispatchcloud/worker - sdk/go/arvados - sdk/go/arvadosclient - sdk/go/auth - sdk/go/blockdigest - sdk/go/dispatch - sdk/go/health - sdk/go/httpserver - sdk/go/manifest - sdk/go/asyncbuf - sdk/go/crunchrunner - sdk/go/stats - services/arv-git-httpd - services/crunchstat - services/health - services/keep-web - services/keepstore - sdk/go/keepclient - services/keep-balance - services/keepproxy - services/crunch-dispatch-local - services/crunch-dispatch-slurm - services/crunch-run - services/ws - tools/keep-block-check - tools/keep-exercise - tools/keep-rsync - tools/sync-groups +declare -a pythonstuff +pythonstuff=( + sdk/pam + sdk/python + sdk/python:py3 + sdk/cwl:py3 + services/dockercleaner:py3 + services/fuse + services/fuse:py3 + services/nodemanager + tools/crunchstat-summary + tools/crunchstat-summary:py3 ) -for g in "${gostuff[@]}" -do - do_install "$g" go -done -install_workbench() { +declare -a gostuff +gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u)) + +install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ && mkdir -p tmp/cache \ && RAILS_ENV=test bundle_install_trylocal \ - && RAILS_ENV=test RAILS_GROUPS=assets bundle exec rake npm:install + && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install } -do_install apps/workbench workbench - -unset http_proxy https_proxy no_proxy -test_doclinkchecker() { +test_doc() { ( set -e cd "$WORKSPACE/doc" ARVADOS_API_HOST=qr1hi.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! - PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST + PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST ) } -do_test doc doclinkchecker -stop_services +test_gofmt() { + cd "$WORKSPACE" || return 1 + dirs=$(ls -d */ | egrep -v 'vendor|tmp') + [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]] +} -test_apiserver() { +test_services/api() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=-v ${testargs[services/api]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]} } -do_test services/api apiserver - -# Shortcut for when we're only running apiserver tests. This saves a bit of time, -# because we don't need to start up the api server for subsequent tests. -if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then - rotate_logfile "$WORKSPACE/services/api/log/" "test.log" - exit_cleanly -fi - -start_services || { stop_services; fatal "start_services"; } -test_ruby_sdk() { +test_sdk/ruby() { cd "$WORKSPACE/sdk/ruby" \ - && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} } -do_test sdk/ruby ruby_sdk -test_R_sdk() { +test_sdk/R() { if [[ "$NEED_SDK_R" = true ]]; then cd "$WORKSPACE/sdk/R" \ && Rscript --vanilla run_test.R fi } -do_test sdk/R R_sdk - -test_cli() { +test_sdk/cli() { cd "$WORKSPACE/sdk/cli" \ && mkdir -p /tmp/keep \ - && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]} + && KEEP_LOCAL_STORE=/tmp/keep "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/cli]} +} + +test_sdk/java-v2() { + cd "$WORKSPACE/sdk/java-v2" && gradle test } -do_test sdk/cli cli -test_login-sync() { +test_services/login-sync() { cd "$WORKSPACE/services/login-sync" \ - && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]} } -do_test services/login-sync login-sync -test_nodemanager_integration() { +test_services/nodemanager_integration() { cd "$WORKSPACE/services/nodemanager" \ && tests/integration_test.py ${testargs[services/nodemanager_integration]} } -do_test services/nodemanager_integration nodemanager_integration -for p in "${pythonstuff[@]}" -do - dir=${p%:py3} - if [[ ${dir} = ${p} ]]; then - if [[ -z ${skip[python2]} ]]; then - do_test ${dir} pip - fi - elif [[ -n ${PYTHON3} ]]; then - if [[ -z ${skip[python3]} ]]; then - do_test ${dir} pip "$VENV3DIR/bin/" - fi - fi -done +test_apps/workbench_units() { + local TASK="test:units" + cd "$WORKSPACE/apps/workbench" \ + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} +} -for g in "${gostuff[@]}" -do - do_test "$g" go -done +test_apps/workbench_functionals() { + local TASK="test:functionals" + cd "$WORKSPACE/apps/workbench" \ + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} +} -test_workbench_units() { +test_apps/workbench_integration() { + local TASK="test:integration" cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} } -do_test apps/workbench_units workbench_units -test_workbench_functionals() { +test_apps/workbench_benchmark() { + local TASK="test:benchmark" cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_benchmark]} } -do_test apps/workbench_functionals workbench_functionals -test_workbench_integration() { +test_apps/workbench_profile() { + local TASK="test:profile" cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_profile]} } -do_test apps/workbench_integration workbench_integration +install_deps() { + # Install parts needed by test suites + do_install env + do_install cmd/arvados-server go + do_install sdk/cli + do_install sdk/perl + do_install sdk/python pip + do_install sdk/python pip "${VENV3DIR}/bin/" + do_install sdk/ruby + do_install services/api + do_install services/arv-git-httpd go + do_install services/keepproxy go + do_install services/keepstore go + do_install services/keep-web go + do_install services/ws go +} -test_workbench_benchmark() { - cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]} +install_all() { + do_install env + do_install doc + do_install sdk/ruby + do_install sdk/R + do_install sdk/perl + do_install sdk/cli + do_install services/login-sync + for p in "${pythonstuff[@]}" + do + dir=${p%:py3} + if [[ ${dir} = ${p} ]]; then + if [[ -z ${skip[python2]} ]]; then + do_install ${dir} pip + fi + elif [[ -n ${PYTHON3} ]]; then + if [[ -z ${skip[python3]} ]]; then + do_install ${dir} pip "$VENV3DIR/bin/" + fi + fi + done + for g in "${gostuff[@]}" + do + do_install "$g" go + done + do_install services/api + do_install apps/workbench } -do_test apps/workbench_benchmark workbench_benchmark -test_workbench_profile() { - cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]} +test_all() { + stop_services + do_test services/api + + # Shortcut for when we're only running apiserver tests. This saves a bit of time, + # because we don't need to start up the api server for subsequent tests. + if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then + rotate_logfile "$WORKSPACE/services/api/log/" "test.log" + exit_cleanly + fi + + do_test gofmt + do_test doc + do_test sdk/ruby + do_test sdk/R + do_test sdk/cli + do_test services/login-sync + do_test sdk/java-v2 + do_test services/nodemanager_integration + for p in "${pythonstuff[@]}" + do + dir=${p%:py3} + if [[ ${dir} = ${p} ]]; then + if [[ -z ${skip[python2]} ]]; then + do_test ${dir} pip + fi + elif [[ -n ${PYTHON3} ]]; then + if [[ -z ${skip[python3]} ]]; then + do_test ${dir} pip "$VENV3DIR/bin/" + fi + fi + done + + for g in "${gostuff[@]}" + do + do_test "$g" go + done + do_test apps/workbench_units + do_test apps/workbench_functionals + do_test apps/workbench_integration + do_test apps/workbench_benchmark + do_test apps/workbench_profile +} + +help_interactive() { + echo "== Interactive commands:" + echo "TARGET (short for 'test DIR')" + echo "test TARGET" + echo "test TARGET:py3 (test with python3)" + echo "test TARGET -check.vv (pass arguments to test)" + echo "install TARGET" + echo "install env (go/python libs)" + echo "install deps (go/python libs + arvados components needed for integration tests)" + echo "reset (...services used by integration tests)" + echo "exit" + echo "== Test targets:" + echo "${!testfuncargs[@]}" | tr ' ' '\n' | sort | column } -do_test apps/workbench_profile workbench_profile +initialize + +declare -A testfuncargs=() +for g in "${gostuff[@]}"; do + testfuncargs[$g]="$g go" +done +for p in "${pythonstuff[@]}"; do + dir=${p%:py3} + testfuncargs[$dir]="$dir pip $VENVDIR/bin/" + testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/" +done + +testfuncargs["sdk/cli"]="sdk/cli" +testfuncargs["sdk/R"]="sdk/R" +testfuncargs["sdk/java-v2"]="sdk/java-v2" +testfuncargs["apps/workbench_units"]="apps/workbench_units" +testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals" +testfuncargs["apps/workbench_integration"]="apps/workbench_integration" +testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark" +testfuncargs["apps/workbench_profile"]="apps/workbench_profile" + +if [[ -z ${interactive} ]]; then + install_all + test_all +else + skip=() + only=() + only_install=() + if [[ -e "$VENVDIR/bin/activate" ]]; then stop_services; fi + setnextcmd() { + if [[ "$TERM" = dumb ]]; then + # assume emacs, or something, is offering a history buffer + # and pre-populating the command will only cause trouble + nextcmd= + elif [[ ! -e "$VENVDIR/bin/activate" ]]; then + nextcmd="install deps" + else + nextcmd="" + fi + } + echo + help_interactive + nextcmd="install deps" + setnextcmd + HISTFILE="$WORKSPACE/tmp/.history" + history -r + while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do + history -s "$nextcmd" + history -w + read verb target opts <<<"${nextcmd}" + target="${target%/}" + target="${target/\/:/:}" + case "${verb}" in + "exit" | "quit") + exit_cleanly + ;; + "reset") + stop_services + ;; + "test" | "install") + case "$target" in + "") + help_interactive + ;; + all | deps) + ${verb}_${target} + ;; + *) + argstarget=${target%:py3} + testargs["$argstarget"]="${opts}" + tt="${testfuncargs[${target}]}" + tt="${tt:-$target}" + do_$verb $tt + ;; + esac + ;; + "" | "help" | *) + help_interactive + ;; + esac + if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then + report_outcomes + successes=() + failures=() + fi + cd "$WORKSPACE" + setnextcmd + done + echo +fi exit_cleanly