X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/50b36937d0bd921a9c3dc27080e0a3d7591b465a..3a126e4c7e3ab4235a8d5f84f5addebb4691ac7a:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index ebdf4c413d..edf9f3ab23 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -19,6 +19,10 @@ Syntax: Options: --skip FOO Do not test the FOO component. +--skip sanity Skip initial dev environment sanity checks. +--skip install Do not run any install steps. Just run tests. + You should provide GOPATH, GEMHOME, and VENVDIR options + from a previous invocation if you use this option. --only FOO Do not test anything except the FOO component. --temp DIR Install components and dependencies under DIR instead of making a new temporary directory. Implies --leave-temp. @@ -27,17 +31,15 @@ Options: subsequent invocations. --repeat N Repeat each install/test step until it succeeds N times. --retry Prompt to retry if an install or test suite fails. ---skip-install Do not run any install steps. Just run tests. - You should provide GOPATH, GEMHOME, and VENVDIR options - from a previous invocation if you use this option. --only-install Run specific install step --short Skip (or scale down) some slow tests. +--interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. CONFIGSRC=path Dir with api server config files to copy into source tree. (If none given, leave config files alone in source tree.) services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file -sdk/python_test="--test-suite test.test_keep_locator" +sdk/python_test="--test-suite tests.test_keep_locator" Restrict Python SDK tests to the given class apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb" Restrict Workbench tests to the given file @@ -49,7 +51,7 @@ ARVADOS_DEBUG=1 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE, *_test, and other examples shown above. -Assuming --skip-install is not given, all components are installed +Assuming "--skip install" is not given, all components are installed into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many test suites depend on other components being installed, and installing everything tends to be quicker than debugging dependencies. @@ -69,7 +71,25 @@ apps/workbench_functionals (*) apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile +cmd/arvados-client +cmd/arvados-server doc +lib/cli +lib/cmd +lib/controller +lib/controller/federation +lib/controller/railsproxy +lib/controller/router +lib/controller/rpc +lib/crunchstat +lib/cloud +lib/cloud/azure +lib/dispatchcloud +lib/dispatchcloud/container +lib/dispatchcloud/scheduler +lib/dispatchcloud/ssh_executor +lib/dispatchcloud/worker +lib/service services/api services/arv-git-httpd services/crunchstat @@ -89,11 +109,13 @@ services/crunch-dispatch-slurm services/ws sdk/cli sdk/pam +sdk/pam:py3 sdk/python sdk/python:py3 sdk/ruby sdk/go/arvados sdk/go/arvadosclient +sdk/go/auth sdk/go/dispatch sdk/go/keepclient sdk/go/health @@ -104,8 +126,11 @@ sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner sdk/cwl +sdk/R +sdk/java-v2 tools/sync-groups tools/crunchstat-summary +tools/crunchstat-summary:py3 tools/keep-exercise tools/keep-rsync tools/keep-block-check @@ -119,6 +144,8 @@ EOF # environment that could interfere with the tests. unset $(env | cut -d= -f1 | grep \^ARVADOS_) +export ARVADOS_EXPERIMENTAL=14287 + # Reset other variables that could affect our [tests'] behavior by # accident. GITDIR= @@ -128,6 +155,8 @@ VENV3DIR= PYTHONPATH= GEMHOME= PERLINSTALLBASE= +R_LIBS= +export LANG=en_US.UTF-8 short= only_install= @@ -153,7 +182,9 @@ fatal() { exit_cleanly() { trap - INT - create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/" + if which create-plot-data-from-log.sh >/dev/null; then + create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/" + fi rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log" stop_services rotate_logfile "$WORKSPACE/services/api/log/" "test.log" @@ -163,9 +194,13 @@ exit_cleanly() { } sanity_checks() { + [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" echo Checking dependencies: + echo "locale: ${LANG}" + [[ "$(locale charmap)" = "UTF-8" ]] \ + || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen" echo -n 'virtualenv: ' virtualenv --version \ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" @@ -175,8 +210,8 @@ sanity_checks() { echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \ - || fatal "Go >= 1.8 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \ + || fatal "Go >= 1.10 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -235,6 +270,20 @@ sanity_checks() { which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" + echo -n 'geckodriver: ' + geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver" + + if [[ "$NEED_SDK_R" = true ]]; then + # R SDK stuff + echo -n 'R: ' + which Rscript || fatal "No Rscript. Try: apt-get install r-base" + echo -n 'testthat: ' + Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat" + # needed for roxygen2, needed for devtools, needed for R sdk + pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev" + # needed for pkgdown, builds R SDK doc pages + which pandoc || fatal "No pandoc. Try: apt-get install pandoc" + fi } rotate_logfile() { @@ -249,8 +298,11 @@ rotate_logfile() { declare -a failures declare -A skip +declare -A only declare -A testargs skip[apps/workbench_profile]=1 +# nodemanager_integration tests are not reliable, see #12061. +skip[services/nodemanager_integration]=1 while [[ -n "$1" ]] do @@ -265,13 +317,16 @@ do skip[$1]=1; shift ;; --only) - only="$1"; skip[$1]=""; shift + only[$1]=1; skip[$1]=""; shift ;; --short) short=1 ;; + --interactive) + interactive=1 + ;; --skip-install) - only_install=nothing + skip[install]=1 ;; --only-install) only_install="$1"; shift @@ -304,8 +359,64 @@ do esac done -start_api() { - echo 'Starting API server...' +# R SDK installation is very slow (~360s in a clean environment) and only +# required when testing it. Skip that step if it is not needed. +NEED_SDK_R=true + +if [[ ${#only[@]} -ne 0 ]] && + [[ -z "${only['sdk/R']}" && -z "${only['doc']}" ]]; then + NEED_SDK_R=false +fi + +if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then + NEED_SDK_R=false +fi + +if [[ $NEED_SDK_R == false ]]; then + echo "R SDK not needed, it will not be installed." +fi + +checkpidfile() { + svc="$1" + pid="$(cat "$WORKSPACE/tmp/${svc}.pid")" + if [[ -z "$pid" ]] || ! kill -0 "$pid"; then + tail $WORKSPACE/tmp/${1}*.log + echo "${svc} pid ${pid} not running" + return 1 + fi + echo "${svc} pid ${pid} ok" +} + +checkhealth() { + svc="$1" + port="$(cat "$WORKSPACE/tmp/${svc}.port")" + scheme=http + if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then + scheme=https + fi + url="$scheme://localhost:${port}/_health/ping" + if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then + echo "${url} failed" + return 1 + fi +} + +checkdiscoverydoc() { + dd="https://${1}/discovery/v1/apis/arvados/v1/rest" + if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then + echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd" + tail -v $WORKSPACE/services/api/log/test.log + return 1 + fi + echo "${dd} ok" +} + +start_services() { + if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then + return 0 + fi + . "$VENVDIR/bin/activate" + echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -313,43 +424,56 @@ start_api() { if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then rm -f "$WORKSPACE/tmp/api.pid" fi + all_services_stopped= + fail=1 cd "$WORKSPACE" \ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ - && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && (env | egrep ^ARVADOS) -} - -start_nginx_proxy_services() { - echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' - cd "$WORKSPACE" \ + && checkpidfile api \ + && checkdiscoverydoc $ARVADOS_API_HOST \ + && python sdk/python/tests/run_test_server.py start_controller \ + && checkpidfile controller \ + && checkhealth controller \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && checkpidfile keepproxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ + && checkpidfile keep-web \ + && checkhealth keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ + && checkpidfile arv-git-httpd \ + && checkhealth arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_ws \ - && python sdk/python/tests/run_test_server.py start_nginx \ - && export ARVADOS_TEST_PROXY_SERVICES=1 + && checkpidfile ws \ + && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ + && checkdiscoverydoc $ARVADOS_API_HOST \ + && checkpidfile nginx \ + && export ARVADOS_TEST_PROXY_SERVICES=1 \ + && (env | egrep ^ARVADOS) \ + && fail=0 + deactivate + if [[ $fail != 0 ]]; then + unset ARVADOS_TEST_API_HOST + fi + return $fail } stop_services() { - if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then - unset ARVADOS_TEST_PROXY_SERVICES - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy - fi - if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then - unset ARVADOS_TEST_API_HOST - cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop + if [[ -n "$all_services_stopped" ]]; then + return fi + unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES + . "$VENVDIR/bin/activate" || return + cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py stop_nginx \ + && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_ws \ + && python sdk/python/tests/run_test_server.py stop_keep-web \ + && python sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python sdk/python/tests/run_test_server.py stop_controller \ + && python sdk/python/tests/run_test_server.py stop \ + && all_services_stopped=1 + deactivate } interrupt() { @@ -358,34 +482,6 @@ interrupt() { } trap interrupt INT -sanity_checks - -echo "WORKSPACE=$WORKSPACE" - -if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then - # Jenkins expects us to use this by default. - CONFIGSRC="$HOME/arvados-api-server" -fi - -# Clean up .pyc files that may exist in the workspace -cd "$WORKSPACE" -find -name '*.pyc' -delete - -if [[ -z "$temp" ]]; then - temp="$(mktemp -d)" -fi - -# Set up temporary install dirs (unless existing dirs were supplied) -for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE -do - if [[ -z "${!tmpdir}" ]]; then - eval "$tmpdir"="$temp/$tmpdir" - fi - if ! [[ -d "${!tmpdir}" ]]; then - mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)" - fi -done - setup_ruby_environment() { if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then source "$HOME/.rvm/scripts/rvm" @@ -471,100 +567,149 @@ setup_virtualenv() { local venvdest="$1"; shift if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + elif [[ -n "$short" ]]; then + return fi if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' else - "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7' + "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' -} - -export PERLINSTALLBASE -export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}" - -export GOPATH -mkdir -p "$GOPATH/src/git.curoverse.com" -rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" -ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \ - || fatal "symlink failed" -go get -v github.com/kardianos/govendor \ - || fatal "govendor install failed" - -setup_virtualenv "$VENVDIR" --python python2.7 -. "$VENVDIR/bin/activate" - -# Needed for run_test_server.py which is used by certain (non-Python) tests. -pip freeze 2>/dev/null | egrep ^PyYAML= \ - || pip install PyYAML >/dev/null \ - || fatal "pip install PyYAML failed" - -# Preinstall forked version of libcloud, because nodemanager "pip install" -# won't pick it up by default. -pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ - || fatal "pip install apache-libcloud failed" - -# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks. -pip freeze | grep -x llfuse==1.2.0 || ( - set -e - yes | pip uninstall llfuse || true - cython --version || fatal "no cython; try sudo apt-get install cython" - cd "$temp" - (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse) - cd python-llfuse - git checkout 620722fd990ea642ddb8e7412676af482c090c0c - git checkout setup.py - sed -i -e "s:'1\\.2':'1.2.0':" setup.py - python setup.py build_cython - python setup.py install --force -) || fatal "llfuse fork failed" -pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???" - -# Deactivate Python 2 virtualenv -deactivate + "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' +} -declare -a pythonstuff -pythonstuff=( - sdk/pam - sdk/python - sdk/python:py3 - sdk/cwl - services/dockercleaner:py3 - services/fuse - services/nodemanager - tools/crunchstat-summary - ) +initialize() { + sanity_checks -# If Python 3 is available, set up its virtualenv in $VENV3DIR. -# Otherwise, skip dependent tests. -PYTHON3=$(which python3) -if [[ ${?} = 0 ]]; then - setup_virtualenv "$VENV3DIR" --python python3 -else - PYTHON3= - cat >&2 </dev/null -then - gem install --user-install bundler || fatal 'Could not install bundler' -fi + export PERLINSTALLBASE + export PERL5LIB="$PERLINSTALLBASE/lib/perl5${PERL5LIB:+:$PERL5LIB}" + + export R_LIBS + + export GOPATH + + # Jenkins config requires that glob tmp/*.log match something. Ensure + # that happens even if we don't end up running services that set up + # logging. + mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp" + touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log" + + unset http_proxy https_proxy no_proxy + + + # Note: this must be the last time we change PATH, otherwise rvm will + # whine a lot. + setup_ruby_environment + + echo "PATH is $PATH" +} + +install_env() { + ( + set -e + mkdir -p "$GOPATH/src/git.curoverse.com" + if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then + for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \ + "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -d "$d" ]] && rmdir "$d" + done + fi + for d in \ + "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \ + "$GOPATH/src/git.curoverse.com/arvados.git"; do + [[ -h "$d" ]] && rm "$d" + done + ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" + go get -v github.com/kardianos/govendor + cd "$GOPATH/src/git.curoverse.com/arvados.git" + if [[ -n "$short" ]]; then + go get -v -d ... + "$GOPATH/bin/govendor" sync + else + # Remove cached source dirs in workdir. Otherwise, they will + # not qualify as +missing or +external below, and we won't be + # able to detect that they're missing from vendor/vendor.json. + rm -rf vendor/*/ + go get -v -d ... + "$GOPATH/bin/govendor" sync + [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \ + || fatal "vendor/vendor.json has unused or missing dependencies -- try: + +(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) + +"; + fi + ) || fatal "Go setup failed" + + setup_virtualenv "$VENVDIR" --python python2.7 + . "$VENVDIR/bin/activate" + + # Needed for run_test_server.py which is used by certain (non-Python) tests. + pip install --no-cache-dir PyYAML \ + || fatal "pip install PyYAML failed" + + # Preinstall libcloud if using a fork; otherwise nodemanager "pip + # install" won't pick it up by default. + if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then + pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ + || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ + || fatal "pip install apache-libcloud failed" + fi + + # Deactivate Python 2 virtualenv + deactivate + + # If Python 3 is available, set up its virtualenv in $VENV3DIR. + # Otherwise, skip dependent tests. + PYTHON3=$(which python3) + if [[ ${?} = 0 ]]; then + setup_virtualenv "$VENV3DIR" --python python3 + else + PYTHON3= + cat >&2 </dev/null + then + gem install --user-install bundler || fatal 'Could not install bundler' + fi +} retry() { remain="${repeat}" @@ -573,7 +718,7 @@ retry() { if ${@}; then if [[ "$remain" -gt 1 ]]; then remain=$((${remain}-1)) - title "Repeating ${remain} more times" + title "(repeating ${remain} more times)" else break fi @@ -601,21 +746,40 @@ do_test() { suite="${1}" ;; esac - if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \ - (-z "${only}" || "${only}" == "${suite}" || \ - "${only}" == "${1}") ]]; then - retry do_test_once ${@} - else - title "Skipping ${1} tests" + if [[ -n "${skip[$suite]}" || \ + -n "${skip[$1]}" || \ + (${#only[@]} -ne 0 && ${only[$suite]} -eq 0 && ${only[$1]} -eq 0) ]]; then + return 0 fi + case "${1}" in + services/api) + stop_services + ;; + gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + # don't care whether services are running + ;; + *) + if ! start_services; then + title "test $1 -- failed to start services" + return 1 + fi + ;; + esac + retry do_test_once ${@} } do_test_once() { unset result - title "Running $1 tests" + title "test $1" timer_reset - if [[ "$2" == "go" ]] + + result= + if which deactivate >/dev/null; then deactivate; fi + if ! . "$VENVDIR/bin/activate" + then + result=1 + elif [[ "$2" == "go" ]] then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") @@ -623,9 +787,8 @@ do_test_once() { # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -t "git.curoverse.com/arvados.git/$1" && \ + go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ - [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \ if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this @@ -642,6 +805,7 @@ do_test_once() { go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html" rm "$WORKSPACE/tmp/.$covername.tmp" fi + [[ $result = 0 ]] && gofmt -e -d *.go elif [[ "$2" == "pip" ]] then tries=0 @@ -668,24 +832,28 @@ do_test_once() { fi result=${result:-$?} checkexit $result "$1 tests" - title "End of $1 tests (`timer`)" + title "test $1 -- `timer`" return $result } do_install() { - if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then - retry do_install_once ${@} - else - title "Skipping $1 install" + if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then + return 0 fi + retry do_install_once ${@} } do_install_once() { - title "Running $1 install" + title "install $1" timer_reset - if [[ "$2" == "go" ]] + + result= + if which deactivate >/dev/null; then deactivate; fi + if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then + result=1 + elif [[ "$2" == "go" ]] then - go get -t "git.curoverse.com/arvados.git/$1" + go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -703,17 +871,17 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" else "install_$1" fi - result=$? + result=${result:-$?} checkexit $result "$1 install" - title "End of $1 install (`timer`)" + title "install $1 -- `timer`" return $result } @@ -734,7 +902,6 @@ install_doc() { && bundle_install_trylocal \ && rm -rf .site } -do_install doc install_gem() { gemname=$1 @@ -743,51 +910,35 @@ install_gem() { && cd "$WORKSPACE/$srcpath" \ && bundle_install_trylocal \ && gem build "$gemname.gemspec" \ - && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1) + && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1) } -install_ruby_sdk() { +install_sdk/ruby() { install_gem arvados sdk/ruby } -do_install sdk/ruby ruby_sdk -install_perl_sdk() { +install_sdk/R() { + if [[ "$NEED_SDK_R" = true ]]; then + cd "$WORKSPACE/sdk/R" \ + && Rscript --vanilla install_deps.R + fi +} + +install_sdk/perl() { cd "$WORKSPACE/sdk/perl" \ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \ && make install INSTALLDIRS=perl } -do_install sdk/perl perl_sdk -install_cli() { +install_sdk/cli() { install_gem arvados-cli sdk/cli } -do_install sdk/cli cli -install_login-sync() { +install_services/login-sync() { install_gem arvados-login-sync services/login-sync } -do_install services/login-sync login-sync -# Install the Python SDK early. Various other test suites (like -# keepproxy) bring up run_test_server.py, which imports the arvados -# module. We can't actually *test* the Python SDK yet though, because -# its own test suite brings up some of those other programs (like -# keepproxy). -for p in "${pythonstuff[@]}" -do - dir=${p%:py3} - if [[ ${dir} = ${p} ]]; then - if [[ -z ${skip[python2]} ]]; then - do_install ${dir} pip - fi - elif [[ -n ${PYTHON3} ]]; then - if [[ -z ${skip[python3]} ]]; then - do_install ${dir} pip "$VENV3DIR/bin/" - fi - fi -done - -install_apiserver() { +install_services/api() { cd "$WORKSPACE/services/api" \ && RAILS_ENV=test bundle_install_trylocal @@ -806,67 +957,62 @@ install_apiserver() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \ - && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null + && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \ + && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" + cert="$WORKSPACE/services/api/tmp/self-signed" + if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then + ( + dir="$WORKSPACE/services/api/tmp" + set -ex + openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"