X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f3e3a6cc4b72120f856e14f3039b1c0c1f0326bd..8a27fe370239ecb8e50d53f46b45ed61203a35ca:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 4d76589336..4fbb4e6f04 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: AGPL-3.0 -. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh - COLUMNS=80 . `dirname "$(readlink -f "$0")"`/run-library.sh @@ -42,7 +40,7 @@ sdk/python_test="--test-suite tests.test_keep_locator" Restrict Python SDK tests to the given class apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb" Restrict Workbench tests to the given file -services/arv-git-httpd_test="-check.vv" +services/githttpd_test="-check.vv" Show all log messages, even when tests pass (also works with services/keepstore_test etc.) ARVADOS_DEBUG=1 @@ -88,13 +86,13 @@ lib/cloud/cloudtest lib/dispatchcloud lib/dispatchcloud/container lib/dispatchcloud/scheduler -lib/dispatchcloud/ssh_executor +lib/dispatchcloud/sshexecutor lib/dispatchcloud/worker lib/mount lib/pam lib/service services/api -services/arv-git-httpd +services/githttpd services/crunchstat services/dockercleaner services/fuse @@ -160,6 +158,8 @@ only_install= temp= temp_preserve= +ignore_sigint= + clear_temp() { if [[ -z "$temp" ]]; then # we did not even get as far as making a temp dir @@ -203,9 +203,6 @@ sanity_checks() { echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen" - echo -n 'virtualenv: ' - virtualenv --version \ - || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" echo -n 'ruby: ' ruby -v \ || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" @@ -223,9 +220,9 @@ sanity_checks() { echo -n 'gnutls.h: ' find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev" - echo -n 'Python2 pyconfig.h: ' - find /usr/include -path '*/python2*/pyconfig.h' | egrep --max-count=1 . \ - || fatal "No Python2 pyconfig.h. Try: apt-get install python2.7-dev" + echo -n 'virtualenv: ' + python3 -m venv -h | egrep --max-count=1 . \ + || fatal "No virtualenv. Try: apt-get install python3-venv" echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" @@ -247,7 +244,7 @@ sanity_checks() { || fatal "No gitolite. Try: apt-get install gitolite3" echo -n 'npm: ' npm --version \ - || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/" + || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/" echo -n 'cadaver: ' cadaver --version | grep -w cadaver \ || fatal "No cadaver. Try: apt-get install cadaver" @@ -272,7 +269,13 @@ sanity_checks() { echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" echo -n 'geckodriver: ' - geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver" + geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install" + echo -n 'singularity: ' + singularity --version || fatal "No singularity. Try: arvados-server install" + echo -n 'docker client: ' + docker --version || echo "No docker client. Try: arvados-server install" + echo -n 'docker server: ' + docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install" if [[ "$NEED_SDK_R" = true ]]; then # R SDK stuff @@ -392,7 +395,7 @@ checkpidfile() { checkhealth() { svc="$1" - base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENV3DIR}/bin/python3" -c "import yaml; print(list(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0])") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@ -414,8 +417,8 @@ start_services() { if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then return 0 fi - . "$VENVDIR/bin/activate" - echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' + . "$VENV3DIR/bin/activate" + echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -427,26 +430,26 @@ start_services() { fail=1 cd "$WORKSPACE" \ - && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ + && eval $(python3 sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ && checkpidfile api \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ + && eval $(python3 sdk/python/tests/run_test_server.py start_nginx) \ && checkpidfile nginx \ - && python sdk/python/tests/run_test_server.py start_controller \ + && python3 sdk/python/tests/run_test_server.py start_controller \ && checkpidfile controller \ && checkhealth Controller \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py start_keep_proxy \ && checkpidfile keepproxy \ - && python sdk/python/tests/run_test_server.py start_keep-web \ + && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ - && checkpidfile arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_githttpd \ + && checkpidfile githttpd \ && checkhealth GitHTTP \ - && python sdk/python/tests/run_test_server.py start_ws \ + && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ && export ARVADOS_TEST_PROXY_SERVICES=1 \ && (env | egrep ^ARVADOS) \ @@ -463,21 +466,25 @@ stop_services() { return fi unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES - . "$VENVDIR/bin/activate" || return + . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy \ - && python sdk/python/tests/run_test_server.py stop_controller \ - && python sdk/python/tests/run_test_server.py stop \ + && python3 sdk/python/tests/run_test_server.py stop_nginx \ + && python3 sdk/python/tests/run_test_server.py stop_githttpd \ + && python3 sdk/python/tests/run_test_server.py stop_ws \ + && python3 sdk/python/tests/run_test_server.py stop_keep-web \ + && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py stop_controller \ + && python3 sdk/python/tests/run_test_server.py stop \ && all_services_stopped=1 deactivate unset ARVADOS_CONFIG } interrupt() { + if [[ -n "$ignore_sigint" ]]; then + echo >&2 "ignored SIGINT" + return + fi failures+=("($(basename $0) interrupted)") exit_cleanly } @@ -520,10 +527,10 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env - (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2 + (bundle version | grep -q 2.2.19) || gem install --no-document bundler -v 2.2.19 bundle="$(which bundle)" echo "$bundle" - "$bundle" version | grep 2.0.2 || fatal 'install bundler' + "$bundle" version | grep 2.2.19 || fatal 'install bundler' else # When our "bundle install"s need to install new gems to # satisfy dependencies, we want them to go where "gem install @@ -549,19 +556,24 @@ setup_ruby_environment() { echo "Will install dependencies to $(gem env gemdir)" echo "Will install bundler and arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" - bundle="$tmpdir_gem_home/bin/bundle" + bundle="bundle" ( export HOME=$GEMHOME - bundlers="$(gem list --details bundler)" - versions=(1.11.0 1.17.3 2.0.2) + versions=(2.2.19) for v in ${versions[@]}; do - if ! echo "$bundlers" | fgrep -q "($v)"; then - gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + if ! gem list --installed --version "${v}" bundler >/dev/null; then + gem install --no-document --user $(for v in ${versions[@]}; do echo bundler:${v}; done) break fi done "$bundle" version | tee /dev/stderr | grep -q 'version 2' ) || fatal 'install bundler' + if test -d /var/lib/arvados-arvbox/ ; then + # Inside arvbox, use bundler-installed binstubs. The + # system bundler and rail's own bin/bundle refuse to work. + # I don't know why. + bundle=binstubs/bundle + fi fi } @@ -581,17 +593,12 @@ gem_uninstall_if_exists() { setup_virtualenv() { local venvdest="$1"; shift - if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then - virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip3" ]]; then + python3 -m venv "$@" "$venvdest" || fatal "virtualenv $venvdest failed" elif [[ -n "$short" ]]; then return fi - if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then - # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' - else - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' - fi + "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' } initialize() { @@ -613,7 +620,7 @@ initialize() { fi # Set up temporary install dirs (unless existing dirs were supplied) - for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS + for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -652,36 +659,27 @@ initialize() { install_env() { go mod download || fatal "Go deps failed" - which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" + which goimports >/dev/null || go install golang.org/x/tools/cmd/goimports@latest || fatal "Go setup failed" - setup_virtualenv "$VENVDIR" --python python2.7 - . "$VENVDIR/bin/activate" + setup_virtualenv "$VENV3DIR" + . "$VENV3DIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. + # pdoc3 needed to generate the Python SDK documentation. ( set -e - "${VENVDIR}/bin/pip" install PyYAML - "${VENV3DIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip3" install wheel + "${VENV3DIR}/bin/pip3" install PyYAML + "${VENV3DIR}/bin/pip3" install httplib2 + "${VENV3DIR}/bin/pip3" install future + "${VENV3DIR}/bin/pip3" install google-api-python-client + "${VENV3DIR}/bin/pip3" install ciso8601 + "${VENV3DIR}/bin/pip3" install pycurl + "${VENV3DIR}/bin/pip3" install ws4py + "${VENV3DIR}/bin/pip3" install pdoc3 cd "$WORKSPACE/sdk/python" - python setup.py install + python3 setup.py install ) || fatal "installing PyYAML and sdk/python failed" - - # Deactivate Python 2 virtualenv - deactivate - - # If Python 3 is available, set up its virtualenv in $VENV3DIR. - # Otherwise, skip dependent tests. - PYTHON3=$(which python3) - if [[ ${?} = 0 ]]; then - setup_virtualenv "$VENV3DIR" --python python3 - else - PYTHON3= - cat >&2 </dev/null; then deactivate; fi - if ! . "$VENVDIR/bin/activate" + if ! . "$VENV3DIR/bin/activate" then result=1 elif [[ "$2" == "go" ]] @@ -826,12 +824,12 @@ check_arvados_config() { # Create config file. The run_test_server script requires PyYAML, # so virtualenv needs to be active. Downstream steps like # workbench install which require a valid config.yml. - if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + if [[ ! -s "$VENV3DIR/bin/activate" ]] ; then install_env fi - . "$VENVDIR/bin/activate" + . "$VENV3DIR/bin/activate" cd "$WORKSPACE" - eval $(python sdk/python/tests/run_test_server.py setup_config) + eval $(python3 sdk/python/tests/run_test_server.py setup_config) deactivate fi } @@ -850,7 +848,7 @@ do_install_once() { result= if which deactivate >/dev/null; then deactivate; fi - if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then + if [[ "$1" != "env" ]] && ! . "$VENV3DIR/bin/activate"; then result=1 elif [[ "$2" == "go" ]] then @@ -870,10 +868,10 @@ do_install_once() { # install" ensures that we've actually installed the local package # we just built. cd "$WORKSPACE/$1" \ - && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ + && "${3}python3" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip3" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip3" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -953,7 +951,7 @@ install_services/api() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENV3DIR}/bin/python3" -c "import yaml; print(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname'])") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -962,7 +960,7 @@ install_services/api() { if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then ( dir="$WORKSPACE/services/api/tmp" - set -ex + set -e openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"