X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2f66d4cc05e9442a9bb69969744d0750a02a1ed4..add09f355520b2bd1214b8c74d5a8d6c8e76dc88:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 7a3302de9b..f83b045dce 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -35,7 +35,7 @@ Options: --short Skip (or scale down) some slow tests. --interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. -CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required) +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file sdk/python_test="--test-suite tests.test_keep_locator" @@ -91,6 +91,7 @@ lib/dispatchcloud/scheduler lib/dispatchcloud/ssh_executor lib/dispatchcloud/worker lib/mount +lib/pam lib/service services/api services/arv-git-httpd @@ -104,14 +105,10 @@ services/keepproxy services/keepstore services/keep-balance services/login-sync -services/nodemanager -services/nodemanager_integration services/crunch-dispatch-local services/crunch-dispatch-slurm services/ws sdk/cli -sdk/pam -sdk/pam:py3 sdk/python sdk/python:py3 sdk/ruby @@ -197,17 +194,12 @@ sanity_checks() { [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" - [[ -n "$CONFIGSRC" ]] \ - || fatal "CONFIGSRC environment not set (see: $0 --help)" - [[ -s "$CONFIGSRC/config.yml" ]] \ - || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" + [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \ + || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen" - echo -n 'virtualenv: ' - virtualenv --version \ - || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" echo -n 'ruby: ' ruby -v \ || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" @@ -225,9 +217,9 @@ sanity_checks() { echo -n 'gnutls.h: ' find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev" - echo -n 'Python2 pyconfig.h: ' - find /usr/include -path '*/python2*/pyconfig.h' | egrep --max-count=1 . \ - || fatal "No Python2 pyconfig.h. Try: apt-get install python2.7-dev" + echo -n 'virtualenv: ' + python3 -m venv -h | egrep --max-count=1 . \ + || fatal "No virtualenv. Try: apt-get install python3-venv" echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" @@ -262,6 +254,9 @@ sanity_checks() { echo -n 'libpq libpq-fe.h: ' find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" + echo -n 'libpam pam_appl.h: ' + find /usr/include -path '*/security/pam_appl.h' | egrep --max-count=1 . \ + || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev" echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" echo -n 'phantomjs: ' @@ -305,8 +300,6 @@ declare -A skip declare -A only declare -A testargs skip[apps/workbench_profile]=1 -# nodemanager_integration tests are not reliable, see #12061. -skip[services/nodemanager_integration]=1 while [[ -n "$1" ]] do @@ -377,7 +370,7 @@ if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then fi if [[ $NEED_SDK_R == false ]]; then - echo "R SDK not needed, it will not be installed." + echo "R SDK not needed, it will not be installed." fi checkpidfile() { @@ -393,7 +386,7 @@ checkpidfile() { checkhealth() { svc="$1" - base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENV3DIR}/bin/python3" -c "import yaml; print(list(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0])") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@ -415,39 +408,39 @@ start_services() { if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then return 0 fi - . "$VENVDIR/bin/activate" + . "$VENV3DIR/bin/activate" echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then - mkdir -p "$WORKSPACE/services/api/log" + mkdir -p "$WORKSPACE/services/api/log" fi # Remove empty api.pid file if it exists if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then - rm -f "$WORKSPACE/tmp/api.pid" + rm -f "$WORKSPACE/tmp/api.pid" fi all_services_stopped= fail=1 cd "$WORKSPACE" \ - && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ + && eval $(python3 sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ && checkpidfile api \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ + && eval $(python3 sdk/python/tests/run_test_server.py start_nginx) \ && checkpidfile nginx \ - && python sdk/python/tests/run_test_server.py start_controller \ + && python3 sdk/python/tests/run_test_server.py start_controller \ && checkpidfile controller \ && checkhealth Controller \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py start_keep_proxy \ && checkpidfile keepproxy \ - && python sdk/python/tests/run_test_server.py start_keep-web \ + && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \ && checkpidfile arv-git-httpd \ && checkhealth GitHTTP \ - && python sdk/python/tests/run_test_server.py start_ws \ + && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ && export ARVADOS_TEST_PROXY_SERVICES=1 \ && (env | egrep ^ARVADOS) \ @@ -464,15 +457,15 @@ stop_services() { return fi unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES - . "$VENVDIR/bin/activate" || return + . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy \ - && python sdk/python/tests/run_test_server.py stop_controller \ - && python sdk/python/tests/run_test_server.py stop \ + && python3 sdk/python/tests/run_test_server.py stop_nginx \ + && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py stop_ws \ + && python3 sdk/python/tests/run_test_server.py stop_keep-web \ + && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py stop_controller \ + && python3 sdk/python/tests/run_test_server.py stop \ && all_services_stopped=1 deactivate unset ARVADOS_CONFIG @@ -553,10 +546,16 @@ setup_ruby_environment() { bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" ( export HOME=$GEMHOME - ("$bundle" version | grep -q 2.0.2) \ - || gem install --user bundler -v 2.0.2 + bundlers="$(gem list --details bundler)" + versions=(1.11.0 1.17.3 2.0.2) + for v in ${versions[@]}; do + if ! echo "$bundlers" | fgrep -q "($v)"; then + gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + break + fi + done "$bundle" version | tee /dev/stderr | grep -q 'version 2' - ) || fatal 'install bundler' + ) #|| fatal 'install bundler' ## This is needed for now. fi } @@ -576,20 +575,20 @@ gem_uninstall_if_exists() { setup_virtualenv() { local venvdest="$1"; shift - if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then - virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip3" ]]; then + python3 -m venv "$@" "$venvdest" || fatal "virtualenv $venvdest failed" elif [[ -n "$short" ]]; then return fi - if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then - # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' - else - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' - fi + "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' } initialize() { + # If dependencies like ruby, go, etc. are installed in + # /var/lib/arvados -- presumably by "arvados-server install" -- + # then we want to use those versions, instead of whatever happens + # to be installed in /usr. + PATH="/var/lib/arvados/bin:${PATH}" sanity_checks echo "WORKSPACE=$WORKSPACE" @@ -603,7 +602,7 @@ initialize() { fi # Set up temporary install dirs (unless existing dirs were supplied) - for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS + for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@ -644,42 +643,21 @@ install_env() { go mod download || fatal "Go deps failed" which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" - setup_virtualenv "$VENVDIR" --python python2.7 - . "$VENVDIR/bin/activate" + setup_virtualenv "$VENV3DIR" # Needed for run_test_server.py which is used by certain (non-Python) tests. ( set -e - "${VENVDIR}/bin/pip" install PyYAML - "${VENV3DIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip3" install PyYAML + "${VENV3DIR}/bin/pip3" install httplib2 + "${VENV3DIR}/bin/pip3" install future + "${VENV3DIR}/bin/pip3" install google-api-python-client + "${VENV3DIR}/bin/pip3" install ciso8601 + "${VENV3DIR}/bin/pip3" install pycurl + "${VENV3DIR}/bin/pip3" install ws4py cd "$WORKSPACE/sdk/python" - python setup.py install + python3 setup.py install ) || fatal "installing PyYAML and sdk/python failed" - - # Preinstall libcloud if using a fork; otherwise nodemanager "pip - # install" won't pick it up by default. - if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then - pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ - || fatal "pip install apache-libcloud failed" - fi - - # Deactivate Python 2 virtualenv - deactivate - - # If Python 3 is available, set up its virtualenv in $VENV3DIR. - # Otherwise, skip dependent tests. - PYTHON3=$(which python3) - if [[ ${?} = 0 ]]; then - setup_virtualenv "$VENV3DIR" --python python3 - else - PYTHON3= - cat >&2 </dev/null; then deactivate; fi - if ! . "$VENVDIR/bin/activate" + if ! . "$VENV3DIR/bin/activate" then result=1 elif [[ "$2" == "go" ]] @@ -821,19 +796,19 @@ do_test_once() { check_arvados_config() { if [[ "$1" = "env" ]] ; then - return + return fi if [[ -z "$ARVADOS_CONFIG" ]] ; then - # Create config file. The run_test_server script requires PyYAML, - # so virtualenv needs to be active. Downstream steps like - # workbench install which require a valid config.yml. - if [[ ! -s "$VENVDIR/bin/activate" ]] ; then - install_env - fi - . "$VENVDIR/bin/activate" + # Create config file. The run_test_server script requires PyYAML, + # so virtualenv needs to be active. Downstream steps like + # workbench install which require a valid config.yml. + if [[ ! -s "$VENV3DIR/bin/activate" ]] ; then + install_env + fi + . "$VENV3DIR/bin/activate" cd "$WORKSPACE" - eval $(python sdk/python/tests/run_test_server.py setup_config) - deactivate + eval $(python3 sdk/python/tests/run_test_server.py setup_config) + deactivate fi } @@ -851,7 +826,7 @@ do_install_once() { result= if which deactivate >/dev/null; then deactivate; fi - if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then + if [[ "$1" != "env" ]] && ! . "$VENV3DIR/bin/activate"; then result=1 elif [[ "$2" == "go" ]] then @@ -871,10 +846,10 @@ do_install_once() { # install" ensures that we've actually installed the local package # we just built. cd "$WORKSPACE/$1" \ - && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ + && "${3}python3" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip3" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip3" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -954,7 +929,7 @@ install_services/api() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENV3DIR}/bin/python3" -c "import yaml; print(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname'])") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -963,7 +938,7 @@ install_services/api() { if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then ( dir="$WORKSPACE/services/api/tmp" - set -ex + set -e openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key"