X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c7cf228372bee18d84b14fc4089a758bc157fe76..95c835b12a32a5fa6dc94799d6fea16e5b20080c:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index a869d0f6a7..b8d2081e6e 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -208,7 +208,8 @@ sanity_checks() { find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev" echo -n 'virtualenv: ' - python3 -m venv -h | egrep --max-count=1 . \ + python3 -m venv --help | grep -q '^usage: venv ' \ + && echo "venv module found" \ || fatal "No virtualenv. Try: apt-get install python3-venv" echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ @@ -223,7 +224,7 @@ sanity_checks() { || fatal "No gitolite. Try: apt-get install gitolite3" echo -n 'npm: ' npm --version \ - || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/" + || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v12.22.12-linux-x64/bin/{node,npm} /usr/local/bin/" echo -n 'cadaver: ' cadaver --version | grep -w cadaver \ || fatal "No cadaver. Try: apt-get install cadaver" @@ -241,14 +242,10 @@ sanity_checks() { || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev" echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" - echo -n 'phantomjs: ' - phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs" echo -n 'xvfb: ' which Xvfb || fatal "No xvfb. Try: apt-get install xvfb" echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" - echo -n 'geckodriver: ' - geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install" echo -n 'singularity: ' singularity --version || fatal "No singularity. Try: arvados-server install" echo -n 'docker client: ' @@ -639,13 +636,15 @@ install_env() { setup_virtualenv "$VENV3DIR" . "$VENV3DIR/bin/activate" + # wheel modernizes the venv (as of early 2024) and makes it more closely + # match our package build environment. # PyYAML is a test requirement used by run_test_server.py and needed for # other, non-Python tests. # pdoc is needed to build PySDK documentation. # We run `setup.py build` first to generate _version.py. - env -C "$WORKSPACE/sdk/python" python3 setup.py build \ - && python3 -m pip install "$WORKSPACE/sdk/python" \ - && python3 -m pip install PyYAML pdoc \ + pip install PyYAML pdoc wheel \ + && env -C "$WORKSPACE/sdk/python" python3 setup.py build \ + && pip install "$WORKSPACE/sdk/python" \ || fatal "installing Python SDK and related dependencies failed" } @@ -691,7 +690,22 @@ do_test() { stop_services check_arvados_config "$1" ;; - gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker | services/workbench2_units | services/workbench2_integration) + gofmt \ + | arvados_version.py \ + | cmd/arvados-package \ + | doc \ + | lib/boot \ + | lib/cli \ + | lib/cloud/azure \ + | lib/cloud/cloudtest \ + | lib/cloud/ec2 \ + | lib/cmd \ + | lib/dispatchcloud/sshexecutor \ + | lib/dispatchcloud/worker \ + | lib/install \ + | services/workbench2_integration \ + | services/workbench2_units \ + ) check_arvados_config "$1" # don't care whether services are running ;; @@ -960,11 +974,14 @@ install_services/api() { declare -a pythonstuff pythonstuff=( + # The ordering of sdk/python, tools/crunchstat-summary, and + # sdk/cwl here is significant. See + # https://dev.arvados.org/issues/19744#note-26 sdk/python:py3 + tools/crunchstat-summary:py3 sdk/cwl:py3 services/dockercleaner:py3 services/fuse:py3 - tools/crunchstat-summary:py3 ) declare -a gostuff @@ -976,20 +993,37 @@ install_services/workbench2() { } test_doc() { - ( - set -e - cd "$WORKSPACE/doc" - ARVADOS_API_HOST=pirca.arvadosapi.com - # Make sure python-epydoc is installed or the next line won't - # do much good! - PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST - ) + local arvados_api_host=pirca.arvadosapi.com && \ + env -C "$WORKSPACE/doc" \ + "$bundle" exec rake linkchecker \ + arvados_api_host="$arvados_api_host" \ + arvados_workbench_host="https://workbench.$arvados_api_host" \ + baseurl="file://$WORKSPACE/doc/.site/" \ + ${testargs[doc]} } test_gofmt() { cd "$WORKSPACE" || return 1 dirs=$(ls -d */ | egrep -v 'vendor|tmp') [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]] + go vet -composites=false ./... +} + +test_arvados_version.py() { + local orig_fn="" + local fail_count=0 + while read -d "" fn; do + if [[ -z "$orig_fn" ]]; then + orig_fn="$fn" + elif ! cmp "$orig_fn" "$fn"; then + fail_count=$(( $fail_count + 1 )) + printf "FAIL: %s and %s are not identical\n" "$orig_fn" "$fn" + fi + done < <(git -C "$WORKSPACE" ls-files -z | grep -z '/arvados_version\.py$') + case "$orig_fn" in + "") return 66 ;; # EX_NOINPUT + *) return "$fail_count" ;; + esac } test_services/api() { @@ -1042,11 +1076,14 @@ install_deps() { # Install parts needed by test suites do_install env do_install cmd/arvados-server go - do_install sdk/cli - do_install sdk/python pip "${VENV3DIR}/bin/" + do_install tools/crunchstat-summary pip "${VENV3DIR}/bin/" do_install sdk/ruby-google-api-client do_install sdk/ruby + do_install sdk/cli do_install services/api + # lib/controller integration tests depend on arv-mount to run + # containers. + do_install services/fuse pip "${VENV3DIR}/bin/" do_install services/keepproxy go do_install services/keep-web go } @@ -1086,6 +1123,7 @@ test_all() { fi do_test gofmt + do_test arvados_version.py do_test doc do_test sdk/ruby-google-api-client do_test sdk/ruby