X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2b4eccc7386b33d2de4a06f5d8da18cd94391662..2e74236fa27822addd856f194befc28382990ce0:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 4d8075fab9..cc8e070a5c 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: AGPL-3.0 -. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh - COLUMNS=80 . `dirname "$(readlink -f "$0")"`/run-library.sh @@ -42,7 +40,7 @@ sdk/python_test="--test-suite tests.test_keep_locator" Restrict Python SDK tests to the given class apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb" Restrict Workbench tests to the given file -services/arv-git-httpd_test="-check.vv" +services/githttpd_test="-check.vv" Show all log messages, even when tests pass (also works with services/keepstore_test etc.) ARVADOS_DEBUG=1 @@ -88,13 +86,13 @@ lib/cloud/cloudtest lib/dispatchcloud lib/dispatchcloud/container lib/dispatchcloud/scheduler -lib/dispatchcloud/ssh_executor +lib/dispatchcloud/sshexecutor lib/dispatchcloud/worker lib/mount lib/pam lib/service services/api -services/arv-git-httpd +services/githttpd services/crunchstat services/dockercleaner services/fuse @@ -160,11 +158,16 @@ only_install= temp= temp_preserve= +ignore_sigint= + clear_temp() { if [[ -z "$temp" ]]; then - # we didn't even get as far as making a temp dir + # we did not even get as far as making a temp dir : elif [[ -z "$temp_preserve" ]]; then + # Go creates readonly dirs in the module cache, which cause + # "rm -rf" to fail unless we chmod first. + chmod -R u+w "$temp" rm -rf "$temp" else echo "Leaving behind temp dirs in $temp" @@ -241,7 +244,7 @@ sanity_checks() { || fatal "No gitolite. Try: apt-get install gitolite3" echo -n 'npm: ' npm --version \ - || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/" + || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/" echo -n 'cadaver: ' cadaver --version | grep -w cadaver \ || fatal "No cadaver. Try: apt-get install cadaver" @@ -266,7 +269,13 @@ sanity_checks() { echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" echo -n 'geckodriver: ' - geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver" + geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install" + echo -n 'singularity: ' + singularity --version || fatal "No singularity. Try: arvados-server install" + echo -n 'docker client: ' + docker --version || echo "No docker client. Try: arvados-server install" + echo -n 'docker server: ' + docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install" if [[ "$NEED_SDK_R" = true ]]; then # R SDK stuff @@ -409,7 +418,7 @@ start_services() { return 0 fi . "$VENV3DIR/bin/activate" - echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' + echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -437,8 +446,8 @@ start_services() { && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \ - && checkpidfile arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_githttpd \ + && checkpidfile githttpd \ && checkhealth GitHTTP \ && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ @@ -460,7 +469,7 @@ stop_services() { . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ && python3 sdk/python/tests/run_test_server.py stop_nginx \ - && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py stop_githttpd \ && python3 sdk/python/tests/run_test_server.py stop_ws \ && python3 sdk/python/tests/run_test_server.py stop_keep-web \ && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ @@ -472,6 +481,10 @@ stop_services() { } interrupt() { + if [[ -n "$ignore_sigint" ]]; then + echo >&2 "ignored SIGINT" + return + fi failures+=("($(basename $0) interrupted)") exit_cleanly } @@ -514,10 +527,10 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env - (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2 + (bundle version | grep -q 2.2.19) || gem install --no-document bundler -v 2.2.19 bundle="$(which bundle)" echo "$bundle" - "$bundle" version | grep 2.0.2 || fatal 'install bundler' + "$bundle" version | grep 2.2.19 || fatal 'install bundler' else # When our "bundle install"s need to install new gems to # satisfy dependencies, we want them to go where "gem install @@ -538,24 +551,29 @@ setup_ruby_environment() { tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)" PATH="$tmpdir_gem_home/bin:$PATH" - export GEM_PATH="$tmpdir_gem_home" + export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)" echo "Will install dependencies to $(gem env gemdir)" - echo "Will install arvados gems to $tmpdir_gem_home" + echo "Will install bundler and arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" - bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" + bundle="bundle" ( export HOME=$GEMHOME - bundlers="$(gem list --details bundler)" - versions=(1.11.0 1.17.3 2.0.2) + versions=(2.2.19) for v in ${versions[@]}; do - if ! echo "$bundlers" | fgrep -q "($v)"; then - gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + if ! gem list --installed --version "${v}" bundler >/dev/null; then + gem install --no-document --user $(for v in ${versions[@]}; do echo bundler:${v}; done) break fi done "$bundle" version | tee /dev/stderr | grep -q 'version 2' - ) #|| fatal 'install bundler' ## This is needed for now. + ) || fatal 'install bundler' + if test -d /var/lib/arvados-arvbox/ ; then + # Inside arvbox, use bundler-installed binstubs. The + # system bundler and rail's own bin/bundle refuse to work. + # I don't know why. + bundle=binstubs/bundle + fi fi } @@ -641,21 +659,24 @@ initialize() { install_env() { go mod download || fatal "Go deps failed" - which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" + which goimports >/dev/null || go install golang.org/x/tools/cmd/goimports@latest || fatal "Go setup failed" setup_virtualenv "$VENV3DIR" + . "$VENV3DIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. + # pdoc3 needed to generate the Python SDK documentation. ( set -e + "${VENV3DIR}/bin/pip3" install wheel "${VENV3DIR}/bin/pip3" install PyYAML - "${VENV3DIR}/bin/pip3" install httplib2 + "${VENV3DIR}/bin/pip3" install httplib2 "${VENV3DIR}/bin/pip3" install future "${VENV3DIR}/bin/pip3" install google-api-python-client "${VENV3DIR}/bin/pip3" install ciso8601 - "${VENV3DIR}/bin/pip3" install wheel "${VENV3DIR}/bin/pip3" install pycurl "${VENV3DIR}/bin/pip3" install ws4py + "${VENV3DIR}/bin/pip3" install pdoc3 cd "$WORKSPACE/sdk/python" python3 setup.py install ) || fatal "installing PyYAML and sdk/python failed" @@ -703,7 +724,7 @@ do_test() { stop_services check_arvados_config "$1" ;; - gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker) check_arvados_config "$1" # don't care whether services are running ;; @@ -721,7 +742,7 @@ do_test() { go_ldflags() { version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev} - echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}" + echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version} -s -w" } do_test_once() { @@ -955,14 +976,14 @@ install_services/api() { || return 1 ( - set -e + set -ex cd "$WORKSPACE/services/api" export RAILS_ENV=test - if "$bundle" exec rails db:environment:set ; then - "$bundle" exec rake db:drop + if bin/rails db:environment:set ; then + bin/rake db:drop fi - "$bundle" exec rake db:setup - "$bundle" exec rake db:fixtures:load + bin/rake db:setup + bin/rake db:fixtures:load ) || return 1 } @@ -976,7 +997,7 @@ pythonstuff=( ) declare -a gostuff -gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u)) +gostuff=($(cd "$WORKSPACE" && git ls-files | grep '\.go$' | sed -e 's/\/[^\/]*$//' | sort -u)) install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ @@ -989,7 +1010,7 @@ test_doc() { ( set -e cd "$WORKSPACE/doc" - ARVADOS_API_HOST=qr1hi.arvadosapi.com + ARVADOS_API_HOST=pirca.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST @@ -1074,11 +1095,8 @@ install_deps() { do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api - do_install services/arv-git-httpd go do_install services/keepproxy go - do_install services/keepstore go do_install services/keep-web go - do_install services/ws go } install_all() { @@ -1089,19 +1107,12 @@ install_all() { do_install sdk/perl do_install sdk/cli do_install services/login-sync - ## FIXME, ignore all python2 for p in "${pythonstuff[@]}" do - dir=${p%:py3} - if [[ ${dir} = ${p} ]]; then - if [[ -z ${skip[python2]} ]]; then - do_install ${dir} pip - fi - elif [[ -n ${PYTHON3} ]]; then - if [[ -z ${skip[python3]} ]]; then - do_install ${dir} pip "$VENV3DIR/bin/" - fi - fi + dir=${p%:py3} + if [[ -z ${skip[python3]} ]]; then + do_install ${dir} pip "$VENV3DIR/bin/" + fi done for g in "${gostuff[@]}" do @@ -1129,18 +1140,11 @@ test_all() { do_test sdk/cli do_test services/login-sync do_test sdk/java-v2 - ## FIXME, ignore all python2 for p in "${pythonstuff[@]}" do dir=${p%:py3} - if [[ ${dir} = ${p} ]]; then - if [[ -z ${skip[python2]} ]]; then - do_test ${dir} pip - fi - elif [[ -n ${PYTHON3} ]]; then - if [[ -z ${skip[python3]} ]]; then - do_test ${dir} pip "$VENV3DIR/bin/" - fi + if [[ -z ${skip[python3]} ]]; then + do_test ${dir} pip "$VENV3DIR/bin/" fi done @@ -1155,6 +1159,14 @@ test_all() { do_test apps/workbench_profile } +test_go() { + do_test gofmt + for g in "${gostuff[@]}" + do + do_test "$g" go + done +} + help_interactive() { echo "== Interactive commands:" echo "TARGET (short for 'test DIR')" @@ -1216,6 +1228,7 @@ else setnextcmd HISTFILE="$WORKSPACE/tmp/.history" history -r + ignore_sigint=1 while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do history -s "$nextcmd" history -w