X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dec8b52c3b3ff10ca63fe4b0bd2bf16cd9f8809d..86660414472d4ff0d8267f9845a753497bd41692:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 29e0d83d03..ee52c4b9dd 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -40,7 +40,7 @@ sdk/python_test="--test-suite tests.test_keep_locator" Restrict Python SDK tests to the given class apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb" Restrict Workbench tests to the given file -services/arv-git-httpd_test="-check.vv" +services/githttpd_test="-check.vv" Show all log messages, even when tests pass (also works with services/keepstore_test etc.) ARVADOS_DEBUG=1 @@ -69,6 +69,7 @@ apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile cmd/arvados-client +cmd/arvados-package cmd/arvados-server doc lib/cli @@ -92,7 +93,7 @@ lib/mount lib/pam lib/service services/api -services/arv-git-httpd +services/githttpd services/crunchstat services/dockercleaner services/fuse @@ -158,6 +159,8 @@ only_install= temp= temp_preserve= +ignore_sigint= + clear_temp() { if [[ -z "$temp" ]]; then # we did not even get as far as making a temp dir @@ -267,7 +270,13 @@ sanity_checks() { echo -n 'graphviz: ' dot -V || fatal "No graphviz. Try: apt-get install graphviz" echo -n 'geckodriver: ' - geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver" + geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install" + echo -n 'singularity: ' + singularity --version || fatal "No singularity. Try: arvados-server install" + echo -n 'docker client: ' + docker --version || echo "No docker client. Try: arvados-server install" + echo -n 'docker server: ' + docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install" if [[ "$NEED_SDK_R" = true ]]; then # R SDK stuff @@ -410,7 +419,7 @@ start_services() { return 0 fi . "$VENV3DIR/bin/activate" - echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' + echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -438,8 +447,8 @@ start_services() { && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \ - && checkpidfile arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_githttpd \ + && checkpidfile githttpd \ && checkhealth GitHTTP \ && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ @@ -461,7 +470,7 @@ stop_services() { . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ && python3 sdk/python/tests/run_test_server.py stop_nginx \ - && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py stop_githttpd \ && python3 sdk/python/tests/run_test_server.py stop_ws \ && python3 sdk/python/tests/run_test_server.py stop_keep-web \ && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ @@ -473,6 +482,10 @@ stop_services() { } interrupt() { + if [[ -n "$ignore_sigint" ]]; then + echo >&2 "ignored SIGINT" + return + fi failures+=("($(basename $0) interrupted)") exit_cleanly } @@ -515,7 +528,7 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env - (bundle version | grep -q 2.2.19) || gem install bundler -v 2.2.19 + (bundle version | grep -q 2.2.19) || gem install --no-document bundler -v 2.2.19 bundle="$(which bundle)" echo "$bundle" "$bundle" version | grep 2.2.19 || fatal 'install bundler' @@ -544,19 +557,24 @@ setup_ruby_environment() { echo "Will install dependencies to $(gem env gemdir)" echo "Will install bundler and arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" - bundle="$tmpdir_gem_home/bin/bundle" + bundle="bundle" ( export HOME=$GEMHOME - bundlers="$(gem list --details bundler)" versions=(2.2.19) for v in ${versions[@]}; do - if ! echo "$bundlers" | fgrep -q "($v)"; then - gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + if ! gem list --installed --version "${v}" bundler >/dev/null; then + gem install --no-document --user $(for v in ${versions[@]}; do echo bundler:${v}; done) break fi done "$bundle" version | tee /dev/stderr | grep -q 'version 2' ) || fatal 'install bundler' + if test -d /var/lib/arvados-arvbox/ ; then + # Inside arvbox, use bundler-installed binstubs. The + # system bundler and rail's own bin/bundle refuse to work. + # I don't know why. + bundle=binstubs/bundle + fi fi } @@ -642,7 +660,7 @@ initialize() { install_env() { go mod download || fatal "Go deps failed" - which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" + which goimports >/dev/null || go install golang.org/x/tools/cmd/goimports@latest || fatal "Go setup failed" setup_virtualenv "$VENV3DIR" . "$VENV3DIR/bin/activate" @@ -725,7 +743,7 @@ do_test() { go_ldflags() { version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev} - echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}" + echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version} -s -w" } do_test_once() { @@ -743,6 +761,10 @@ do_test_once() { then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") + testflags=() + if [[ "$1" == "cmd/arvados-package" ]]; then + testflags+=("-timeout" "20m") + fi # We do "go install" here to catch compilation errors # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting @@ -753,11 +775,11 @@ do_test_once() { then # "go test -check.vv giturl" doesn't work, but this # does: - go test ${short:+-short} ${testargs[$1]} + go test ${short:+-short} ${testflags[@]} ${testargs[$1]} else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1" + go test ${short:+-short} ${testflags[@]} ${coverflags[@]} "git.arvados.org/arvados.git/$1" fi result=${result:-$?} if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]] @@ -962,11 +984,11 @@ install_services/api() { set -ex cd "$WORKSPACE/services/api" export RAILS_ENV=test - if "$bundle" exec rails db:environment:set ; then - "$bundle" exec rake db:drop + if bin/rails db:environment:set ; then + bin/rake db:drop fi - "$bundle" exec rake db:setup - "$bundle" exec rake db:fixtures:load + bin/rake db:setup + bin/rake db:fixtures:load ) || return 1 } @@ -980,7 +1002,7 @@ pythonstuff=( ) declare -a gostuff -gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u)) +gostuff=($(cd "$WORKSPACE" && git ls-files | grep '\.go$' | sed -e 's/\/[^\/]*$//' | sort -u)) install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ @@ -1078,7 +1100,6 @@ install_deps() { do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api - do_install services/arv-git-httpd go do_install services/keepproxy go do_install services/keep-web go } @@ -1212,6 +1233,7 @@ else setnextcmd HISTFILE="$WORKSPACE/tmp/.history" history -r + ignore_sigint=1 while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do history -s "$nextcmd" history -w