3 read -rd "\000" helpmessage <<EOF
4 $(basename $0): Install and test Arvados components.
6 Exit non-zero if any tests fail.
9 $(basename $0) WORKSPACE=/path/to/arvados [options]
13 --skip FOO Do not test the FOO component.
14 --only FOO Do not test anything except the FOO component.
15 --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
16 Instead, show which directories were used this time so they
17 can be reused in subsequent invocations.
18 --skip-install Do not run any install steps. Just run tests.
19 You should provide GOPATH, GEMHOME, and VENVDIR options
20 from a previous invocation if you use this option.
21 WORKSPACE=path Arvados source tree to test.
22 CONFIGSRC=path Dir with api server config files to copy into source tree.
23 (If none given, leave config files alone in source tree.)
24 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
25 Restrict apiserver tests to the given file
26 sdk/python_test="--test-suite test.test_keep_locator"
27 Restrict Python SDK tests to the given class
28 apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
29 Restrict Workbench tests to the given file
30 services/arv-git-httpd_test="-check.vv"
31 Show all log messages, even when tests pass (also works
32 with services/keepstore_test etc.)
34 Print more debug messages
35 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
36 *_test, and other examples shown above.
38 Assuming --skip-install is not given, all components are installed
39 into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
40 test suites depend on other components being installed, and installing
41 everything tends to be quicker than debugging dependencies.
43 As a special concession to the current CI server config, CONFIGSRC
44 defaults to $HOME/arvados-api-server if that directory exists.
46 More information and background:
48 https://arvados.org/projects/arvados/wiki/Running_tests
53 apps/workbench_benchmark
54 apps/workbench_profile
62 services/arv-git-httpd
72 # First make sure to remove any ARVADOS_ variables from the calling
73 # environment that could interfere with the tests.
74 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
76 # Reset other variables that could affect our [tests'] behavior by
92 for var in VENVDIR GOPATH GITDIR GEMHOME
94 if [[ -z "${leave_temp[$var]}" ]]
101 leaving+=" $var=\"${!var}\""
104 if [[ -n "$leaving" ]]; then
105 echo "Leaving behind temp dirs: $leaving"
111 echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
116 for x in "${successes[@]}"
121 if [[ ${#failures[@]} == 0 ]]
123 echo "All test suites passed."
125 echo "Failures (${#failures[@]}):"
126 for x in "${failures[@]}"
135 rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
137 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
144 # Make sure WORKSPACE is set
145 if ! [[ -n "$WORKSPACE" ]]; then
146 echo >&2 "$helpmessage"
148 echo >&2 "Error: WORKSPACE environment variable not set"
153 # Make sure virtualenv is installed
154 `virtualenv --help >/dev/null 2>&1`
156 if [[ "$?" != "0" ]]; then
158 echo >&2 "Error: virtualenv could not be found"
163 # Make sure go is installed
164 `go env >/dev/null 2>&1`
166 if [[ "$?" != "0" ]]; then
168 echo >&2 "Error: go could not be found"
173 # Make sure gcc is installed
174 `gcc --help >/dev/null 2>&1`
176 if [[ "$?" != "0" ]]; then
178 echo >&2 "Error: gcc could not be found"
185 # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
186 if [[ -f "$1/$2" ]]; then
187 THEDATE=`date +%Y%m%d%H%M%S`
188 mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
189 gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
196 skip[apps/workbench_profile]=1
203 echo >&2 "$helpmessage"
212 only="$1"; skip[$1]=""; shift
218 leave_temp[VENVDIR]=1
220 leave_temp[GEMHOME]=1
226 suite="${arg%%_test=*}"
228 testargs["$suite"]="$args"
231 eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
234 echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
241 echo 'Starting API server...'
243 && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
244 && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
245 && export ARVADOS_TEST_API_INSTALLED="$$" \
246 && (env | egrep ^ARVADOS)
249 start_nginx_proxy_services() {
250 echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
252 && python sdk/python/tests/run_test_server.py start_keep_proxy \
253 && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
254 && python sdk/python/tests/run_test_server.py start_nginx \
255 && export ARVADOS_TEST_PROXY_SERVICES=1
259 if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
260 unset ARVADOS_TEST_PROXY_SERVICES
262 && python sdk/python/tests/run_test_server.py stop_nginx \
263 && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
264 && python sdk/python/tests/run_test_server.py stop_keep_proxy
266 if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
267 unset ARVADOS_TEST_API_HOST
269 && python sdk/python/tests/run_test_server.py stop
274 failures+=("($(basename $0) interrupted)")
281 echo "WORKSPACE=$WORKSPACE"
283 if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
284 # Jenkins expects us to use this by default.
285 CONFIGSRC="$HOME/arvados-api-server"
288 # Clean up .pyc files that may exist in the workspace
290 find -name '*.pyc' -delete
292 # Set up temporary install dirs (unless existing dirs were supplied)
293 for tmpdir in VENVDIR GOPATH GEMHOME
295 if [[ -n "${!tmpdir}" ]]; then
296 leave_temp[$tmpdir]=1
298 eval $tmpdir=$(mktemp -d)
302 setup_ruby_environment() {
303 if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
304 source "$HOME/.rvm/scripts/rvm"
306 elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
307 source "/usr/local/rvm/scripts/rvm"
313 if [[ "$using_rvm" == true ]]; then
314 # If rvm is in use, we can't just put separate "dependencies"
315 # and "gems-under-test" paths to GEM_PATH: passenger resets
316 # the environment to the "current gemset", which would lose
317 # our GEM_PATH and prevent our test suites from running ruby
318 # programs (for example, the Workbench test suite could not
319 # boot an API server or run arv). Instead, we have to make an
320 # rvm gemset and use it for everything.
322 [[ `type rvm | head -n1` == "rvm is a function" ]] \
325 # Put rvm's favorite path back in first place (overriding
326 # virtualenv, which just put itself there). Ignore rvm's
327 # complaint about not being in first place already.
328 rvm use @default 2>/dev/null
330 # Create (if needed) and switch to an @arvados-tests
331 # gemset. (Leave the choice of ruby to the caller.)
332 rvm use @arvados-tests --create \
333 || fatal 'rvm gemset setup'
337 # When our "bundle install"s need to install new gems to
338 # satisfy dependencies, we want them to go where "gem install
339 # --user-install" would put them. (However, if the caller has
340 # already set GEM_HOME, we assume that's where dependencies
341 # should be installed, and we should leave it alone.)
343 if [ -z "$GEM_HOME" ]; then
344 user_gempath="$(gem env gempath)"
345 export GEM_HOME="${user_gempath%%:*}"
347 PATH="$(gem env gemdir)/bin:$PATH"
349 # When we build and install our own gems, we install them in our
350 # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
351 # PATH so integration tests prefer them over other versions that
352 # happen to be installed in $user_gempath, system dirs, etc.
354 tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
355 PATH="$tmpdir_gem_home/bin:$PATH"
356 export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
358 echo "Will install dependencies to $(gem env gemdir)"
359 echo "Will install arvados gems to $tmpdir_gem_home"
360 echo "Gem search path is GEM_PATH=$GEM_PATH"
365 if [[ "$using_rvm" == true ]]; then
368 GEM_HOME="$tmpdir_gem_home" "$@"
373 mkdir -p "$GOPATH/src/git.curoverse.com"
374 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
375 || fatal "symlink failed"
377 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
378 . "$VENVDIR/bin/activate"
380 # Note: this must be the last time we change PATH, otherwise rvm will
382 setup_ruby_environment
386 if ! which bundler >/dev/null
388 gem install --user-install bundler || fatal 'Could not install bundler'
391 # Needed for run_test_server.py which is used by certain (non-Python) tests.
392 echo "pip install -q PyYAML"
393 pip install -q PyYAML || fatal "pip install PyYAML failed"
396 if [[ "$1" != "0" ]]; then
397 title "!!!!!! $2 FAILED !!!!!!"
398 failures+=("$2 (`timer`)")
400 successes+=("$2 (`timer`)")
409 echo -n "$(($SECONDS - $t0))s"
413 while ! do_test_once ${@} && [[ "$retry" == 1 ]]
415 read -p 'Try again? [Y/n] ' x
416 if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
424 if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
426 title "Running $1 tests"
428 if [[ "$2" == "go" ]]
430 if [[ -n "${testargs[$1]}" ]]
432 # "go test -check.vv giturl" doesn't work, but this
434 cd "$WORKSPACE/$1" && go test ${testargs[$1]}
436 # The above form gets verbose even when testargs is
437 # empty, so use this form in such cases:
438 go test "git.curoverse.com/arvados.git/$1"
440 elif [[ "$2" == "pip" ]]
443 && python setup.py test ${testargs[$1]}
444 elif [[ "$2" != "" ]]
451 checkexit $result "$1 tests"
452 title "End of $1 tests (`timer`)"
455 title "Skipping $1 tests"
460 if [[ -z "$skip_install" ]]
462 title "Running $1 install"
464 if [[ "$2" == "go" ]]
466 go get -t "git.curoverse.com/arvados.git/$1"
467 elif [[ "$2" == "pip" ]]
470 && python setup.py sdist rotate --keep=1 --match .tar.gz \
471 && pip install -q --upgrade dist/*.tar.gz
472 elif [[ "$2" != "" ]]
478 checkexit $? "$1 install"
479 title "End of $1 install (`timer`)"
481 title "Skipping $1 install"
486 txt="********** $1 **********"
487 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
490 bundle_install_trylocal() {
493 echo "(Running bundle install --local. 'could not find package' messages are OK.)"
494 if ! bundle install --local --no-deployment; then
495 echo "(Running bundle install again, without --local.)"
496 bundle install --no-deployment
503 cd "$WORKSPACE/doc" \
504 && bundle_install_trylocal \
510 with_test_gemset gem uninstall --force --all --executables arvados \
511 && cd "$WORKSPACE/sdk/ruby" \
512 && bundle_install_trylocal \
513 && gem build arvados.gemspec \
514 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
516 do_install sdk/ruby ruby_sdk
519 with_test_gemset gem uninstall --force --all --executables arvados-cli \
520 && cd "$WORKSPACE/sdk/cli" \
521 && bundle_install_trylocal \
522 && gem build arvados-cli.gemspec \
523 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
525 do_install sdk/cli cli
527 # Install the Python SDK early. Various other test suites (like
528 # keepproxy) bring up run_test_server.py, which imports the arvados
529 # module. We can't actually *test* the Python SDK yet though, because
530 # its own test suite brings up some of those other programs (like
532 declare -a pythonstuff
538 for p in "${pythonstuff[@]}"
543 install_apiserver() {
544 cd "$WORKSPACE/services/api" \
545 && RAILS_ENV=test bundle_install_trylocal
547 rm -f config/environments/test.rb
548 cp config/environments/test.rb.example config/environments/test.rb
550 if [ -n "$CONFIGSRC" ]
552 for f in database.yml application.yml
554 cp "$CONFIGSRC/$f" config/ || fatal "$f"
558 # Fill in a random secret_token and blob_signing_key for testing
559 SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
560 BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
562 sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
563 sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
565 # Set up empty git repo (for git tests)
567 sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
570 mkdir -p $GITDIR/test
573 && git config user.email "jenkins@ci.curoverse.com" \
574 && git config user.name "Jenkins, CI" \
577 && git commit -m 'initial commit'
579 # Clear out any lingering postgresql connections to the test
580 # database, so that we can drop it. This assumes the current user
581 # is a postgresql superuser.
582 test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
583 psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
585 cd "$WORKSPACE/services/api" \
586 && RAILS_ENV=test bundle exec rake db:drop \
587 && RAILS_ENV=test bundle exec rake db:setup \
588 && RAILS_ENV=test bundle exec rake db:fixtures:load
590 do_install services/api apiserver
594 services/arv-git-httpd
602 for g in "${gostuff[@]}"
607 install_workbench() {
608 cd "$WORKSPACE/apps/workbench" \
609 && mkdir -p tmp/cache \
610 && RAILS_ENV=test bundle_install_trylocal
612 do_install apps/workbench workbench
614 test_doclinkchecker() {
618 ARVADOS_API_HOST=qr1hi.arvadosapi.com
619 # Make sure python-epydoc is installed or the next line won't
621 PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
624 do_test doc doclinkchecker
629 cd "$WORKSPACE/services/api" \
630 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
632 do_test services/api apiserver
634 # Shortcut for when we're only running apiserver tests. This saves a bit of time,
635 # because we don't need to start up the api server for subsequent tests.
636 if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
637 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
644 cd "$WORKSPACE/sdk/ruby" \
645 && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
647 do_test sdk/ruby ruby_sdk
650 cd "$WORKSPACE/sdk/cli" \
651 && mkdir -p /tmp/keep \
652 && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
656 for p in "${pythonstuff[@]}"
661 for g in "${gostuff[@]}"
667 start_nginx_proxy_services \
668 && cd "$WORKSPACE/apps/workbench" \
669 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
671 do_test apps/workbench workbench
673 test_workbench_benchmark() {
674 start_nginx_proxy_services \
675 && cd "$WORKSPACE/apps/workbench" \
676 && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
678 do_test apps/workbench_benchmark workbench_benchmark
680 test_workbench_profile() {
681 start_nginx_proxy_services \
682 && cd "$WORKSPACE/apps/workbench" \
683 && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
685 do_test apps/workbench_profile workbench_profile