3 read -rd "\000" helpmessage <<EOF
4 $(basename $0): Install and test Arvados components.
6 Exit non-zero if any tests fail.
9 $(basename $0) WORKSPACE=/path/to/arvados [options]
13 --skip FOO Do not test the FOO component.
14 --only FOO Do not test anything except the FOO component.
15 --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
16 Instead, show which directories were used this time so they
17 can be reused in subsequent invocations.
18 --skip-install Do not run any install steps. Just run tests.
19 You should provide GOPATH, GEMHOME, and VENVDIR options
20 from a previous invocation if you use this option.
21 --only-install Run specific install step
22 WORKSPACE=path Arvados source tree to test.
23 CONFIGSRC=path Dir with api server config files to copy into source tree.
24 (If none given, leave config files alone in source tree.)
25 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
26 Restrict apiserver tests to the given file
27 sdk/python_test="--test-suite test.test_keep_locator"
28 Restrict Python SDK tests to the given class
29 apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
30 Restrict Workbench tests to the given file
31 services/arv-git-httpd_test="-check.vv"
32 Show all log messages, even when tests pass (also works
33 with services/keepstore_test etc.)
35 Print more debug messages
36 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
37 *_test, and other examples shown above.
39 Assuming --skip-install is not given, all components are installed
40 into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
41 test suites depend on other components being installed, and installing
42 everything tends to be quicker than debugging dependencies.
44 As a special concession to the current CI server config, CONFIGSRC
45 defaults to $HOME/arvados-api-server if that directory exists.
47 More information and background:
49 https://arvados.org/projects/arvados/wiki/Running_tests
54 apps/workbench_benchmark
55 apps/workbench_profile
59 services/dockercleaner
64 services/arv-git-httpd
74 # First make sure to remove any ARVADOS_ variables from the calling
75 # environment that could interfere with the tests.
76 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
78 # Reset other variables that could affect our [tests'] behavior by
95 for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
97 if [[ -z "${leave_temp[$var]}" ]]
104 leaving+=" $var=\"${!var}\""
107 if [[ -n "$leaving" ]]; then
108 echo "Leaving behind temp dirs: $leaving"
114 echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
119 for x in "${successes[@]}"
124 if [[ ${#failures[@]} == 0 ]]
126 echo "All test suites passed."
128 echo "Failures (${#failures[@]}):"
129 for x in "${failures[@]}"
138 rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
140 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
147 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
148 || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
149 echo Checking dependencies:
150 echo -n 'virtualenv: '
151 virtualenv --version \
152 || fatal "No virtualenv. Try: apt-get install virtualenv"
155 || fatal "No go binary. See http://golang.org/doc/install"
157 gcc --version | egrep ^gcc \
158 || fatal "No gcc. Try: apt-get install build-essential"
160 find /usr/include -wholename '*fuse/fuse.h' \
161 || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
162 echo -n 'pyconfig.h: '
163 find /usr/include -name pyconfig.h | egrep --max-count=1 . \
164 || fatal "No pyconfig.h. Try: apt-get install python-dev"
166 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
167 || fatal "No nginx. Try: apt-get install nginx"
171 # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
172 if [[ -f "$1/$2" ]]; then
173 THEDATE=`date +%Y%m%d%H%M%S`
174 mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
175 gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
182 skip[apps/workbench_profile]=1
189 echo >&2 "$helpmessage"
198 only="$1"; skip[$1]=""; shift
205 only_install="$1"; shift
208 leave_temp[VENVDIR]=1
209 leave_temp[VENV3DIR]=1
211 leave_temp[GEMHOME]=1
217 suite="${arg%%_test=*}"
219 testargs["$suite"]="$args"
222 eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
225 echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
232 echo 'Starting API server...'
234 && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
235 && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
236 && export ARVADOS_TEST_API_INSTALLED="$$" \
237 && (env | egrep ^ARVADOS)
240 start_nginx_proxy_services() {
241 echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
243 && python sdk/python/tests/run_test_server.py start_keep_proxy \
244 && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
245 && python sdk/python/tests/run_test_server.py start_nginx \
246 && export ARVADOS_TEST_PROXY_SERVICES=1
250 if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
251 unset ARVADOS_TEST_PROXY_SERVICES
253 && python sdk/python/tests/run_test_server.py stop_nginx \
254 && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
255 && python sdk/python/tests/run_test_server.py stop_keep_proxy
257 if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
258 unset ARVADOS_TEST_API_HOST
260 && python sdk/python/tests/run_test_server.py stop
265 failures+=("($(basename $0) interrupted)")
272 echo "WORKSPACE=$WORKSPACE"
274 if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
275 # Jenkins expects us to use this by default.
276 CONFIGSRC="$HOME/arvados-api-server"
279 # Clean up .pyc files that may exist in the workspace
281 find -name '*.pyc' -delete
283 # Set up temporary install dirs (unless existing dirs were supplied)
284 for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
286 if [[ -n "${!tmpdir}" ]]; then
287 leave_temp[$tmpdir]=1
289 eval $tmpdir=$(mktemp -d)
293 setup_ruby_environment() {
294 if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
295 source "$HOME/.rvm/scripts/rvm"
297 elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
298 source "/usr/local/rvm/scripts/rvm"
304 if [[ "$using_rvm" == true ]]; then
305 # If rvm is in use, we can't just put separate "dependencies"
306 # and "gems-under-test" paths to GEM_PATH: passenger resets
307 # the environment to the "current gemset", which would lose
308 # our GEM_PATH and prevent our test suites from running ruby
309 # programs (for example, the Workbench test suite could not
310 # boot an API server or run arv). Instead, we have to make an
311 # rvm gemset and use it for everything.
313 [[ `type rvm | head -n1` == "rvm is a function" ]] \
316 # Put rvm's favorite path back in first place (overriding
317 # virtualenv, which just put itself there). Ignore rvm's
318 # complaint about not being in first place already.
319 rvm use @default 2>/dev/null
321 # Create (if needed) and switch to an @arvados-tests
322 # gemset. (Leave the choice of ruby to the caller.)
323 rvm use @arvados-tests --create \
324 || fatal 'rvm gemset setup'
328 # When our "bundle install"s need to install new gems to
329 # satisfy dependencies, we want them to go where "gem install
330 # --user-install" would put them. (However, if the caller has
331 # already set GEM_HOME, we assume that's where dependencies
332 # should be installed, and we should leave it alone.)
334 if [ -z "$GEM_HOME" ]; then
335 user_gempath="$(gem env gempath)"
336 export GEM_HOME="${user_gempath%%:*}"
338 PATH="$(gem env gemdir)/bin:$PATH"
340 # When we build and install our own gems, we install them in our
341 # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
342 # PATH so integration tests prefer them over other versions that
343 # happen to be installed in $user_gempath, system dirs, etc.
345 tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
346 PATH="$tmpdir_gem_home/bin:$PATH"
347 export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
349 echo "Will install dependencies to $(gem env gemdir)"
350 echo "Will install arvados gems to $tmpdir_gem_home"
351 echo "Gem search path is GEM_PATH=$GEM_PATH"
356 if [[ "$using_rvm" == true ]]; then
359 GEM_HOME="$tmpdir_gem_home" "$@"
364 mkdir -p "$GOPATH/src/git.curoverse.com"
365 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
366 || fatal "symlink failed"
368 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
369 . "$VENVDIR/bin/activate"
371 # Note: this must be the last time we change PATH, otherwise rvm will
373 setup_ruby_environment
377 if ! which bundler >/dev/null
379 gem install --user-install bundler || fatal 'Could not install bundler'
382 # Needed for run_test_server.py which is used by certain (non-Python) tests.
383 pip freeze 2>/dev/null | egrep ^PyYAML= \
384 || pip install PyYAML \
385 || fatal "pip install PyYAML failed"
387 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
388 # Otherwise, skip dependent tests.
389 PYTHON3=$(which python3)
390 if [ "0" = "$?" ]; then
391 virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
392 || fatal "python3 virtualenv $VENV3DIR failed"
395 skip[services/dockercleaner]=1
398 Warning: python3 could not be found
399 services/dockercleaner install and tests will be skipped
405 if [[ "$1" != "0" ]]; then
406 title "!!!!!! $2 FAILED !!!!!!"
407 failures+=("$2 (`timer`)")
409 successes+=("$2 (`timer`)")
418 echo -n "$(($SECONDS - $t0))s"
422 while ! do_test_once ${@} && [[ "$retry" == 1 ]]
424 read -p 'Try again? [Y/n] ' x
425 if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
433 if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
435 title "Running $1 tests"
437 if [[ "$2" == "go" ]]
439 if [[ -n "${testargs[$1]}" ]]
441 # "go test -check.vv giturl" doesn't work, but this
443 cd "$WORKSPACE/$1" && go test ${testargs[$1]}
445 # The above form gets verbose even when testargs is
446 # empty, so use this form in such cases:
447 go test "git.curoverse.com/arvados.git/$1"
449 elif [[ "$2" == "pip" ]]
451 # $3 can name a path directory for us to use, including trailing
452 # slash; e.g., the bin/ subdirectory of a virtualenv.
454 && "${3}python" setup.py test ${testargs[$1]}
455 elif [[ "$2" != "" ]]
462 checkexit $result "$1 tests"
463 title "End of $1 tests (`timer`)"
466 title "Skipping $1 tests"
471 if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
473 title "Running $1 install"
475 if [[ "$2" == "go" ]]
477 go get -t "git.curoverse.com/arvados.git/$1"
478 elif [[ "$2" == "pip" ]]
480 # $3 can name a path directory for us to use, including trailing
481 # slash; e.g., the bin/ subdirectory of a virtualenv.
483 # Need to change to a different directory after creating
484 # the source dist package to avoid a pip bug.
485 # see https://arvados.org/issues/5766 for details.
487 # Also need to install twice, because if it belives the package is
488 # already installed, pip it won't install it. So the first "pip
489 # install" ensures that the dependencies are met, the second "pip
490 # install" ensures that we've actually install the local package
493 && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
495 && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
496 && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
497 elif [[ "$2" != "" ]]
503 checkexit $? "$1 install"
504 title "End of $1 install (`timer`)"
506 title "Skipping $1 install"
511 txt="********** $1 **********"
512 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
515 bundle_install_trylocal() {
518 echo "(Running bundle install --local. 'could not find package' messages are OK.)"
519 if ! bundle install --local --no-deployment; then
520 echo "(Running bundle install again, without --local.)"
521 bundle install --no-deployment
528 cd "$WORKSPACE/doc" \
529 && bundle_install_trylocal \
535 with_test_gemset gem uninstall --force --all --executables arvados \
536 && cd "$WORKSPACE/sdk/ruby" \
537 && bundle_install_trylocal \
538 && gem build arvados.gemspec \
539 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
541 do_install sdk/ruby ruby_sdk
544 with_test_gemset gem uninstall --force --all --executables arvados-cli \
545 && cd "$WORKSPACE/sdk/cli" \
546 && bundle_install_trylocal \
547 && gem build arvados-cli.gemspec \
548 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
550 do_install sdk/cli cli
552 # Install the Python SDK early. Various other test suites (like
553 # keepproxy) bring up run_test_server.py, which imports the arvados
554 # module. We can't actually *test* the Python SDK yet though, because
555 # its own test suite brings up some of those other programs (like
557 declare -a pythonstuff
563 for p in "${pythonstuff[@]}"
567 if [ -n "$PYTHON3" ]; then
568 do_install services/dockercleaner pip "$VENV3DIR/bin/"
571 install_apiserver() {
572 cd "$WORKSPACE/services/api" \
573 && RAILS_ENV=test bundle_install_trylocal
575 rm -f config/environments/test.rb
576 cp config/environments/test.rb.example config/environments/test.rb
578 if [ -n "$CONFIGSRC" ]
580 for f in database.yml application.yml
582 cp "$CONFIGSRC/$f" config/ || fatal "$f"
586 # Fill in a random secret_token and blob_signing_key for testing
587 SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
588 BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
590 sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
591 sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
593 # Set up empty git repo (for git tests)
595 sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
598 mkdir -p $GITDIR/test
601 && git config user.email "jenkins@ci.curoverse.com" \
602 && git config user.name "Jenkins, CI" \
605 && git commit -m 'initial commit'
607 # Clear out any lingering postgresql connections to the test
608 # database, so that we can drop it. This assumes the current user
609 # is a postgresql superuser.
610 test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
611 psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
613 cd "$WORKSPACE/services/api" \
614 && RAILS_ENV=test bundle exec rake db:drop \
615 && RAILS_ENV=test bundle exec rake db:setup \
616 && RAILS_ENV=test bundle exec rake db:fixtures:load
618 do_install services/api apiserver
622 services/arv-git-httpd
630 for g in "${gostuff[@]}"
635 install_workbench() {
636 cd "$WORKSPACE/apps/workbench" \
637 && mkdir -p tmp/cache \
638 && RAILS_ENV=test bundle_install_trylocal
640 do_install apps/workbench workbench
642 test_doclinkchecker() {
646 ARVADOS_API_HOST=qr1hi.arvadosapi.com
647 # Make sure python-epydoc is installed or the next line won't
649 PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
652 do_test doc doclinkchecker
657 cd "$WORKSPACE/services/api" \
658 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
660 do_test services/api apiserver
662 # Shortcut for when we're only running apiserver tests. This saves a bit of time,
663 # because we don't need to start up the api server for subsequent tests.
664 if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
665 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
672 cd "$WORKSPACE/sdk/ruby" \
673 && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
675 do_test sdk/ruby ruby_sdk
678 cd "$WORKSPACE/sdk/cli" \
679 && mkdir -p /tmp/keep \
680 && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
684 for p in "${pythonstuff[@]}"
688 do_test services/dockercleaner pip "$VENV3DIR/bin/"
690 for g in "${gostuff[@]}"
696 start_nginx_proxy_services \
697 && cd "$WORKSPACE/apps/workbench" \
698 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
700 do_test apps/workbench workbench
702 test_workbench_benchmark() {
703 start_nginx_proxy_services \
704 && cd "$WORKSPACE/apps/workbench" \
705 && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
707 do_test apps/workbench_benchmark workbench_benchmark
709 test_workbench_profile() {
710 start_nginx_proxy_services \
711 && cd "$WORKSPACE/apps/workbench" \
712 && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
714 do_test apps/workbench_profile workbench_profile