3 read -rd "\000" helpmessage <<EOF
4 $(basename $0): Install and test Arvados components.
6 Exit non-zero if any tests fail.
9 $(basename $0) WORKSPACE=/path/to/arvados [options]
13 --skip FOO Do not test the FOO component.
14 --only FOO Do not test anything except the FOO component.
15 --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
16 Instead, show which directories were used this time so they
17 can be reused in subsequent invocations.
18 --skip-install Do not run any install steps. Just run tests.
19 You should provide GOPATH, GEMHOME, and VENVDIR options
20 from a previous invocation if you use this option.
21 --only-install Run specific install step
22 WORKSPACE=path Arvados source tree to test.
23 CONFIGSRC=path Dir with api server config files to copy into source tree.
24 (If none given, leave config files alone in source tree.)
25 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
26 Restrict apiserver tests to the given file
27 sdk/python_test="--test-suite test.test_keep_locator"
28 Restrict Python SDK tests to the given class
29 apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
30 Restrict Workbench tests to the given file
31 services/arv-git-httpd_test="-check.vv"
32 Show all log messages, even when tests pass (also works
33 with services/keepstore_test etc.)
35 Print more debug messages
36 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
37 *_test, and other examples shown above.
39 Assuming --skip-install is not given, all components are installed
40 into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
41 test suites depend on other components being installed, and installing
42 everything tends to be quicker than debugging dependencies.
44 As a special concession to the current CI server config, CONFIGSRC
45 defaults to $HOME/arvados-api-server if that directory exists.
47 More information and background:
49 https://arvados.org/projects/arvados/wiki/Running_tests
54 apps/workbench_benchmark
55 apps/workbench_profile
59 services/dockercleaner
65 services/arv-git-httpd
76 # First make sure to remove any ARVADOS_ variables from the calling
77 # environment that could interfere with the tests.
78 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
80 # Reset other variables that could affect our [tests'] behavior by
98 for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME PERLINSTALLBASE
100 if [[ -z "${leave_temp[$var]}" ]]
102 if [[ -n "${!var}" ]]
107 leaving+=" $var=\"${!var}\""
110 if [[ -n "$leaving" ]]; then
111 echo "Leaving behind temp dirs: $leaving"
117 echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
122 for x in "${successes[@]}"
127 if [[ ${#failures[@]} == 0 ]]
129 echo "All test suites passed."
131 echo "Failures (${#failures[@]}):"
132 for x in "${failures[@]}"
141 create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
142 rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
144 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
151 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
152 || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
153 echo Checking dependencies:
154 echo -n 'virtualenv: '
155 virtualenv --version \
156 || fatal "No virtualenv. Try: apt-get install virtualenv"
159 || fatal "No go binary. See http://golang.org/doc/install"
161 gcc --version | egrep ^gcc \
162 || fatal "No gcc. Try: apt-get install build-essential"
164 find /usr/include -wholename '*fuse/fuse.h' \
165 || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
166 echo -n 'pyconfig.h: '
167 find /usr/include -name pyconfig.h | egrep --max-count=1 . \
168 || fatal "No pyconfig.h. Try: apt-get install python-dev"
170 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
171 || fatal "No nginx. Try: apt-get install nginx"
173 perl -v | grep version \
174 || fatal "No perl. Try: apt-get install perl"
175 for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
176 echo -n "perl $mod: "
177 perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
178 || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl"
182 || fatal "No gitolite. Try: apt-get install gitolite3"
186 # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
187 # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
188 if [[ -f "$1/$2" ]]; then
189 THEDATE=`date +%Y%m%d%H%M%S`
190 mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
191 gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
198 skip[apps/workbench_profile]=1
205 echo >&2 "$helpmessage"
214 only="$1"; skip[$1]=""; shift
221 only_install="$1"; shift
224 leave_temp[VENVDIR]=1
225 leave_temp[VENV3DIR]=1
227 leave_temp[GEMHOME]=1
228 leave_temp[PERLINSTALLBASE]=1
234 suite="${arg%%_test=*}"
236 testargs["$suite"]="$args"
239 eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
242 echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
249 echo 'Starting API server...'
251 && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
252 && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
253 && export ARVADOS_TEST_API_INSTALLED="$$" \
254 && (env | egrep ^ARVADOS)
257 start_nginx_proxy_services() {
258 echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
260 && python sdk/python/tests/run_test_server.py start_keep_proxy \
261 && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
262 && python sdk/python/tests/run_test_server.py start_nginx \
263 && export ARVADOS_TEST_PROXY_SERVICES=1
267 if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
268 unset ARVADOS_TEST_PROXY_SERVICES
270 && python sdk/python/tests/run_test_server.py stop_nginx \
271 && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
272 && python sdk/python/tests/run_test_server.py stop_keep_proxy
274 if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
275 unset ARVADOS_TEST_API_HOST
277 && python sdk/python/tests/run_test_server.py stop
282 failures+=("($(basename $0) interrupted)")
289 echo "WORKSPACE=$WORKSPACE"
291 if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
292 # Jenkins expects us to use this by default.
293 CONFIGSRC="$HOME/arvados-api-server"
296 # Clean up .pyc files that may exist in the workspace
298 find -name '*.pyc' -delete
300 # Set up temporary install dirs (unless existing dirs were supplied)
301 for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
303 if [[ -n "${!tmpdir}" ]]; then
304 leave_temp[$tmpdir]=1
305 mkdir -p "${!tmpdir}"
307 eval "$tmpdir"='$(mktemp -d)'
311 setup_ruby_environment() {
312 if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
313 source "$HOME/.rvm/scripts/rvm"
315 elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
316 source "/usr/local/rvm/scripts/rvm"
322 if [[ "$using_rvm" == true ]]; then
323 # If rvm is in use, we can't just put separate "dependencies"
324 # and "gems-under-test" paths to GEM_PATH: passenger resets
325 # the environment to the "current gemset", which would lose
326 # our GEM_PATH and prevent our test suites from running ruby
327 # programs (for example, the Workbench test suite could not
328 # boot an API server or run arv). Instead, we have to make an
329 # rvm gemset and use it for everything.
331 [[ `type rvm | head -n1` == "rvm is a function" ]] \
334 # Put rvm's favorite path back in first place (overriding
335 # virtualenv, which just put itself there). Ignore rvm's
336 # complaint about not being in first place already.
337 rvm use @default 2>/dev/null
339 # Create (if needed) and switch to an @arvados-tests
340 # gemset. (Leave the choice of ruby to the caller.)
341 rvm use @arvados-tests --create \
342 || fatal 'rvm gemset setup'
346 # When our "bundle install"s need to install new gems to
347 # satisfy dependencies, we want them to go where "gem install
348 # --user-install" would put them. (However, if the caller has
349 # already set GEM_HOME, we assume that's where dependencies
350 # should be installed, and we should leave it alone.)
352 if [ -z "$GEM_HOME" ]; then
353 user_gempath="$(gem env gempath)"
354 export GEM_HOME="${user_gempath%%:*}"
356 PATH="$(gem env gemdir)/bin:$PATH"
358 # When we build and install our own gems, we install them in our
359 # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
360 # PATH so integration tests prefer them over other versions that
361 # happen to be installed in $user_gempath, system dirs, etc.
363 tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
364 PATH="$tmpdir_gem_home/bin:$PATH"
365 export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
367 echo "Will install dependencies to $(gem env gemdir)"
368 echo "Will install arvados gems to $tmpdir_gem_home"
369 echo "Gem search path is GEM_PATH=$GEM_PATH"
374 if [[ "$using_rvm" == true ]]; then
377 GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
381 gem_uninstall_if_exists() {
382 if gem list "$1\$" | egrep '^\w'; then
383 gem uninstall --force --all --executables "$1"
387 export PERLINSTALLBASE
388 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
391 mkdir -p "$GOPATH/src/git.curoverse.com"
392 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
393 || fatal "symlink failed"
395 if ! [[ -e "$VENVDIR/bin/activate" ]] || ! [[ -e "$VENVDIR/bin/pip" ]]; then
396 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
398 . "$VENVDIR/bin/activate"
400 if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\ -f2 | cut -d. -f1)" -lt 18 ]; then
401 pip install --upgrade setuptools pip
404 # Note: this must be the last time we change PATH, otherwise rvm will
406 setup_ruby_environment
410 if ! which bundler >/dev/null
412 gem install --user-install bundler || fatal 'Could not install bundler'
415 # Needed for run_test_server.py which is used by certain (non-Python) tests.
416 pip freeze 2>/dev/null | egrep ^PyYAML= \
417 || pip install PyYAML >/dev/null \
418 || fatal "pip install PyYAML failed"
420 # Preinstall forked version of libcloud, because nodemanager "pip install"
421 # won't pick it up by default.
422 pip freeze 2>/dev/null | egrep ^apache-libcloud==0.18.1.dev2 \
423 || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev2.zip >/dev/null \
424 || fatal "pip install apache-libcloud failed"
426 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
427 # Otherwise, skip dependent tests.
428 PYTHON3=$(which python3)
429 if [ "0" = "$?" ]; then
430 virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
431 || fatal "python3 virtualenv $VENV3DIR failed"
434 skip[services/dockercleaner]=1
437 Warning: python3 could not be found
438 services/dockercleaner install and tests will be skipped
444 if [[ "$1" != "0" ]]; then
445 title "!!!!!! $2 FAILED !!!!!!"
446 failures+=("$2 (`timer`)")
448 successes+=("$2 (`timer`)")
457 echo -n "$(($SECONDS - $t0))s"
461 while ! do_test_once ${@} && [[ "$retry" == 1 ]]
463 read -p 'Try again? [Y/n] ' x
464 if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
473 if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
475 title "Running $1 tests"
477 if [[ "$2" == "go" ]]
479 covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
480 coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
481 if [[ -n "${testargs[$1]}" ]]
483 # "go test -check.vv giturl" doesn't work, but this
485 cd "$WORKSPACE/$1" && go test ${coverflags[@]} ${testargs[$1]}
487 # The above form gets verbose even when testargs is
488 # empty, so use this form in such cases:
489 go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
492 go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
493 rm "$WORKSPACE/tmp/.$covername.tmp"
494 elif [[ "$2" == "pip" ]]
496 # $3 can name a path directory for us to use, including trailing
497 # slash; e.g., the bin/ subdirectory of a virtualenv.
499 && "${3}python" setup.py test ${testargs[$1]}
500 elif [[ "$2" != "" ]]
507 checkexit $result "$1 tests"
508 title "End of $1 tests (`timer`)"
511 title "Skipping $1 tests"
516 if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
518 title "Running $1 install"
520 if [[ "$2" == "go" ]]
522 go get -t "git.curoverse.com/arvados.git/$1"
523 elif [[ "$2" == "pip" ]]
525 # $3 can name a path directory for us to use, including trailing
526 # slash; e.g., the bin/ subdirectory of a virtualenv.
528 # Need to change to a different directory after creating
529 # the source dist package to avoid a pip bug.
530 # see https://arvados.org/issues/5766 for details.
532 # Also need to install twice, because if it believes the package is
533 # already installed, pip it won't install it. So the first "pip
534 # install" ensures that the dependencies are met, the second "pip
535 # install" ensures that we've actually installed the local package
538 && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
540 && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
541 && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
542 elif [[ "$2" != "" ]]
548 checkexit $? "$1 install"
549 title "End of $1 install (`timer`)"
551 title "Skipping $1 install"
556 txt="********** $1 **********"
557 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
560 bundle_install_trylocal() {
563 echo "(Running bundle install --local. 'could not find package' messages are OK.)"
564 if ! bundle install --local --no-deployment; then
565 echo "(Running bundle install again, without --local.)"
566 bundle install --no-deployment
573 cd "$WORKSPACE/doc" \
574 && bundle_install_trylocal \
582 with_test_gemset gem_uninstall_if_exists "$gemname" \
583 && cd "$WORKSPACE/$srcpath" \
584 && bundle_install_trylocal \
585 && gem build "$gemname.gemspec" \
586 && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
590 install_gem arvados sdk/ruby
592 do_install sdk/ruby ruby_sdk
595 cd "$WORKSPACE/sdk/perl" \
596 && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
597 && make install INSTALLDIRS=perl
599 do_install sdk/perl perl_sdk
602 install_gem arvados-cli sdk/cli
604 do_install sdk/cli cli
606 install_login-sync() {
607 install_gem arvados-login-sync services/login-sync
609 do_install services/login-sync login-sync
611 # Install the Python SDK early. Various other test suites (like
612 # keepproxy) bring up run_test_server.py, which imports the arvados
613 # module. We can't actually *test* the Python SDK yet though, because
614 # its own test suite brings up some of those other programs (like
616 declare -a pythonstuff
623 for p in "${pythonstuff[@]}"
627 if [ -n "$PYTHON3" ]; then
628 do_install services/dockercleaner pip "$VENV3DIR/bin/"
631 install_apiserver() {
632 cd "$WORKSPACE/services/api" \
633 && RAILS_ENV=test bundle_install_trylocal
635 rm -f config/environments/test.rb
636 cp config/environments/test.rb.example config/environments/test.rb
638 if [ -n "$CONFIGSRC" ]
640 for f in database.yml application.yml
642 cp "$CONFIGSRC/$f" config/ || fatal "$f"
646 # Fill in a random secret_token and blob_signing_key for testing
647 SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
648 BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
650 sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
651 sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
653 # Set up empty git repo (for git tests)
655 sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
658 mkdir -p $GITDIR/test
661 && git config user.email "jenkins@ci.curoverse.com" \
662 && git config user.name "Jenkins, CI" \
665 && git commit -m 'initial commit'
667 # Clear out any lingering postgresql connections to the test
668 # database, so that we can drop it. This assumes the current user
669 # is a postgresql superuser.
670 cd "$WORKSPACE/services/api" \
671 && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
672 && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
674 cd "$WORKSPACE/services/api" \
675 && RAILS_ENV=test bundle exec rake db:drop \
676 && RAILS_ENV=test bundle exec rake db:setup \
677 && RAILS_ENV=test bundle exec rake db:fixtures:load
679 do_install services/api apiserver
683 services/arv-git-httpd
687 services/datamanager/summary
688 services/datamanager/collection
694 for g in "${gostuff[@]}"
699 install_workbench() {
700 cd "$WORKSPACE/apps/workbench" \
701 && mkdir -p tmp/cache \
702 && RAILS_ENV=test bundle_install_trylocal
704 do_install apps/workbench workbench
706 test_doclinkchecker() {
710 ARVADOS_API_HOST=qr1hi.arvadosapi.com
711 # Make sure python-epydoc is installed or the next line won't
713 PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
716 do_test doc doclinkchecker
721 cd "$WORKSPACE/services/api" \
722 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
724 do_test services/api apiserver
726 # Shortcut for when we're only running apiserver tests. This saves a bit of time,
727 # because we don't need to start up the api server for subsequent tests.
728 if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
729 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
736 cd "$WORKSPACE/sdk/ruby" \
737 && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
739 do_test sdk/ruby ruby_sdk
742 cd "$WORKSPACE/sdk/cli" \
743 && mkdir -p /tmp/keep \
744 && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
749 cd "$WORKSPACE/services/login-sync" \
750 && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
752 do_test services/login-sync login-sync
754 for p in "${pythonstuff[@]}"
758 do_test services/dockercleaner pip "$VENV3DIR/bin/"
760 for g in "${gostuff[@]}"
762 if [[ "$g" == "services/datamanager" ]]
764 testargs["$g"]+=" -data-manager-token-file $WORKSPACE/tmp/keep.data-manager-token-file"
770 start_nginx_proxy_services \
771 && cd "$WORKSPACE/apps/workbench" \
772 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
774 do_test apps/workbench workbench
776 test_workbench_benchmark() {
777 start_nginx_proxy_services \
778 && cd "$WORKSPACE/apps/workbench" \
779 && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
781 do_test apps/workbench_benchmark workbench_benchmark
783 test_workbench_profile() {
784 start_nginx_proxy_services \
785 && cd "$WORKSPACE/apps/workbench" \
786 && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
788 do_test apps/workbench_profile workbench_profile