3 read -rd "\000" helpmessage <<EOF
4 $(basename $0): Install and test Arvados components.
6 Exit non-zero if any tests fail.
9 $(basename $0) WORKSPACE=/path/to/arvados [options]
13 --skip FOO Do not test the FOO component.
14 --only FOO Do not test anything except the FOO component.
15 --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
16 Instead, show which directories were used this time so they
17 can be reused in subsequent invocations.
18 --skip-install Do not run any install steps. Just run tests.
19 You should provide GOPATH, GEMHOME, and VENVDIR options
20 from a previous invocation if you use this option.
21 --only-install Run specific install step
22 WORKSPACE=path Arvados source tree to test.
23 CONFIGSRC=path Dir with api server config files to copy into source tree.
24 (If none given, leave config files alone in source tree.)
25 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
26 Restrict apiserver tests to the given file
27 sdk/python_test="--test-suite test.test_keep_locator"
28 Restrict Python SDK tests to the given class
29 apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
30 Restrict Workbench tests to the given file
31 services/arv-git-httpd_test="-check.vv"
32 Show all log messages, even when tests pass (also works
33 with services/keepstore_test etc.)
35 Print more debug messages
36 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
37 *_test, and other examples shown above.
39 Assuming --skip-install is not given, all components are installed
40 into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
41 test suites depend on other components being installed, and installing
42 everything tends to be quicker than debugging dependencies.
44 As a special concession to the current CI server config, CONFIGSRC
45 defaults to $HOME/arvados-api-server if that directory exists.
47 More information and background:
49 https://arvados.org/projects/arvados/wiki/Running_tests
54 apps/workbench_benchmark
55 apps/workbench_profile
59 services/dockercleaner
64 services/arv-git-httpd
75 # First make sure to remove any ARVADOS_ variables from the calling
76 # environment that could interfere with the tests.
77 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
79 # Reset other variables that could affect our [tests'] behavior by
97 for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME PERLINSTALLBASE
99 if [[ -z "${leave_temp[$var]}" ]]
101 if [[ -n "${!var}" ]]
106 leaving+=" $var=\"${!var}\""
109 if [[ -n "$leaving" ]]; then
110 echo "Leaving behind temp dirs: $leaving"
116 echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
121 for x in "${successes[@]}"
126 if [[ ${#failures[@]} == 0 ]]
128 echo "All test suites passed."
130 echo "Failures (${#failures[@]}):"
131 for x in "${failures[@]}"
140 create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
141 rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
143 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
150 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
151 || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
152 echo Checking dependencies:
153 echo -n 'virtualenv: '
154 virtualenv --version \
155 || fatal "No virtualenv. Try: apt-get install virtualenv"
158 || fatal "No go binary. See http://golang.org/doc/install"
160 gcc --version | egrep ^gcc \
161 || fatal "No gcc. Try: apt-get install build-essential"
163 find /usr/include -wholename '*fuse/fuse.h' \
164 || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
165 echo -n 'pyconfig.h: '
166 find /usr/include -name pyconfig.h | egrep --max-count=1 . \
167 || fatal "No pyconfig.h. Try: apt-get install python-dev"
169 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
170 || fatal "No nginx. Try: apt-get install nginx"
172 perl -v | grep version \
173 || fatal "No perl. Try: apt-get install perl"
174 for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
175 echo -n "perl $mod: "
176 perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
177 || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl"
182 # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
183 # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
184 if [[ -f "$1/$2" ]]; then
185 THEDATE=`date +%Y%m%d%H%M%S`
186 mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
187 gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
194 skip[apps/workbench_profile]=1
201 echo >&2 "$helpmessage"
210 only="$1"; skip[$1]=""; shift
217 only_install="$1"; shift
220 leave_temp[VENVDIR]=1
221 leave_temp[VENV3DIR]=1
223 leave_temp[GEMHOME]=1
224 leave_temp[PERLINSTALLBASE]=1
230 suite="${arg%%_test=*}"
232 testargs["$suite"]="$args"
235 eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
238 echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
245 echo 'Starting API server...'
247 && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
248 && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
249 && export ARVADOS_TEST_API_INSTALLED="$$" \
250 && (env | egrep ^ARVADOS)
253 start_nginx_proxy_services() {
254 echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
256 && python sdk/python/tests/run_test_server.py start_keep_proxy \
257 && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
258 && python sdk/python/tests/run_test_server.py start_nginx \
259 && export ARVADOS_TEST_PROXY_SERVICES=1
263 if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
264 unset ARVADOS_TEST_PROXY_SERVICES
266 && python sdk/python/tests/run_test_server.py stop_nginx \
267 && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
268 && python sdk/python/tests/run_test_server.py stop_keep_proxy
270 if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
271 unset ARVADOS_TEST_API_HOST
273 && python sdk/python/tests/run_test_server.py stop
278 failures+=("($(basename $0) interrupted)")
285 echo "WORKSPACE=$WORKSPACE"
287 if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
288 # Jenkins expects us to use this by default.
289 CONFIGSRC="$HOME/arvados-api-server"
292 # Clean up .pyc files that may exist in the workspace
294 find -name '*.pyc' -delete
296 # Set up temporary install dirs (unless existing dirs were supplied)
297 for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
299 if [[ -n "${!tmpdir}" ]]; then
300 leave_temp[$tmpdir]=1
301 mkdir -p "${!tmpdir}"
303 eval "$tmpdir"='$(mktemp -d)'
307 setup_ruby_environment() {
308 if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
309 source "$HOME/.rvm/scripts/rvm"
311 elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
312 source "/usr/local/rvm/scripts/rvm"
318 if [[ "$using_rvm" == true ]]; then
319 # If rvm is in use, we can't just put separate "dependencies"
320 # and "gems-under-test" paths to GEM_PATH: passenger resets
321 # the environment to the "current gemset", which would lose
322 # our GEM_PATH and prevent our test suites from running ruby
323 # programs (for example, the Workbench test suite could not
324 # boot an API server or run arv). Instead, we have to make an
325 # rvm gemset and use it for everything.
327 [[ `type rvm | head -n1` == "rvm is a function" ]] \
330 # Put rvm's favorite path back in first place (overriding
331 # virtualenv, which just put itself there). Ignore rvm's
332 # complaint about not being in first place already.
333 rvm use @default 2>/dev/null
335 # Create (if needed) and switch to an @arvados-tests
336 # gemset. (Leave the choice of ruby to the caller.)
337 rvm use @arvados-tests --create \
338 || fatal 'rvm gemset setup'
342 # When our "bundle install"s need to install new gems to
343 # satisfy dependencies, we want them to go where "gem install
344 # --user-install" would put them. (However, if the caller has
345 # already set GEM_HOME, we assume that's where dependencies
346 # should be installed, and we should leave it alone.)
348 if [ -z "$GEM_HOME" ]; then
349 user_gempath="$(gem env gempath)"
350 export GEM_HOME="${user_gempath%%:*}"
352 PATH="$(gem env gemdir)/bin:$PATH"
354 # When we build and install our own gems, we install them in our
355 # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
356 # PATH so integration tests prefer them over other versions that
357 # happen to be installed in $user_gempath, system dirs, etc.
359 tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
360 PATH="$tmpdir_gem_home/bin:$PATH"
361 export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
363 echo "Will install dependencies to $(gem env gemdir)"
364 echo "Will install arvados gems to $tmpdir_gem_home"
365 echo "Gem search path is GEM_PATH=$GEM_PATH"
370 if [[ "$using_rvm" == true ]]; then
373 GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
377 gem_uninstall_if_exists() {
378 if gem list "$1\$" | egrep '^\w'; then
379 gem uninstall --force --all --executables "$1"
383 export PERLINSTALLBASE
384 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
387 mkdir -p "$GOPATH/src/git.curoverse.com"
388 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
389 || fatal "symlink failed"
391 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
392 . "$VENVDIR/bin/activate"
394 if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\ -f2 | cut -d. -f1)" -lt 18 ]; then
395 pip install --upgrade setuptools
398 # Note: this must be the last time we change PATH, otherwise rvm will
400 setup_ruby_environment
404 if ! which bundler >/dev/null
406 gem install --user-install bundler || fatal 'Could not install bundler'
409 # Needed for run_test_server.py which is used by certain (non-Python) tests.
410 pip freeze 2>/dev/null | egrep ^PyYAML= \
411 || pip install PyYAML >/dev/null \
412 || fatal "pip install PyYAML failed"
414 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
415 # Otherwise, skip dependent tests.
416 PYTHON3=$(which python3)
417 if [ "0" = "$?" ]; then
418 virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
419 || fatal "python3 virtualenv $VENV3DIR failed"
422 skip[services/dockercleaner]=1
425 Warning: python3 could not be found
426 services/dockercleaner install and tests will be skipped
432 if [[ "$1" != "0" ]]; then
433 title "!!!!!! $2 FAILED !!!!!!"
434 failures+=("$2 (`timer`)")
436 successes+=("$2 (`timer`)")
445 echo -n "$(($SECONDS - $t0))s"
449 while ! do_test_once ${@} && [[ "$retry" == 1 ]]
451 read -p 'Try again? [Y/n] ' x
452 if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
460 if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
462 title "Running $1 tests"
464 if [[ "$2" == "go" ]]
466 if [[ -n "${testargs[$1]}" ]]
468 # "go test -check.vv giturl" doesn't work, but this
470 cd "$WORKSPACE/$1" && go test ${testargs[$1]}
472 # The above form gets verbose even when testargs is
473 # empty, so use this form in such cases:
474 go test "git.curoverse.com/arvados.git/$1"
476 elif [[ "$2" == "pip" ]]
478 # $3 can name a path directory for us to use, including trailing
479 # slash; e.g., the bin/ subdirectory of a virtualenv.
481 && "${3}python" setup.py test ${testargs[$1]}
482 elif [[ "$2" != "" ]]
489 checkexit $result "$1 tests"
490 title "End of $1 tests (`timer`)"
493 title "Skipping $1 tests"
498 if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
500 title "Running $1 install"
502 if [[ "$2" == "go" ]]
504 go get -t "git.curoverse.com/arvados.git/$1"
505 elif [[ "$2" == "pip" ]]
507 # $3 can name a path directory for us to use, including trailing
508 # slash; e.g., the bin/ subdirectory of a virtualenv.
510 # Need to change to a different directory after creating
511 # the source dist package to avoid a pip bug.
512 # see https://arvados.org/issues/5766 for details.
514 # Also need to install twice, because if it believes the package is
515 # already installed, pip it won't install it. So the first "pip
516 # install" ensures that the dependencies are met, the second "pip
517 # install" ensures that we've actually installed the local package
520 && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
522 && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
523 && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
524 elif [[ "$2" != "" ]]
530 checkexit $? "$1 install"
531 title "End of $1 install (`timer`)"
533 title "Skipping $1 install"
538 txt="********** $1 **********"
539 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
542 bundle_install_trylocal() {
545 echo "(Running bundle install --local. 'could not find package' messages are OK.)"
546 if ! bundle install --local --no-deployment; then
547 echo "(Running bundle install again, without --local.)"
548 bundle install --no-deployment
555 cd "$WORKSPACE/doc" \
556 && bundle_install_trylocal \
562 with_test_gemset gem_uninstall_if_exists arvados \
563 && cd "$WORKSPACE/sdk/ruby" \
564 && bundle_install_trylocal \
565 && gem build arvados.gemspec \
566 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
568 do_install sdk/ruby ruby_sdk
571 cd "$WORKSPACE/sdk/perl" \
572 && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
573 && make install INSTALLDIRS=perl
575 do_install sdk/perl perl_sdk
578 with_test_gemset gem_uninstall_if_exists arvados-cli \
579 && cd "$WORKSPACE/sdk/cli" \
580 && bundle_install_trylocal \
581 && gem build arvados-cli.gemspec \
582 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
584 do_install sdk/cli cli
586 # Install the Python SDK early. Various other test suites (like
587 # keepproxy) bring up run_test_server.py, which imports the arvados
588 # module. We can't actually *test* the Python SDK yet though, because
589 # its own test suite brings up some of those other programs (like
591 declare -a pythonstuff
598 for p in "${pythonstuff[@]}"
602 if [ -n "$PYTHON3" ]; then
603 do_install services/dockercleaner pip "$VENV3DIR/bin/"
606 install_apiserver() {
607 cd "$WORKSPACE/services/api" \
608 && RAILS_ENV=test bundle_install_trylocal
610 rm -f config/environments/test.rb
611 cp config/environments/test.rb.example config/environments/test.rb
613 if [ -n "$CONFIGSRC" ]
615 for f in database.yml application.yml
617 cp "$CONFIGSRC/$f" config/ || fatal "$f"
621 # Fill in a random secret_token and blob_signing_key for testing
622 SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
623 BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
625 sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
626 sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
628 # Set up empty git repo (for git tests)
630 sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
633 mkdir -p $GITDIR/test
636 && git config user.email "jenkins@ci.curoverse.com" \
637 && git config user.name "Jenkins, CI" \
640 && git commit -m 'initial commit'
642 # Clear out any lingering postgresql connections to the test
643 # database, so that we can drop it. This assumes the current user
644 # is a postgresql superuser.
645 cd "$WORKSPACE/services/api" \
646 && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
647 && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
649 cd "$WORKSPACE/services/api" \
650 && RAILS_ENV=test bundle exec rake db:drop \
651 && RAILS_ENV=test bundle exec rake db:setup \
652 && RAILS_ENV=test bundle exec rake db:fixtures:load
654 do_install services/api apiserver
658 services/arv-git-httpd
662 services/datamanager/summary
663 services/datamanager/collection
668 for g in "${gostuff[@]}"
673 install_workbench() {
674 cd "$WORKSPACE/apps/workbench" \
675 && mkdir -p tmp/cache \
676 && RAILS_ENV=test bundle_install_trylocal
678 do_install apps/workbench workbench
680 test_doclinkchecker() {
684 ARVADOS_API_HOST=qr1hi.arvadosapi.com
685 # Make sure python-epydoc is installed or the next line won't
687 PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
690 do_test doc doclinkchecker
695 cd "$WORKSPACE/services/api" \
696 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
698 do_test services/api apiserver
700 # Shortcut for when we're only running apiserver tests. This saves a bit of time,
701 # because we don't need to start up the api server for subsequent tests.
702 if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
703 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
710 cd "$WORKSPACE/sdk/ruby" \
711 && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
713 do_test sdk/ruby ruby_sdk
716 cd "$WORKSPACE/sdk/cli" \
717 && mkdir -p /tmp/keep \
718 && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
722 for p in "${pythonstuff[@]}"
726 do_test services/dockercleaner pip "$VENV3DIR/bin/"
728 for g in "${gostuff[@]}"
734 start_nginx_proxy_services \
735 && cd "$WORKSPACE/apps/workbench" \
736 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
738 do_test apps/workbench workbench
740 test_workbench_benchmark() {
741 start_nginx_proxy_services \
742 && cd "$WORKSPACE/apps/workbench" \
743 && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
745 do_test apps/workbench_benchmark workbench_benchmark
747 test_workbench_profile() {
748 start_nginx_proxy_services \
749 && cd "$WORKSPACE/apps/workbench" \
750 && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
752 do_test apps/workbench_profile workbench_profile