3 read -rd "\000" helpmessage <<EOF
4 $(basename $0): Install and test Arvados components.
6 Exit non-zero if any tests fail.
9 $(basename $0) WORKSPACE=/path/to/arvados [options]
13 --skip FOO Do not test the FOO component.
14 --only FOO Do not test anything except the FOO component.
15 --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
16 Instead, show which directories were used this time so they
17 can be reused in subsequent invocations.
18 --skip-install Do not run any install steps. Just run tests.
19 You should provide GOPATH, GEMHOME, and VENVDIR options
20 from a previous invocation if you use this option.
21 --only-install Run specific install step
22 WORKSPACE=path Arvados source tree to test.
23 CONFIGSRC=path Dir with api server config files to copy into source tree.
24 (If none given, leave config files alone in source tree.)
25 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
26 Restrict apiserver tests to the given file
27 sdk/python_test="--test-suite test.test_keep_locator"
28 Restrict Python SDK tests to the given class
29 apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
30 Restrict Workbench tests to the given file
31 services/arv-git-httpd_test="-check.vv"
32 Show all log messages, even when tests pass (also works
33 with services/keepstore_test etc.)
35 Print more debug messages
36 envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
37 *_test, and other examples shown above.
39 Assuming --skip-install is not given, all components are installed
40 into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
41 test suites depend on other components being installed, and installing
42 everything tends to be quicker than debugging dependencies.
44 As a special concession to the current CI server config, CONFIGSRC
45 defaults to $HOME/arvados-api-server if that directory exists.
47 More information and background:
49 https://arvados.org/projects/arvados/wiki/Running_tests
54 apps/workbench_benchmark
55 apps/workbench_profile
59 services/dockercleaner
64 services/arv-git-httpd
74 # First make sure to remove any ARVADOS_ variables from the calling
75 # environment that could interfere with the tests.
76 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
78 # Reset other variables that could affect our [tests'] behavior by
95 for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
97 if [[ -z "${leave_temp[$var]}" ]]
104 leaving+=" $var=\"${!var}\""
107 if [[ -n "$leaving" ]]; then
108 echo "Leaving behind temp dirs: $leaving"
114 echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
119 for x in "${successes[@]}"
124 if [[ ${#failures[@]} == 0 ]]
126 echo "All test suites passed."
128 echo "Failures (${#failures[@]}):"
129 for x in "${failures[@]}"
138 create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
139 rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
141 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
148 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
149 || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
150 echo Checking dependencies:
151 echo -n 'virtualenv: '
152 virtualenv --version \
153 || fatal "No virtualenv. Try: apt-get install virtualenv"
156 || fatal "No go binary. See http://golang.org/doc/install"
158 gcc --version | egrep ^gcc \
159 || fatal "No gcc. Try: apt-get install build-essential"
161 find /usr/include -wholename '*fuse/fuse.h' \
162 || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
163 echo -n 'pyconfig.h: '
164 find /usr/include -name pyconfig.h | egrep --max-count=1 . \
165 || fatal "No pyconfig.h. Try: apt-get install python-dev"
167 PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
168 || fatal "No nginx. Try: apt-get install nginx"
172 # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
173 # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
174 if [[ -f "$1/$2" ]]; then
175 THEDATE=`date +%Y%m%d%H%M%S`
176 mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
177 gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
184 skip[apps/workbench_profile]=1
191 echo >&2 "$helpmessage"
200 only="$1"; skip[$1]=""; shift
207 only_install="$1"; shift
210 leave_temp[VENVDIR]=1
211 leave_temp[VENV3DIR]=1
213 leave_temp[GEMHOME]=1
219 suite="${arg%%_test=*}"
221 testargs["$suite"]="$args"
224 eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
227 echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
234 echo 'Starting API server...'
236 && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
237 && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
238 && export ARVADOS_TEST_API_INSTALLED="$$" \
239 && (env | egrep ^ARVADOS)
242 start_nginx_proxy_services() {
243 echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
245 && python sdk/python/tests/run_test_server.py start_keep_proxy \
246 && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
247 && python sdk/python/tests/run_test_server.py start_nginx \
248 && export ARVADOS_TEST_PROXY_SERVICES=1
252 if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
253 unset ARVADOS_TEST_PROXY_SERVICES
255 && python sdk/python/tests/run_test_server.py stop_nginx \
256 && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
257 && python sdk/python/tests/run_test_server.py stop_keep_proxy
259 if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
260 unset ARVADOS_TEST_API_HOST
262 && python sdk/python/tests/run_test_server.py stop
267 failures+=("($(basename $0) interrupted)")
274 echo "WORKSPACE=$WORKSPACE"
276 if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
277 # Jenkins expects us to use this by default.
278 CONFIGSRC="$HOME/arvados-api-server"
281 # Clean up .pyc files that may exist in the workspace
283 find -name '*.pyc' -delete
285 # Set up temporary install dirs (unless existing dirs were supplied)
286 for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
288 if [[ -n "${!tmpdir}" ]]; then
289 leave_temp[$tmpdir]=1
290 mkdir -p "${!tmpdir}"
292 eval "$tmpdir"='$(mktemp -d)'
296 setup_ruby_environment() {
297 if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
298 source "$HOME/.rvm/scripts/rvm"
300 elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
301 source "/usr/local/rvm/scripts/rvm"
307 if [[ "$using_rvm" == true ]]; then
308 # If rvm is in use, we can't just put separate "dependencies"
309 # and "gems-under-test" paths to GEM_PATH: passenger resets
310 # the environment to the "current gemset", which would lose
311 # our GEM_PATH and prevent our test suites from running ruby
312 # programs (for example, the Workbench test suite could not
313 # boot an API server or run arv). Instead, we have to make an
314 # rvm gemset and use it for everything.
316 [[ `type rvm | head -n1` == "rvm is a function" ]] \
319 # Put rvm's favorite path back in first place (overriding
320 # virtualenv, which just put itself there). Ignore rvm's
321 # complaint about not being in first place already.
322 rvm use @default 2>/dev/null
324 # Create (if needed) and switch to an @arvados-tests
325 # gemset. (Leave the choice of ruby to the caller.)
326 rvm use @arvados-tests --create \
327 || fatal 'rvm gemset setup'
331 # When our "bundle install"s need to install new gems to
332 # satisfy dependencies, we want them to go where "gem install
333 # --user-install" would put them. (However, if the caller has
334 # already set GEM_HOME, we assume that's where dependencies
335 # should be installed, and we should leave it alone.)
337 if [ -z "$GEM_HOME" ]; then
338 user_gempath="$(gem env gempath)"
339 export GEM_HOME="${user_gempath%%:*}"
341 PATH="$(gem env gemdir)/bin:$PATH"
343 # When we build and install our own gems, we install them in our
344 # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
345 # PATH so integration tests prefer them over other versions that
346 # happen to be installed in $user_gempath, system dirs, etc.
348 tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
349 PATH="$tmpdir_gem_home/bin:$PATH"
350 export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
352 echo "Will install dependencies to $(gem env gemdir)"
353 echo "Will install arvados gems to $tmpdir_gem_home"
354 echo "Gem search path is GEM_PATH=$GEM_PATH"
359 if [[ "$using_rvm" == true ]]; then
362 GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
366 gem_uninstall_if_exists() {
367 if gem list "$1\$" | egrep '^\w'; then
368 gem uninstall --force --all --executables "$1"
373 mkdir -p "$GOPATH/src/git.curoverse.com"
374 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
375 || fatal "symlink failed"
377 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
378 . "$VENVDIR/bin/activate"
380 # Note: this must be the last time we change PATH, otherwise rvm will
382 setup_ruby_environment
386 if ! which bundler >/dev/null
388 gem install --user-install bundler || fatal 'Could not install bundler'
391 # Needed for run_test_server.py which is used by certain (non-Python) tests.
392 pip freeze 2>/dev/null | egrep ^PyYAML= \
393 || pip install PyYAML >/dev/null \
394 || fatal "pip install PyYAML failed"
396 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
397 # Otherwise, skip dependent tests.
398 PYTHON3=$(which python3)
399 if [ "0" = "$?" ]; then
400 virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
401 || fatal "python3 virtualenv $VENV3DIR failed"
404 skip[services/dockercleaner]=1
407 Warning: python3 could not be found
408 services/dockercleaner install and tests will be skipped
414 if [[ "$1" != "0" ]]; then
415 title "!!!!!! $2 FAILED !!!!!!"
416 failures+=("$2 (`timer`)")
418 successes+=("$2 (`timer`)")
427 echo -n "$(($SECONDS - $t0))s"
431 while ! do_test_once ${@} && [[ "$retry" == 1 ]]
433 read -p 'Try again? [Y/n] ' x
434 if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
442 if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
444 title "Running $1 tests"
446 if [[ "$2" == "go" ]]
448 if [[ -n "${testargs[$1]}" ]]
450 # "go test -check.vv giturl" doesn't work, but this
452 cd "$WORKSPACE/$1" && go test ${testargs[$1]}
454 # The above form gets verbose even when testargs is
455 # empty, so use this form in such cases:
456 go test "git.curoverse.com/arvados.git/$1"
458 elif [[ "$2" == "pip" ]]
460 # $3 can name a path directory for us to use, including trailing
461 # slash; e.g., the bin/ subdirectory of a virtualenv.
463 && "${3}python" setup.py test ${testargs[$1]}
464 elif [[ "$2" != "" ]]
471 checkexit $result "$1 tests"
472 title "End of $1 tests (`timer`)"
475 title "Skipping $1 tests"
480 if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
482 title "Running $1 install"
484 if [[ "$2" == "go" ]]
486 go get -t "git.curoverse.com/arvados.git/$1"
487 elif [[ "$2" == "pip" ]]
489 # $3 can name a path directory for us to use, including trailing
490 # slash; e.g., the bin/ subdirectory of a virtualenv.
492 # Need to change to a different directory after creating
493 # the source dist package to avoid a pip bug.
494 # see https://arvados.org/issues/5766 for details.
496 # Also need to install twice, because if it belives the package is
497 # already installed, pip it won't install it. So the first "pip
498 # install" ensures that the dependencies are met, the second "pip
499 # install" ensures that we've actually install the local package
502 && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
504 && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
505 && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
506 elif [[ "$2" != "" ]]
512 checkexit $? "$1 install"
513 title "End of $1 install (`timer`)"
515 title "Skipping $1 install"
520 txt="********** $1 **********"
521 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
524 bundle_install_trylocal() {
527 echo "(Running bundle install --local. 'could not find package' messages are OK.)"
528 if ! bundle install --local --no-deployment; then
529 echo "(Running bundle install again, without --local.)"
530 bundle install --no-deployment
537 cd "$WORKSPACE/doc" \
538 && bundle_install_trylocal \
544 with_test_gemset gem_uninstall_if_exists arvados \
545 && cd "$WORKSPACE/sdk/ruby" \
546 && bundle_install_trylocal \
547 && gem build arvados.gemspec \
548 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
550 do_install sdk/ruby ruby_sdk
553 with_test_gemset gem_uninstall_if_exists arvados-cli \
554 && cd "$WORKSPACE/sdk/cli" \
555 && bundle_install_trylocal \
556 && gem build arvados-cli.gemspec \
557 && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
559 do_install sdk/cli cli
561 # Install the Python SDK early. Various other test suites (like
562 # keepproxy) bring up run_test_server.py, which imports the arvados
563 # module. We can't actually *test* the Python SDK yet though, because
564 # its own test suite brings up some of those other programs (like
566 declare -a pythonstuff
572 for p in "${pythonstuff[@]}"
576 if [ -n "$PYTHON3" ]; then
577 do_install services/dockercleaner pip "$VENV3DIR/bin/"
580 install_apiserver() {
581 cd "$WORKSPACE/services/api" \
582 && RAILS_ENV=test bundle_install_trylocal
584 rm -f config/environments/test.rb
585 cp config/environments/test.rb.example config/environments/test.rb
587 if [ -n "$CONFIGSRC" ]
589 for f in database.yml application.yml
591 cp "$CONFIGSRC/$f" config/ || fatal "$f"
595 # Fill in a random secret_token and blob_signing_key for testing
596 SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
597 BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
599 sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
600 sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
602 # Set up empty git repo (for git tests)
604 sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
607 mkdir -p $GITDIR/test
610 && git config user.email "jenkins@ci.curoverse.com" \
611 && git config user.name "Jenkins, CI" \
614 && git commit -m 'initial commit'
616 # Clear out any lingering postgresql connections to the test
617 # database, so that we can drop it. This assumes the current user
618 # is a postgresql superuser.
619 cd "$WORKSPACE/services/api" \
620 && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
621 && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
623 cd "$WORKSPACE/services/api" \
624 && RAILS_ENV=test bundle exec rake db:drop \
625 && RAILS_ENV=test bundle exec rake db:setup \
626 && RAILS_ENV=test bundle exec rake db:fixtures:load
628 do_install services/api apiserver
632 services/arv-git-httpd
640 for g in "${gostuff[@]}"
645 install_workbench() {
646 cd "$WORKSPACE/apps/workbench" \
647 && mkdir -p tmp/cache \
648 && RAILS_ENV=test bundle_install_trylocal
650 do_install apps/workbench workbench
652 test_doclinkchecker() {
656 ARVADOS_API_HOST=qr1hi.arvadosapi.com
657 # Make sure python-epydoc is installed or the next line won't
659 PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
662 do_test doc doclinkchecker
667 cd "$WORKSPACE/services/api" \
668 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
670 do_test services/api apiserver
672 # Shortcut for when we're only running apiserver tests. This saves a bit of time,
673 # because we don't need to start up the api server for subsequent tests.
674 if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
675 rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
682 cd "$WORKSPACE/sdk/ruby" \
683 && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
685 do_test sdk/ruby ruby_sdk
688 cd "$WORKSPACE/sdk/cli" \
689 && mkdir -p /tmp/keep \
690 && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
694 for p in "${pythonstuff[@]}"
698 do_test services/dockercleaner pip "$VENV3DIR/bin/"
700 for g in "${gostuff[@]}"
706 start_nginx_proxy_services \
707 && cd "$WORKSPACE/apps/workbench" \
708 && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
710 do_test apps/workbench workbench
712 test_workbench_benchmark() {
713 start_nginx_proxy_services \
714 && cd "$WORKSPACE/apps/workbench" \
715 && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
717 do_test apps/workbench_benchmark workbench_benchmark
719 test_workbench_profile() {
720 start_nginx_proxy_services \
721 && cd "$WORKSPACE/apps/workbench" \
722 && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
724 do_test apps/workbench_profile workbench_profile