Merge branch '6263-gitolite-test' refs #6263
[arvados-dev.git] / jenkins / run-tests.sh
index 852bd43da7a1da0e11db4ac94a975a53ea2c3d4e..9ca70b86656ec56b9370731cfd23c81b0bee3466 100755 (executable)
@@ -18,14 +18,19 @@ Options:
 --skip-install Do not run any install steps. Just run tests.
                You should provide GOPATH, GEMHOME, and VENVDIR options
                from a previous invocation if you use this option.
 --skip-install Do not run any install steps. Just run tests.
                You should provide GOPATH, GEMHOME, and VENVDIR options
                from a previous invocation if you use this option.
+--only-install Run specific install step
 WORKSPACE=path Arvados source tree to test.
 CONFIGSRC=path Dir with api server config files to copy into source tree.
 WORKSPACE=path Arvados source tree to test.
 CONFIGSRC=path Dir with api server config files to copy into source tree.
-apiserver_test="TEST=test/functional/arvados/v1/collections_test.rb"
+               (If none given, leave config files alone in source tree.)
+services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
                Restrict apiserver tests to the given file
                Restrict apiserver tests to the given file
-python_sdk_test="--test-suite test.test_keep_locator"
+sdk/python_test="--test-suite test.test_keep_locator"
                Restrict Python SDK tests to the given class
                Restrict Python SDK tests to the given class
-workbench_test="TEST=test/integration/pipeline_instances_test.rb"
+apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
                Restrict Workbench tests to the given file
                Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+               Show all log messages, even when tests pass (also works
+               with services/keepstore_test etc.)
 ARVADOS_DEBUG=1
                Print more debug messages
 envvar=value   Set \$envvar to value. Primarily useful for WORKSPACE,
 ARVADOS_DEBUG=1
                Print more debug messages
 envvar=value   Set \$envvar to value. Primarily useful for WORKSPACE,
@@ -36,39 +41,61 @@ into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
 test suites depend on other components being installed, and installing
 everything tends to be quicker than debugging dependencies.
 
 test suites depend on other components being installed, and installing
 everything tends to be quicker than debugging dependencies.
 
+As a special concession to the current CI server config, CONFIGSRC
+defaults to $HOME/arvados-api-server if that directory exists.
+
 More information and background:
 
 https://arvados.org/projects/arvados/wiki/Running_tests
 More information and background:
 
 https://arvados.org/projects/arvados/wiki/Running_tests
+
+Available tests:
+
+apps/workbench
+apps/workbench_benchmark
+apps/workbench_profile
+doc
+services/api
+services/crunchstat
+services/dockercleaner
+services/fuse
+services/keepproxy
+services/keepstore
+services/login-sync
+services/nodemanager
+services/arv-git-httpd
+sdk/cli
+sdk/pam
+sdk/python
+sdk/ruby
+sdk/go/arvadosclient
+sdk/go/keepclient
+sdk/go/streamer
+
 EOF
 
 # First make sure to remove any ARVADOS_ variables from the calling
 # environment that could interfere with the tests.
 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
 
 EOF
 
 # First make sure to remove any ARVADOS_ variables from the calling
 # environment that could interfere with the tests.
 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
 
-COLUMNS=80
-
+# Reset other variables that could affect our [tests'] behavior by
+# accident.
 GITDIR=
 GOPATH=
 VENVDIR=
 GITDIR=
 GOPATH=
 VENVDIR=
+VENV3DIR=
 PYTHONPATH=
 PYTHONPATH=
-cli_test=
-workbench_test=
-apiserver_test=
-python_sdk_test=
-ruby_sdk_test=
-fuse_test=
+GEMHOME=
+PERLINSTALLBASE=
+
+COLUMNS=80
+
 leave_temp=
 skip_install=
 
 leave_temp=
 skip_install=
 
-if [[ -f /etc/profile.d/rvm.sh ]]
-then
-    source /etc/profile.d/rvm.sh
-fi
-
 declare -A leave_temp
 clear_temp() {
     leaving=""
 declare -A leave_temp
 clear_temp() {
     leaving=""
-    for var in VENVDIR GOPATH GITDIR
+    for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME PERLINSTALLBASE
     do
         if [[ -z "${leave_temp[$var]}" ]]
         then
     do
         if [[ -z "${leave_temp[$var]}" ]]
         then
@@ -80,48 +107,136 @@ clear_temp() {
             leaving+=" $var=\"${!var}\""
         fi
     done
             leaving+=" $var=\"${!var}\""
         fi
     done
-    if [[ -z "$leaving" ]]; then
+    if [[ -n "$leaving" ]]; then
         echo "Leaving behind temp dirs: $leaving"
     fi
 }
 
 fatal() {
     clear_temp
         echo "Leaving behind temp dirs: $leaving"
     fi
 }
 
 fatal() {
     clear_temp
-    echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+    echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
     exit 1
 }
 
     exit 1
 }
 
+report_outcomes() {
+    for x in "${successes[@]}"
+    do
+        echo "Pass: $x"
+    done
+
+    if [[ ${#failures[@]} == 0 ]]
+    then
+        echo "All test suites passed."
+    else
+        echo "Failures (${#failures[@]}):"
+        for x in "${failures[@]}"
+        do
+            echo "Fail: $x"
+        done
+    fi
+}
+
+exit_cleanly() {
+    trap - INT
+    create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
+    rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+    stop_services
+    rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+    report_outcomes
+    clear_temp
+    exit ${#failures}
+}
+
+sanity_checks() {
+    ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+        || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+    echo Checking dependencies:
+    echo -n 'virtualenv: '
+    virtualenv --version \
+        || fatal "No virtualenv. Try: apt-get install virtualenv"
+    echo -n 'go: '
+    go version \
+        || fatal "No go binary. See http://golang.org/doc/install"
+    echo -n 'gcc: '
+    gcc --version | egrep ^gcc \
+        || fatal "No gcc. Try: apt-get install build-essential"
+    echo -n 'fuse.h: '
+    find /usr/include -wholename '*fuse/fuse.h' \
+        || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+    echo -n 'pyconfig.h: '
+    find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+        || fatal "No pyconfig.h. Try: apt-get install python-dev"
+    echo -n 'nginx: '
+    PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+        || fatal "No nginx. Try: apt-get install nginx"
+    echo -n 'perl: '
+    perl -v | grep version \
+        || fatal "No perl. Try: apt-get install perl"
+    for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
+        echo -n "perl $mod: "
+        perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
+            || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl"
+    done
+    echo -n 'gitolite: '
+    which gitolite \
+        || fatal "No gitolite. Try: apt-get install gitolite3"
+}
+
+rotate_logfile() {
+  # i.e.  rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+  # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
+  if [[ -f "$1/$2" ]]; then
+    THEDATE=`date +%Y%m%d%H%M%S`
+    mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
+    gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
+  fi
+}
+
 declare -a failures
 declare -A skip
 declare -a failures
 declare -A skip
-
-# Always skip CLI tests. They don't know how to use run_test_server.py.
-skip[cli]=1
+declare -A testargs
+skip[apps/workbench_profile]=1
 
 while [[ -n "$1" ]]
 do
     arg="$1"; shift
     case "$arg" in
 
 while [[ -n "$1" ]]
 do
     arg="$1"; shift
     case "$arg" in
-        --skip)
-            skipwhat="$1"; shift
-            skip[$skipwhat]=1
-            ;;
         --help)
             echo >&2 "$helpmessage"
             echo >&2
             exit 1
             ;;
         --help)
             echo >&2 "$helpmessage"
             echo >&2
             exit 1
             ;;
+        --skip)
+            skipwhat="$1"; shift
+            skip[$skipwhat]=1
+            ;;
         --only)
         --only)
-            only="$1"; shift
+            only="$1"; skip[$1]=""; shift
             ;;
         --skip-install)
             skip_install=1
             ;;
             ;;
         --skip-install)
             skip_install=1
             ;;
+        --only-install)
+            skip_install=1
+            only_install="$1"; shift
+            ;;
         --leave-temp)
             leave_temp[VENVDIR]=1
         --leave-temp)
             leave_temp[VENVDIR]=1
+            leave_temp[VENV3DIR]=1
             leave_temp[GOPATH]=1
             leave_temp[GOPATH]=1
+            leave_temp[GEMHOME]=1
+            leave_temp[PERLINSTALLBASE]=1
+            ;;
+        --retry)
+            retry=1
+            ;;
+        *_test=*)
+            suite="${arg%%_test=*}"
+            args="${arg#*=}"
+            testargs["$suite"]="$args"
             ;;
         *=*)
             ;;
         *=*)
-            eval $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
+            eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
             ;;
         *)
             echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
             ;;
         *)
             echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
@@ -130,42 +245,207 @@ do
     esac
 done
 
     esac
 done
 
-# Sanity check
-echo "WORKSPACE=$WORKSPACE"
-[[ -n "$WORKSPACE" ]] || fatal "WORKSPACE not set"
+start_api() {
+    echo 'Starting API server...'
+    cd "$WORKSPACE" \
+        && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+        && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
+        && export ARVADOS_TEST_API_INSTALLED="$$" \
+        && (env | egrep ^ARVADOS)
+}
 
 
-if [[ -n "$CONFIGSRC" ]]; then
-    if [[ -d "$HOME/arvados-api-server" ]]; then
-        # Jenkins expects us to use this by default.
-        CONFIGSRC="$HOME/arvados-api-server"
+start_nginx_proxy_services() {
+    echo 'Starting keepproxy, arv-git-httpd, and nginx ssl proxy...'
+    cd "$WORKSPACE" \
+        && python sdk/python/tests/run_test_server.py start_keep_proxy \
+        && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+        && python sdk/python/tests/run_test_server.py start_nginx \
+        && export ARVADOS_TEST_PROXY_SERVICES=1
+}
+
+stop_services() {
+    if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+        unset ARVADOS_TEST_PROXY_SERVICES
+        cd "$WORKSPACE" \
+            && python sdk/python/tests/run_test_server.py stop_nginx \
+            && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+            && python sdk/python/tests/run_test_server.py stop_keep_proxy
+    fi
+    if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
+        unset ARVADOS_TEST_API_HOST
+        cd "$WORKSPACE" \
+            && python sdk/python/tests/run_test_server.py stop
     fi
     fi
+}
+
+interrupt() {
+    failures+=("($(basename $0) interrupted)")
+    exit_cleanly
+}
+trap interrupt INT
+
+sanity_checks
+
+echo "WORKSPACE=$WORKSPACE"
+
+if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
+    # Jenkins expects us to use this by default.
+    CONFIGSRC="$HOME/arvados-api-server"
 fi
 
 fi
 
+# Clean up .pyc files that may exist in the workspace
+cd "$WORKSPACE"
+find -name '*.pyc' -delete
+
 # Set up temporary install dirs (unless existing dirs were supplied)
 # Set up temporary install dirs (unless existing dirs were supplied)
-if [[ -n "$VENVDIR" ]]; then
-    leave_temp[VENVDIR]=1
-else
-    VENVDIR=$(mktemp -d)
-fi
-if [[ -n "$GOPATH" ]]; then
-    leave_temp[GOPATH]=1
-else
-    GOPATH=$(mktemp -d)
-fi
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+do
+    if [[ -n "${!tmpdir}" ]]; then
+        leave_temp[$tmpdir]=1
+        mkdir -p "${!tmpdir}"
+    else
+        eval "$tmpdir"='$(mktemp -d)'
+    fi
+done
+
+setup_ruby_environment() {
+    if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
+      source "$HOME/.rvm/scripts/rvm"
+      using_rvm=true
+    elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
+      source "/usr/local/rvm/scripts/rvm"
+      using_rvm=true
+    else
+      using_rvm=false
+    fi
+
+    if [[ "$using_rvm" == true ]]; then
+        # If rvm is in use, we can't just put separate "dependencies"
+        # and "gems-under-test" paths to GEM_PATH: passenger resets
+        # the environment to the "current gemset", which would lose
+        # our GEM_PATH and prevent our test suites from running ruby
+        # programs (for example, the Workbench test suite could not
+        # boot an API server or run arv). Instead, we have to make an
+        # rvm gemset and use it for everything.
+
+        [[ `type rvm | head -n1` == "rvm is a function" ]] \
+            || fatal 'rvm check'
+
+        # Put rvm's favorite path back in first place (overriding
+        # virtualenv, which just put itself there). Ignore rvm's
+        # complaint about not being in first place already.
+        rvm use @default 2>/dev/null
+
+        # Create (if needed) and switch to an @arvados-tests
+        # gemset. (Leave the choice of ruby to the caller.)
+        rvm use @arvados-tests --create \
+            || fatal 'rvm gemset setup'
+
+        rvm env
+    else
+        # When our "bundle install"s need to install new gems to
+        # satisfy dependencies, we want them to go where "gem install
+        # --user-install" would put them. (However, if the caller has
+        # already set GEM_HOME, we assume that's where dependencies
+        # should be installed, and we should leave it alone.)
+
+        if [ -z "$GEM_HOME" ]; then
+            user_gempath="$(gem env gempath)"
+            export GEM_HOME="${user_gempath%%:*}"
+        fi
+        PATH="$(gem env gemdir)/bin:$PATH"
+
+        # When we build and install our own gems, we install them in our
+        # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
+        # PATH so integration tests prefer them over other versions that
+        # happen to be installed in $user_gempath, system dirs, etc.
+
+        tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
+        PATH="$tmpdir_gem_home/bin:$PATH"
+        export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
+
+        echo "Will install dependencies to $(gem env gemdir)"
+        echo "Will install arvados gems to $tmpdir_gem_home"
+        echo "Gem search path is GEM_PATH=$GEM_PATH"
+    fi
+}
+
+with_test_gemset() {
+    if [[ "$using_rvm" == true ]]; then
+        "$@"
+    else
+        GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
+    fi
+}
+
+gem_uninstall_if_exists() {
+    if gem list "$1\$" | egrep '^\w'; then
+        gem uninstall --force --all --executables "$1"
+    fi
+}
+
+export PERLINSTALLBASE
+export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
-virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
-PATH="$VENVDIR/bin:$PATH"
+if ! [[ -e "$VENVDIR/bin/activate" ]] || ! [[ -e "$VENVDIR/bin/pip" ]]; then
+    virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
+fi
+. "$VENVDIR/bin/activate"
+
+if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\  -f2 | cut -d. -f1)" -lt 18 ]; then
+    pip install --upgrade setuptools pip
+fi
+
+# Note: this must be the last time we change PATH, otherwise rvm will
+# whine a lot.
+setup_ruby_environment
+
+echo "PATH is $PATH"
+
+if ! which bundler >/dev/null
+then
+    gem install --user-install bundler || fatal 'Could not install bundler'
+fi
+
+# Needed for run_test_server.py which is used by certain (non-Python) tests.
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+    || pip install PyYAML >/dev/null \
+    || fatal "pip install PyYAML failed"
+
+# Preinstall forked version of libcloud, because nodemanager "pip install"
+# won't pick it up by default.
+pip freeze 2>/dev/null | egrep ^apache-libcloud==0.18.1.dev1 \
+    || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev1.zip >/dev/null \
+    || fatal "pip install apache-libcloud failed"
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+    virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
+        || fatal "python3 virtualenv $VENV3DIR failed"
+else
+    PYTHON3=
+    skip[services/dockercleaner]=1
+    cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
 
 checkexit() {
 
 checkexit() {
-    if [[ "$?" != "0" ]]; then
-        title "!!!!!! $1 FAILED !!!!!!"
-        failures+=("$1 (`timer`)")
+    if [[ "$1" != "0" ]]; then
+        title "!!!!!! $2 FAILED !!!!!!"
+        failures+=("$2 (`timer`)")
     else
     else
-        successes+=("$1 (`timer`)")
+        successes+=("$2 (`timer`)")
     fi
 }
 
     fi
 }
 
@@ -178,35 +458,88 @@ timer() {
 }
 
 do_test() {
 }
 
 do_test() {
+    while ! do_test_once ${@} && [[ "$retry" == 1 ]]
+    do
+        read -p 'Try again? [Y/n] ' x
+        if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
+        then
+            break
+        fi
+    done
+}
+
+do_test_once() {
     if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
     then
         title "Running $1 tests"
         timer_reset
         if [[ "$2" == "go" ]]
         then
     if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
     then
         title "Running $1 tests"
         timer_reset
         if [[ "$2" == "go" ]]
         then
-            go test "git.curoverse.com/arvados.git/$1"
+            if [[ -n "${testargs[$1]}" ]]
+            then
+                # "go test -check.vv giturl" doesn't work, but this
+                # does:
+                cd "$WORKSPACE/$1" && go test ${testargs[$1]}
+            else
+                # The above form gets verbose even when testargs is
+                # empty, so use this form in such cases:
+                go test "git.curoverse.com/arvados.git/$1"
+            fi
+        elif [[ "$2" == "pip" ]]
+        then
+            # $3 can name a path directory for us to use, including trailing
+            # slash; e.g., the bin/ subdirectory of a virtualenv.
+            cd "$WORKSPACE/$1" \
+                && "${3}python" setup.py test ${testargs[$1]}
+        elif [[ "$2" != "" ]]
+        then
+            "test_$2"
         else
             "test_$1"
         fi
         else
             "test_$1"
         fi
-        checkexit "$1 tests"
+        result="$?"
+        checkexit $result "$1 tests"
         title "End of $1 tests (`timer`)"
         title "End of $1 tests (`timer`)"
+        return $result
     else
         title "Skipping $1 tests"
     fi
 }
 
 do_install() {
     else
         title "Skipping $1 tests"
     fi
 }
 
 do_install() {
-    if [[ -z "$skip_install" ]]
+    if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
     then
         title "Running $1 install"
         timer_reset
         if [[ "$2" == "go" ]]
         then
             go get -t "git.curoverse.com/arvados.git/$1"
     then
         title "Running $1 install"
         timer_reset
         if [[ "$2" == "go" ]]
         then
             go get -t "git.curoverse.com/arvados.git/$1"
+        elif [[ "$2" == "pip" ]]
+        then
+            # $3 can name a path directory for us to use, including trailing
+            # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+            # Need to change to a different directory after creating
+            # the source dist package to avoid a pip bug.
+            # see https://arvados.org/issues/5766 for details.
+
+            # Also need to install twice, because if it believes the package is
+            # already installed, pip it won't install it.  So the first "pip
+            # install" ensures that the dependencies are met, the second "pip
+            # install" ensures that we've actually installed the local package
+            # we just built.
+            cd "$WORKSPACE/$1" \
+                && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+                && cd "$WORKSPACE" \
+                && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+                && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+        elif [[ "$2" != "" ]]
+        then
+            "install_$2"
         else
             "install_$1"
         fi
         else
             "install_$1"
         fi
-        checkexit "$1 install"
+        checkexit $? "$1 install"
         title "End of $1 install (`timer`)"
     else
         title "Skipping $1 install"
         title "End of $1 install (`timer`)"
     else
         title "Skipping $1 install"
@@ -218,54 +551,80 @@ title () {
     printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
 }
 
     printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
 }
 
-install_docs() {
-    cd "$WORKSPACE/doc"
-    bundle install --no-deployment
-    rm -rf .site
-    # Make sure python-epydoc is installed or the next line won't do much good!
-    ARVADOS_API_HOST=qr1hi.arvadosapi.com
-    PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake generate baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
-    unset ARVADOS_API_HOST
+bundle_install_trylocal() {
+    (
+        set -e
+        echo "(Running bundle install --local. 'could not find package' messages are OK.)"
+        if ! bundle install --local --no-deployment; then
+            echo "(Running bundle install again, without --local.)"
+            bundle install --no-deployment
+        fi
+        bundle package --all
+    )
 }
 }
-do_install docs
 
 
-install_ruby_sdk() {
-    cd "$WORKSPACE/sdk/ruby" \
-        && gem build arvados.gemspec \
-        && gem install --user-install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
+install_doc() {
+    cd "$WORKSPACE/doc" \
+        && bundle_install_trylocal \
+        && rm -rf .site
+}
+do_install doc
+
+install_gem() {
+    gemname=$1
+    srcpath=$2
+    with_test_gemset gem_uninstall_if_exists "$gemname" \
+        && cd "$WORKSPACE/$srcpath" \
+        && bundle_install_trylocal \
+        && gem build "$gemname.gemspec" \
+        && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
 }
 }
-do_install ruby_sdk
 
 
-install_cli() {
-    cd "$WORKSPACE/sdk/cli" \
-        && gem build arvados-cli.gemspec \
-        && gem install --user-install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
+install_ruby_sdk() {
+    install_gem arvados sdk/ruby
 }
 }
-do_install cli
+do_install sdk/ruby ruby_sdk
 
 
-install_python_sdk() {
-    # Install the Python SDK early. Various other test suites (like
-    # keepproxy) bring up run_test_server.py, which imports the arvados
-    # module. We can't actually *test* the Python SDK yet though, because
-    # its own test suite brings up some of those other programs (like
-    # keepproxy).
+install_perl_sdk() {
+    cd "$WORKSPACE/sdk/perl" \
+        && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
+        && make install INSTALLDIRS=perl
+}
+do_install sdk/perl perl_sdk
 
 
-    cd "$WORKSPACE/sdk/python" \
-        && python setup.py egg_info -b ".$(git log --format=format:%ct.%h -n1 .)" sdist rotate --keep=1 --match .tar.gz \
-        && pip install dist/arvados-python-client-0.1.*.tar.gz
+install_cli() {
+    install_gem arvados-cli sdk/cli
 }
 }
-do_install python_sdk
+do_install sdk/cli cli
 
 
-install_fuse() {
-    cd "$WORKSPACE/services/fuse" \
-        && python setup.py egg_info -b ".$(git log --format=format:%ct.%h -n1 .)" sdist rotate --keep=1 --match .tar.gz \
-        && pip install dist/arvados_fuse-0.1.*.tar.gz
+install_login-sync() {
+    install_gem arvados-login-sync services/login-sync
 }
 }
-do_install fuse
+do_install services/login-sync login-sync
+
+# Install the Python SDK early. Various other test suites (like
+# keepproxy) bring up run_test_server.py, which imports the arvados
+# module. We can't actually *test* the Python SDK yet though, because
+# its own test suite brings up some of those other programs (like
+# keepproxy).
+declare -a pythonstuff
+pythonstuff=(
+    sdk/pam
+    sdk/python
+    services/fuse
+    services/nodemanager
+    )
+for p in "${pythonstuff[@]}"
+do
+    do_install "$p" pip
+done
+if [ -n "$PYTHON3" ]; then
+    do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
 
 install_apiserver() {
 
 install_apiserver() {
-    cd "$WORKSPACE/services/api"
-    bundle install --no-deployment
+    cd "$WORKSPACE/services/api" \
+        && RAILS_ENV=test bundle_install_trylocal
 
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
 
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
@@ -285,8 +644,6 @@ install_apiserver() {
     sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
     sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
 
     sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
     sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
 
-    export RAILS_ENV=test
-
     # Set up empty git repo (for git tests)
     GITDIR=$(mktemp -d)
     sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
     # Set up empty git repo (for git tests)
     GITDIR=$(mktemp -d)
     sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
@@ -301,21 +658,28 @@ install_apiserver() {
         && git add tmp \
         && git commit -m 'initial commit'
 
         && git add tmp \
         && git commit -m 'initial commit'
 
-    # Clear out any lingering postgresql connections to arvados_test, so that we can drop it
-    # This assumes the current user is a postgresql superuser
-    psql arvados_test -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'arvados_test';" 2>/dev/null
+    # Clear out any lingering postgresql connections to the test
+    # database, so that we can drop it. This assumes the current user
+    # is a postgresql superuser.
+    cd "$WORKSPACE/services/api" \
+        && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+        && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     cd "$WORKSPACE/services/api" \
 
     cd "$WORKSPACE/services/api" \
-        && bundle exec rake db:drop \
-        && bundle exec rake db:create \
-        && bundle exec rake db:setup
+        && RAILS_ENV=test bundle exec rake db:drop \
+        && RAILS_ENV=test bundle exec rake db:setup \
+        && RAILS_ENV=test bundle exec rake db:fixtures:load
 }
 }
-do_install apiserver
+do_install services/api apiserver
 
 declare -a gostuff
 gostuff=(
 
 declare -a gostuff
 gostuff=(
+    services/arv-git-httpd
+    services/crunchstat
     services/keepstore
     services/keepproxy
     services/keepstore
     services/keepproxy
+    services/datamanager/summary
+    services/datamanager/collection
     sdk/go/arvadosclient
     sdk/go/keepclient
     sdk/go/streamer
     sdk/go/arvadosclient
     sdk/go/keepclient
     sdk/go/streamer
@@ -325,58 +689,66 @@ do
     do_install "$g" go
 done
 
     do_install "$g" go
 done
 
+install_workbench() {
+    cd "$WORKSPACE/apps/workbench" \
+        && mkdir -p tmp/cache \
+        && RAILS_ENV=test bundle_install_trylocal
+}
+do_install apps/workbench workbench
+
 test_doclinkchecker() {
 test_doclinkchecker() {
-    cd "$WORKSPACE/doc"
-    bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/
+    (
+        set -e
+        cd "$WORKSPACE/doc"
+        ARVADOS_API_HOST=qr1hi.arvadosapi.com
+        # Make sure python-epydoc is installed or the next line won't
+        # do much good!
+        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+    )
 }
 }
-do_test doclinkchecker
+do_test doc doclinkchecker
+
+stop_services
+
+test_apiserver() {
+    cd "$WORKSPACE/services/api" \
+        && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
+}
+do_test services/api apiserver
+
+# Shortcut for when we're only running apiserver tests. This saves a bit of time,
+# because we don't need to start up the api server for subsequent tests.
+if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
+  rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+  exit_cleanly
+fi
+
+start_api
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
-        && bundle install --no-deployment \
-        && bundle exec rake test
+        && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
 }
 }
-do_test ruby_sdk
+do_test sdk/ruby ruby_sdk
 
 test_cli() {
 
 test_cli() {
-    title "Starting SDK CLI tests"
     cd "$WORKSPACE/sdk/cli" \
     cd "$WORKSPACE/sdk/cli" \
-        && bundle install --no-deployment \
         && mkdir -p /tmp/keep \
         && mkdir -p /tmp/keep \
-        && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test $cli_test
-}
-do_test cli
-
-test_apiserver() {
-    cd "$WORKSPACE/services/api"
-    bundle exec rake test $apiserver_test
+        && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
 }
 }
-do_test apiserver
+do_test sdk/cli cli
 
 
-test_python_sdk() {
-    # Python SDK. We test this before testing keepproxy: keepproxy runs
-    # run_test_server.py, which depends on the yaml package, which is in
-    # tests_require but not install_requires, and therefore does not get
-    # installed by setuptools until we run "setup.py test" *and* install
-    # the .egg files that setup.py downloads.
-
-    cd "$WORKSPACE/sdk/python" \
-        && python setup.py test $python_sdk_test
-    r=$?
-    easy_install *.egg
-    return $r
+test_login-sync() {
+    cd "$WORKSPACE/services/login-sync" \
+        && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
 }
 }
-do_test python_sdk
+do_test services/login-sync login-sync
 
 
-test_fuse() {
-    # Install test dependencies here too, in case run_test_server needs them.
-    cd "$WORKSPACE/services/fuse" \
-        && python setup.py test $fuse_test
-    r=$?
-    easy_install *.egg
-    return $r
-}
-do_test fuse
+for p in "${pythonstuff[@]}"
+do
+    do_test "$p" pip
+done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
 
 for g in "${gostuff[@]}"
 do
 
 for g in "${gostuff[@]}"
 do
@@ -384,27 +756,24 @@ do
 done
 
 test_workbench() {
 done
 
 test_workbench() {
-    cd "$WORKSPACE/apps/workbench" \
-        && bundle install --no-deployment \
-        && bundle exec rake test $workbench_test
+    start_nginx_proxy_services \
+        && cd "$WORKSPACE/apps/workbench" \
+        && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
 }
 }
-do_test workbench
+do_test apps/workbench workbench
 
 
-clear_temp
+test_workbench_benchmark() {
+    start_nginx_proxy_services \
+        && cd "$WORKSPACE/apps/workbench" \
+        && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
+}
+do_test apps/workbench_benchmark workbench_benchmark
 
 
-for x in "${successes[@]}"
-do
-    echo "Pass: $x"
-done
+test_workbench_profile() {
+    start_nginx_proxy_services \
+        && cd "$WORKSPACE/apps/workbench" \
+        && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
+}
+do_test apps/workbench_profile workbench_profile
 
 
-if [[ ${#failures[@]} == 0 ]]
-then
-    echo "All test suites passed."
-else
-    echo "Failures (${#failures[@]}):"
-    for x in "${failures[@]}"
-    do
-        echo "Fail: $x"
-    done
-fi
-exit ${#failures}
+exit_cleanly