Merge branch '12308-cgofuse'
[arvados.git] / build / run-tests.sh
index 17832311ad77305842cbb707b9f3c785251772c4..891faca41944469188afebb53f46291a14639e7c 100755 (executable)
@@ -35,8 +35,7 @@ Options:
 --short        Skip (or scale down) some slow tests.
 --interactive  Set up, then prompt for test/install steps to perform.
 WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with api server config files to copy into source tree.
-               (If none given, leave config files alone in source tree.)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
                Restrict apiserver tests to the given file
 sdk/python_test="--test-suite tests.test_keep_locator"
@@ -82,6 +81,7 @@ lib/controller/railsproxy
 lib/controller/router
 lib/controller/rpc
 lib/crunchstat
+lib/crunch-run
 lib/cloud
 lib/cloud/azure
 lib/cloud/cloudtest
@@ -90,12 +90,14 @@ lib/dispatchcloud/container
 lib/dispatchcloud/scheduler
 lib/dispatchcloud/ssh_executor
 lib/dispatchcloud/worker
+lib/mount
 lib/service
 services/api
 services/arv-git-httpd
 services/crunchstat
 services/dockercleaner
 services/fuse
+services/fuse:py3
 services/health
 services/keep-web
 services/keepproxy
@@ -104,7 +106,6 @@ services/keep-balance
 services/login-sync
 services/nodemanager
 services/nodemanager_integration
-services/crunch-run
 services/crunch-dispatch-local
 services/crunch-dispatch-slurm
 services/ws
@@ -196,6 +197,10 @@ sanity_checks() {
     [[ -n "${skip[sanity]}" ]] && return 0
     ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
         || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+    [[ -n "$CONFIGSRC" ]] \
+       || fatal "CONFIGSRC environment not set (see: $0 --help)"
+    [[ -s "$CONFIGSRC/config.yml" ]] \
+       || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
     echo Checking dependencies:
     echo "locale: ${LANG}"
     [[ "$(locale charmap)" = "UTF-8" ]] \
@@ -209,8 +214,8 @@ sanity_checks() {
     echo -n 'go: '
     go version \
         || fatal "No go binary. See http://golang.org/doc/install"
-    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
-        || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
+    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \
+        || fatal "Go >= 1.12 required. See http://golang.org/doc/install"
     echo -n 'gcc: '
     gcc --version | egrep ^gcc \
         || fatal "No gcc. Try: apt-get install build-essential"
@@ -226,6 +231,8 @@ sanity_checks() {
     echo -n 'Python3 pyconfig.h: '
     find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
         || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+    which netstat \
+        || fatal "No netstat. Try: apt-get install net-tools"
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
@@ -255,12 +262,6 @@ sanity_checks() {
     echo -n 'libpq libpq-fe.h: '
     find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
         || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
-    echo -n 'services/api/config/database.yml: '
-    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-           fatal "Please provide a database.yml file for the test suite"
-    else
-           echo "OK"
-    fi
     echo -n 'postgresql: '
     psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
     echo -n 'phantomjs: '
@@ -283,6 +284,10 @@ sanity_checks() {
       # needed for pkgdown, builds R SDK doc pages
       which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
     fi
+    echo 'procs with /dev/fuse open:'
+    find /proc/*/fd -lname /dev/fuse 2>/dev/null | cut -d/ -f3 | xargs --no-run-if-empty ps -lywww
+    echo 'grep fuse /proc/self/mountinfo:'
+    grep fuse /proc/self/mountinfo
 }
 
 rotate_logfile() {
@@ -388,12 +393,8 @@ checkpidfile() {
 
 checkhealth() {
     svc="$1"
-    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
-    scheme=http
-    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
-        scheme=https
-    fi
-    url="$scheme://localhost:${port}/_health/ping"
+    base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]")
+    url="$base/_health/ping"
     if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
         echo "${url} failed"
         return 1
@@ -425,28 +426,29 @@ start_services() {
     fi
     all_services_stopped=
     fail=1
+
     cd "$WORKSPACE" \
         && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
         && checkpidfile api \
         && checkdiscoverydoc $ARVADOS_API_HOST \
+        && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+        && checkpidfile nginx \
         && python sdk/python/tests/run_test_server.py start_controller \
         && checkpidfile controller \
-        && checkhealth controller \
+        && checkhealth Controller \
+        && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && checkpidfile keep-web \
-        && checkhealth keep-web \
+        && checkhealth WebDAV \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && checkpidfile arv-git-httpd \
-        && checkhealth arv-git-httpd \
+        && checkhealth GitHTTP \
         && python sdk/python/tests/run_test_server.py start_ws \
         && checkpidfile ws \
-        && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
-        && checkdiscoverydoc $ARVADOS_API_HOST \
-        && checkpidfile nginx \
         && export ARVADOS_TEST_PROXY_SERVICES=1 \
         && (env | egrep ^ARVADOS) \
         && fail=0
@@ -473,6 +475,7 @@ stop_services() {
         && python sdk/python/tests/run_test_server.py stop \
         && all_services_stopped=1
     deactivate
+    unset ARVADOS_CONFIG
 }
 
 interrupt() {
@@ -518,6 +521,10 @@ setup_ruby_environment() {
             || fatal 'rvm gemset setup'
 
         rvm env
+        (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2
+        bundle="$(which bundle)"
+        echo "$bundle"
+        "$bundle" version | grep 2.0.2 || fatal 'install bundler'
     else
         # When our "bundle install"s need to install new gems to
         # satisfy dependencies, we want them to go where "gem install
@@ -543,9 +550,14 @@ setup_ruby_environment() {
         echo "Will install dependencies to $(gem env gemdir)"
         echo "Will install arvados gems to $tmpdir_gem_home"
         echo "Gem search path is GEM_PATH=$GEM_PATH"
+        bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
+        (
+            export HOME=$GEMHOME
+            ("$bundle" version | grep -q 2.0.2) \
+                || gem install --user bundler -v 2.0.2
+            "$bundle" version | grep 2.0.2
+        ) || fatal 'install bundler'
     fi
-    bundle config || gem install bundler \
-        || fatal 'install bundler'
 }
 
 with_test_gemset() {
@@ -575,8 +587,6 @@ setup_virtualenv() {
     else
         "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
     fi
-    # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
-    "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
 }
 
 initialize() {
@@ -584,11 +594,6 @@ initialize() {
 
     echo "WORKSPACE=$WORKSPACE"
 
-    if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
-        # Jenkins expects us to use this by default.
-        CONFIGSRC="$HOME/arvados-api-server"
-    fi
-
     # Clean up .pyc files that may exist in the workspace
     cd "$WORKSPACE"
     find -name '*.pyc' -delete
@@ -616,6 +621,9 @@ initialize() {
     export R_LIBS
 
     export GOPATH
+    # Make sure our compiled binaries under test override anything
+    # else that might be in the environment.
+    export PATH=$GOPATH/bin:$PATH
 
     # Jenkins config requires that glob tmp/*.log match something. Ensure
     # that happens even if we don't end up running services that set up
@@ -625,7 +633,6 @@ initialize() {
 
     unset http_proxy https_proxy no_proxy
 
-
     # Note: this must be the last time we change PATH, otherwise rvm will
     # whine a lot.
     setup_ruby_environment
@@ -634,36 +641,15 @@ initialize() {
 }
 
 install_env() {
-    (
-        set -e
-        mkdir -p "$GOPATH/src/git.curoverse.com"
-        if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then
-            for d in \
-                "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
-                    "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
-                    "$GOPATH/src/git.curoverse.com/arvados.git"; do
-                [[ -d "$d" ]] && rmdir "$d"
-            done
-        fi
-        for d in \
-            "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
-                "$GOPATH/src/git.curoverse.com/arvados.git"; do
-            [[ -h "$d" ]] && rm "$d"
-        done
-        ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
-        go get -v github.com/kardianos/govendor
-        cd "$GOPATH/src/git.curoverse.com/arvados.git"
-        go get -v -d ...
-        "$GOPATH/bin/govendor" sync
-        which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports
-    ) || fatal "Go setup failed"
+    go mod download || fatal "Go deps failed"
+    which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed"
 
     setup_virtualenv "$VENVDIR" --python python2.7
     . "$VENVDIR/bin/activate"
 
     # Needed for run_test_server.py which is used by certain (non-Python) tests.
-    pip install --no-cache-dir PyYAML \
-        || fatal "pip install PyYAML failed"
+    pip install --no-cache-dir PyYAML future httplib2 \
+        || fatal "`pip install PyYAML future httplib2` failed"
 
     # Preinstall libcloud if using a fork; otherwise nodemanager "pip
     # install" won't pick it up by default.
@@ -689,11 +675,6 @@ Warning: python3 could not be found. Python 3 tests will be skipped.
 
 EOF
     fi
-
-    if ! which bundler >/dev/null
-    then
-        gem install --user-install bundler || fatal 'Could not install bundler'
-    fi
 }
 
 retry() {
@@ -739,12 +720,16 @@ do_test() {
     case "${1}" in
         services/api)
             stop_services
+            check_arvados_config "$1"
             ;;
-        gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+            check_arvados_config "$1"
             # don't care whether services are running
             ;;
         *)
+            check_arvados_config "$1"
             if ! start_services; then
+                checkexit 1 "$1 tests"
                 title "test $1 -- failed to start services"
                 return 1
             fi
@@ -753,6 +738,11 @@ do_test() {
     retry do_test_once ${@}
 }
 
+go_ldflags() {
+    version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}
+    echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}"
+}
+
 do_test_once() {
     unset result
 
@@ -768,12 +758,12 @@ do_test_once() {
     then
         covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
         coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
-        # We do "go get -t" here to catch compilation errors
+        # We do "go install" here to catch compilation errors
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
         # compilation errors.
-        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
-            cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
+        go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" && \
+            cd "$WORKSPACE/$1" && \
             if [[ -n "${testargs[$1]}" ]]
         then
             # "go test -check.vv giturl" doesn't work, but this
@@ -782,7 +772,7 @@ do_test_once() {
         else
             # The above form gets verbose even when testargs is
             # empty, so use this form in such cases:
-            go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+            go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
         fi
         result=${result:-$?}
         if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
@@ -799,7 +789,10 @@ do_test_once() {
             tries=$((${tries}+1))
             # $3 can name a path directory for us to use, including trailing
             # slash; e.g., the bin/ subdirectory of a virtualenv.
-            "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+            if [[ -e "${3}activate" ]]; then
+                . "${3}activate"
+            fi
+            python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
             result=$?
             if [[ ${tries} < 3 && ${result} == 137 ]]
             then
@@ -821,10 +814,29 @@ do_test_once() {
     return $result
 }
 
+check_arvados_config() {
+    if [[ "$1" = "env" ]] ; then
+       return
+    fi
+    if [[ -z "$ARVADOS_CONFIG" ]] ; then
+       # Create config file.  The run_test_server script requires PyYAML,
+       # so virtualenv needs to be active.  Downstream steps like
+       # workbench install which require a valid config.yml.
+       if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
+           install_env
+       fi
+       . "$VENVDIR/bin/activate"
+        cd "$WORKSPACE"
+       eval $(python sdk/python/tests/run_test_server.py setup_config)
+       deactivate
+    fi
+}
+
 do_install() {
     if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then
         return 0
     fi
+    check_arvados_config "$1"
     retry do_install_once ${@}
 }
 
@@ -838,7 +850,7 @@ do_install_once() {
         result=1
     elif [[ "$2" == "go" ]]
     then
-        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
+        go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1"
     elif [[ "$2" == "pip" ]]
     then
         # $3 can name a path directory for us to use, including trailing
@@ -874,11 +886,11 @@ bundle_install_trylocal() {
     (
         set -e
         echo "(Running bundle install --local. 'could not find package' messages are OK.)"
-        if ! bundle install --local --no-deployment; then
+        if ! "$bundle" install --local --no-deployment; then
             echo "(Running bundle install again, without --local.)"
-            bundle install --no-deployment
+            "$bundle" install --no-deployment
         fi
-        bundle package --all
+        "$bundle" package --all
     )
 }
 
@@ -926,24 +938,17 @@ install_services/login-sync() {
 install_services/api() {
     stop_services
     cd "$WORKSPACE/services/api" \
-        && RAILS_ENV=test bundle_install_trylocal
+        && RAILS_ENV=test bundle_install_trylocal \
+            || return 1
 
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
 
-    if [ -n "$CONFIGSRC" ]
-    then
-        for f in database.yml
-        do
-            cp "$CONFIGSRC/$f" config/ || fatal "$f"
-        done
-    fi
-
     # Clear out any lingering postgresql connections to the test
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
-        && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
+        && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
@@ -967,11 +972,15 @@ install_services/api() {
         && git --git-dir internal.git init \
             || return 1
 
-    cd "$WORKSPACE/services/api" \
-        && RAILS_ENV=test bundle exec rails db:environment:set \
-        && RAILS_ENV=test bundle exec rake db:drop \
-        && RAILS_ENV=test bundle exec rake db:setup \
-        && RAILS_ENV=test bundle exec rake db:fixtures:load
+
+    (cd "$WORKSPACE/services/api"
+     export RAILS_ENV=test
+     if "$bundle" exec rails db:environment:set ; then
+        "$bundle" exec rake db:drop
+     fi
+     "$bundle" exec rake db:setup \
+        && "$bundle" exec rake db:fixtures:load
+    )
 }
 
 declare -a pythonstuff
@@ -983,19 +992,20 @@ pythonstuff=(
     sdk/cwl:py3
     services/dockercleaner:py3
     services/fuse
+    services/fuse:py3
     services/nodemanager
     tools/crunchstat-summary
     tools/crunchstat-summary:py3
 )
 
 declare -a gostuff
-gostuff=($(git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
+gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
 
 install_apps/workbench() {
     cd "$WORKSPACE/apps/workbench" \
         && mkdir -p tmp/cache \
         && RAILS_ENV=test bundle_install_trylocal \
-        && RAILS_ENV=test RAILS_GROUPS=assets bundle exec rake npm:install
+        && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install
 }
 
 test_doc() {
@@ -1005,7 +1015,7 @@ test_doc() {
         ARVADOS_API_HOST=qr1hi.arvadosapi.com
         # Make sure python-epydoc is installed or the next line won't
         # do much good!
-        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+        PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
     )
 }
 
@@ -1015,36 +1025,15 @@ test_gofmt() {
     [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
 }
 
-test_govendor() {
-    (
-        set -e
-        cd "$GOPATH/src/git.curoverse.com/arvados.git"
-        # Remove cached source dirs in workdir. Otherwise, they will
-        # not qualify as +missing or +external below, and we won't be
-        # able to detect that they're missing from vendor/vendor.json.
-        rm -rf vendor/*/
-        go get -v -d ...
-        "$GOPATH/bin/govendor" sync
-        if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then
-            echo >&2 "vendor/vendor.json has unused or missing dependencies -- try:
-
-(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
-
-"
-            return 1
-        fi
-    )
-}
-
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]}
 }
 
 test_sdk/ruby() {
     cd "$WORKSPACE/sdk/ruby" \
-        && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
+        && "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
 }
 
 test_sdk/R() {
@@ -1057,7 +1046,7 @@ test_sdk/R() {
 test_sdk/cli() {
     cd "$WORKSPACE/sdk/cli" \
         && mkdir -p /tmp/keep \
-        && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
+        && KEEP_LOCAL_STORE=/tmp/keep "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
 }
 
 test_sdk/java-v2() {
@@ -1066,7 +1055,7 @@ test_sdk/java-v2() {
 
 test_services/login-sync() {
     cd "$WORKSPACE/services/login-sync" \
-        && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
+        && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
 }
 
 test_services/nodemanager_integration() {
@@ -1076,47 +1065,32 @@ test_services/nodemanager_integration() {
 
 test_apps/workbench_units() {
     local TASK="test:units"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
     local TASK="test:functionals"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
     local TASK="test:integration"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} 
 }
 
 test_apps/workbench_benchmark() {
     local TASK="test:benchmark"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
 }
 
 test_apps/workbench_profile() {
     local TASK="test:profile"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_profile]}
 }
 
 install_deps() {
@@ -1126,6 +1100,7 @@ install_deps() {
     do_install sdk/cli
     do_install sdk/perl
     do_install sdk/python pip
+    do_install sdk/python pip3
     do_install sdk/ruby
     do_install services/api
     do_install services/arv-git-httpd go
@@ -1156,11 +1131,11 @@ install_all() {
             fi
         fi
     done
-    do_install services/api
     for g in "${gostuff[@]}"
     do
         do_install "$g" go
     done
+    do_install services/api
     do_install apps/workbench
 }
 
@@ -1176,7 +1151,6 @@ test_all() {
     fi
 
     do_test gofmt
-    do_test govendor
     do_test doc
     do_test sdk/ruby
     do_test sdk/R
@@ -1237,6 +1211,13 @@ for p in "${pythonstuff[@]}"; do
 done
 
 testfuncargs["sdk/cli"]="sdk/cli"
+testfuncargs["sdk/R"]="sdk/R"
+testfuncargs["sdk/java-v2"]="sdk/java-v2"
+testfuncargs["apps/workbench_units"]="apps/workbench_units"
+testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals"
+testfuncargs["apps/workbench_integration"]="apps/workbench_integration"
+testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark"
+testfuncargs["apps/workbench_profile"]="apps/workbench_profile"
 
 if [[ -z ${interactive} ]]; then
     install_all