X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e6f350e56674e3f19d24755b94a22e59cf8478b6..3a126e4c7e3ab4235a8d5f84f5addebb4691ac7a:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 2a1bf9068b..edf9f3ab23 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -77,6 +77,10 @@ doc
 lib/cli
 lib/cmd
 lib/controller
+lib/controller/federation
+lib/controller/railsproxy
+lib/controller/router
+lib/controller/rpc
 lib/crunchstat
 lib/cloud
 lib/cloud/azure
@@ -85,6 +89,7 @@ lib/dispatchcloud/container
 lib/dispatchcloud/scheduler
 lib/dispatchcloud/ssh_executor
 lib/dispatchcloud/worker
+lib/service
 services/api
 services/arv-git-httpd
 services/crunchstat
@@ -104,6 +109,7 @@ services/crunch-dispatch-slurm
 services/ws
 sdk/cli
 sdk/pam
+sdk/pam:py3
 sdk/python
 sdk/python:py3
 sdk/ruby
@@ -121,8 +127,10 @@ sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
 sdk/R
+sdk/java-v2
 tools/sync-groups
 tools/crunchstat-summary
+tools/crunchstat-summary:py3
 tools/keep-exercise
 tools/keep-rsync
 tools/keep-block-check
@@ -136,6 +144,8 @@ EOF
 # environment that could interfere with the tests.
 unset $(env | cut -d= -f1 | grep \^ARVADOS_)
 
+export ARVADOS_EXPERIMENTAL=14287
+
 # Reset other variables that could affect our [tests'] behavior by
 # accident.
 GITDIR=
@@ -146,6 +156,7 @@ PYTHONPATH=
 GEMHOME=
 PERLINSTALLBASE=
 R_LIBS=
+export LANG=en_US.UTF-8
 
 short=
 only_install=
@@ -187,6 +198,9 @@ sanity_checks() {
     ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
         || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
     echo Checking dependencies:
+    echo "locale: ${LANG}"
+    [[ "$(locale charmap)" = "UTF-8" ]] \
+        || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen"
     echo -n 'virtualenv: '
     virtualenv --version \
         || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
@@ -362,12 +376,47 @@ if [[ $NEED_SDK_R == false ]]; then
 	echo "R SDK not needed, it will not be installed."
 fi
 
+checkpidfile() {
+    svc="$1"
+    pid="$(cat "$WORKSPACE/tmp/${svc}.pid")"
+    if [[ -z "$pid" ]] || ! kill -0 "$pid"; then
+        tail $WORKSPACE/tmp/${1}*.log
+        echo "${svc} pid ${pid} not running"
+        return 1
+    fi
+    echo "${svc} pid ${pid} ok"
+}
+
+checkhealth() {
+    svc="$1"
+    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
+    scheme=http
+    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
+        scheme=https
+    fi
+    url="$scheme://localhost:${port}/_health/ping"
+    if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
+        echo "${url} failed"
+        return 1
+    fi
+}
+
+checkdiscoverydoc() {
+    dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
+    if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
+        echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd"
+        tail -v $WORKSPACE/services/api/log/test.log
+        return 1
+    fi
+    echo "${dd} ok"
+}
+
 start_services() {
     if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
         return 0
     fi
     . "$VENVDIR/bin/activate"
-    echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
+    echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
 	mkdir -p "$WORKSPACE/services/api/log"
     fi
@@ -376,19 +425,32 @@ start_services() {
 	rm -f "$WORKSPACE/tmp/api.pid"
     fi
     all_services_stopped=
-    fail=0
+    fail=1
     cd "$WORKSPACE" \
-        && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo "fail=1; false") \
+        && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
+        && checkpidfile api \
+        && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_controller \
+        && checkpidfile controller \
+        && checkhealth controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
+        && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
+        && checkpidfile keep-web \
+        && checkhealth keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+        && checkpidfile arv-git-httpd \
+        && checkhealth arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
-        && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo "fail=1; false") \
+        && checkpidfile ws \
+        && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+        && checkdiscoverydoc $ARVADOS_API_HOST \
+        && checkpidfile nginx \
+        && export ARVADOS_TEST_PROXY_SERVICES=1 \
         && (env | egrep ^ARVADOS) \
-        || fail=1
+        && fail=0
     deactivate
     if [[ $fail != 0 ]]; then
         unset ARVADOS_TEST_API_HOST
@@ -400,7 +462,7 @@ stop_services() {
     if [[ -n "$all_services_stopped" ]]; then
         return
     fi
-    unset ARVADOS_TEST_API_HOST
+    unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES
     . "$VENVDIR/bin/activate" || return
     cd "$WORKSPACE" \
         && python sdk/python/tests/run_test_server.py stop_nginx \
@@ -693,7 +755,7 @@ do_test() {
         services/api)
             stop_services
             ;;
-        doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
             # don't care whether services are running
             ;;
         *)
@@ -712,6 +774,7 @@ do_test_once() {
     title "test $1"
     timer_reset
 
+    result=
     if which deactivate >/dev/null; then deactivate; fi
     if ! . "$VENVDIR/bin/activate"
     then
@@ -726,7 +789,6 @@ do_test_once() {
         # compilation errors.
         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
-            [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
             if [[ -n "${testargs[$1]}" ]]
         then
             # "go test -check.vv giturl" doesn't work, but this
@@ -743,6 +805,7 @@ do_test_once() {
             go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
             rm "$WORKSPACE/tmp/.$covername.tmp"
         fi
+        [[ $result = 0 ]] && gofmt -e -d *.go
     elif [[ "$2" == "pip" ]]
     then
         tries=0
@@ -784,6 +847,7 @@ do_install_once() {
     title "install $1"
     timer_reset
 
+    result=
     if which deactivate >/dev/null; then deactivate; fi
     if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then
         result=1
@@ -846,7 +910,7 @@ install_gem() {
         && cd "$WORKSPACE/$srcpath" \
         && bundle_install_trylocal \
         && gem build "$gemname.gemspec" \
-        && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+        && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1)
 }
 
 install_sdk/ruby() {
@@ -893,7 +957,7 @@ install_services/api() {
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
-        && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+        && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
@@ -918,6 +982,7 @@ install_services/api() {
             || return 1
 
     cd "$WORKSPACE/services/api" \
+        && RAILS_ENV=test bundle exec rails db:environment:set \
         && RAILS_ENV=test bundle exec rake db:drop \
         && RAILS_ENV=test bundle exec rake db:setup \
         && RAILS_ENV=test bundle exec rake db:fixtures:load
@@ -934,52 +999,11 @@ pythonstuff=(
     services/fuse
     services/nodemanager
     tools/crunchstat-summary
+    tools/crunchstat-summary:py3
 )
 
 declare -a gostuff
-gostuff=(
-    cmd/arvados-client
-    cmd/arvados-server
-    lib/cli
-    lib/cmd
-    lib/controller
-    lib/crunchstat
-    lib/cloud
-    lib/cloud/azure
-    lib/cloud/ec2
-    lib/dispatchcloud
-    lib/dispatchcloud/container
-    lib/dispatchcloud/scheduler
-    lib/dispatchcloud/ssh_executor
-    lib/dispatchcloud/worker
-    sdk/go/arvados
-    sdk/go/arvadosclient
-    sdk/go/auth
-    sdk/go/blockdigest
-    sdk/go/dispatch
-    sdk/go/health
-    sdk/go/httpserver
-    sdk/go/manifest
-    sdk/go/asyncbuf
-    sdk/go/crunchrunner
-    sdk/go/stats
-    services/arv-git-httpd
-    services/crunchstat
-    services/health
-    services/keep-web
-    services/keepstore
-    sdk/go/keepclient
-    services/keep-balance
-    services/keepproxy
-    services/crunch-dispatch-local
-    services/crunch-dispatch-slurm
-    services/crunch-run
-    services/ws
-    tools/keep-block-check
-    tools/keep-exercise
-    tools/keep-rsync
-    tools/sync-groups
-)
+gostuff=($(git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
 
 install_apps/workbench() {
     cd "$WORKSPACE/apps/workbench" \
@@ -999,10 +1023,16 @@ test_doc() {
     )
 }
 
+test_gofmt() {
+    cd "$WORKSPACE" || return 1
+    dirs=$(ls -d */ | egrep -v 'vendor|tmp')
+    [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+}
+
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]}
 }
 
 test_sdk/ruby() {
@@ -1023,6 +1053,10 @@ test_sdk/cli() {
         && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
 }
 
+test_sdk/java-v2() {
+    cd "$WORKSPACE/sdk/java-v2" && gradle test
+}
+
 test_services/login-sync() {
     cd "$WORKSPACE/services/login-sync" \
         && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
@@ -1035,17 +1069,17 @@ test_services/nodemanager_integration() {
 
 test_apps/workbench_units() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
 }
 
 test_apps/workbench_benchmark() {
@@ -1061,7 +1095,7 @@ test_apps/workbench_profile() {
 install_deps() {
     # Install parts needed by test suites
     do_install env
-    do_install cmd/arvados-server
+    do_install cmd/arvados-server go
     do_install sdk/cli
     do_install sdk/perl
     do_install sdk/python pip
@@ -1114,11 +1148,13 @@ test_all() {
         exit_cleanly
     fi
 
+    do_test gofmt
     do_test doc
     do_test sdk/ruby
     do_test sdk/R
     do_test sdk/cli
     do_test services/login-sync
+    do_test sdk/java-v2
     do_test services/nodemanager_integration
     for p in "${pythonstuff[@]}"
     do
@@ -1168,57 +1204,69 @@ for g in "${gostuff[@]}"; do
 done
 for p in "${pythonstuff[@]}"; do
     dir=${p%:py3}
-    if [[ ${dir} = ${p} ]]; then
-        testfuncargs[$p]="$dir pip $VENVDIR/bin/"
-    else
-        testfuncargs[$p]="$dir pip $VENV3DIR/bin/"
-    fi
+    testfuncargs[$dir]="$dir pip $VENVDIR/bin/"
+    testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/"
 done
 
+testfuncargs["sdk/cli"]="sdk/cli"
+
 if [[ -z ${interactive} ]]; then
     install_all
     test_all
 else
+    skip=()
+    only=()
+    only_install=()
     if [[ -e "$VENVDIR/bin/activate" ]]; then stop_services; fi
     setnextcmd() {
-        if [[ "$nextcmd" != "install deps" ]]; then
-            :
-        elif [[ -e "$VENVDIR/bin/activate" ]]; then
-            nextcmd="test lib/cmd"
-        else
+        if [[ "$TERM" = dumb ]]; then
+            # assume emacs, or something, is offering a history buffer
+            # and pre-populating the command will only cause trouble
+            nextcmd=
+        elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
             nextcmd="install deps"
+        else
+            nextcmd=""
         fi
     }
     echo
     help_interactive
     nextcmd="install deps"
     setnextcmd
-    while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+    HISTFILE="$WORKSPACE/tmp/.history"
+    history -r
+    while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+        history -s "$nextcmd"
+        history -w
         read verb target opts <<<"${nextcmd}"
+        target="${target%/}"
+        target="${target/\/:/:}"
         case "${verb}" in
-            "" | "help")
-                help_interactive
-                ;;
             "exit" | "quit")
                 exit_cleanly
                 ;;
             "reset")
                 stop_services
                 ;;
-            *)
-                target="${target%/}"
-                testargs["$target"]="${opts}"
+            "test" | "install")
                 case "$target" in
+                    "")
+                        help_interactive
+                        ;;
                     all | deps)
                         ${verb}_${target}
                         ;;
                     *)
+                        testargs["$target"]="${opts}"
                         tt="${testfuncargs[${target}]}"
                         tt="${tt:-$target}"
                         do_$verb $tt
                         ;;
                 esac
                 ;;
+            "" | "help" | *)
+                help_interactive
+                ;;
         esac
         if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then
             report_outcomes