X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1b22ec8483c87fbb5dfb31c5d6e72c8ec986ae9e..231a86fd3f7e30e9f66d71d92ad7c26578637e37:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 2764f7d344..b9dcd777fa 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -77,9 +77,14 @@ doc
 lib/cli
 lib/cmd
 lib/controller
+lib/controller/federation
+lib/controller/railsproxy
+lib/controller/router
+lib/controller/rpc
 lib/crunchstat
 lib/cloud
 lib/cloud/azure
+lib/cloud/cloudtest
 lib/dispatchcloud
 lib/dispatchcloud/container
 lib/dispatchcloud/scheduler
@@ -105,6 +110,7 @@ services/crunch-dispatch-slurm
 services/ws
 sdk/cli
 sdk/pam
+sdk/pam:py3
 sdk/python
 sdk/python:py3
 sdk/ruby
@@ -122,6 +128,7 @@ sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
 sdk/R
+sdk/java-v2
 tools/sync-groups
 tools/crunchstat-summary
 tools/crunchstat-summary:py3
@@ -379,6 +386,20 @@ checkpidfile() {
     echo "${svc} pid ${pid} ok"
 }
 
+checkhealth() {
+    svc="$1"
+    port="$(cat "$WORKSPACE/tmp/${svc}.port")"
+    scheme=http
+    if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
+        scheme=https
+    fi
+    url="$scheme://localhost:${port}/_health/ping"
+    if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
+        echo "${url} failed"
+        return 1
+    fi
+}
+
 checkdiscoverydoc() {
     dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
     if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
@@ -394,7 +415,7 @@ start_services() {
         return 0
     fi
     . "$VENVDIR/bin/activate"
-    echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
+    echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
 	mkdir -p "$WORKSPACE/services/api/log"
     fi
@@ -412,12 +433,15 @@ start_services() {
         && checkdiscoverydoc $ARVADOS_API_HOST \
         && python sdk/python/tests/run_test_server.py start_controller \
         && checkpidfile controller \
+        && checkhealth controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && checkpidfile keepproxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && checkpidfile keep-web \
+        && checkhealth keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && checkpidfile arv-git-httpd \
+        && checkhealth arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
         && checkpidfile ws \
         && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
@@ -629,23 +653,8 @@ install_env() {
         ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
         go get -v github.com/kardianos/govendor
         cd "$GOPATH/src/git.curoverse.com/arvados.git"
-        if [[ -n "$short" ]]; then
-            go get -v -d ...
-            "$GOPATH/bin/govendor" sync
-        else
-            # Remove cached source dirs in workdir. Otherwise, they will
-            # not qualify as +missing or +external below, and we won't be
-            # able to detect that they're missing from vendor/vendor.json.
-            rm -rf vendor/*/
-            go get -v -d ...
-            "$GOPATH/bin/govendor" sync
-            [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
-                || fatal "vendor/vendor.json has unused or missing dependencies -- try:
-
-(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
-
-";
-        fi
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
     ) || fatal "Go setup failed"
 
     setup_virtualenv "$VENVDIR" --python python2.7
@@ -730,7 +739,7 @@ do_test() {
         services/api)
             stop_services
             ;;
-        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+        gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
             # don't care whether services are running
             ;;
         *)
@@ -749,6 +758,7 @@ do_test_once() {
     title "test $1"
     timer_reset
 
+    result=
     if which deactivate >/dev/null; then deactivate; fi
     if ! . "$VENVDIR/bin/activate"
     then
@@ -821,6 +831,7 @@ do_install_once() {
     title "install $1"
     timer_reset
 
+    result=
     if which deactivate >/dev/null; then deactivate; fi
     if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then
         result=1
@@ -883,7 +894,7 @@ install_gem() {
         && cd "$WORKSPACE/$srcpath" \
         && bundle_install_trylocal \
         && gem build "$gemname.gemspec" \
-        && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+        && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1)
 }
 
 install_sdk/ruby() {
@@ -955,6 +966,7 @@ install_services/api() {
             || return 1
 
     cd "$WORKSPACE/services/api" \
+        && RAILS_ENV=test bundle exec rails db:environment:set \
         && RAILS_ENV=test bundle exec rake db:drop \
         && RAILS_ENV=test bundle exec rake db:setup \
         && RAILS_ENV=test bundle exec rake db:fixtures:load
@@ -971,53 +983,11 @@ pythonstuff=(
     services/fuse
     services/nodemanager
     tools/crunchstat-summary
+    tools/crunchstat-summary:py3
 )
 
 declare -a gostuff
-gostuff=(
-    cmd/arvados-client
-    cmd/arvados-server
-    lib/cli
-    lib/cmd
-    lib/controller
-    lib/crunchstat
-    lib/cloud
-    lib/cloud/azure
-    lib/cloud/ec2
-    lib/dispatchcloud
-    lib/dispatchcloud/container
-    lib/dispatchcloud/scheduler
-    lib/dispatchcloud/ssh_executor
-    lib/dispatchcloud/worker
-    lib/service
-    sdk/go/arvados
-    sdk/go/arvadosclient
-    sdk/go/auth
-    sdk/go/blockdigest
-    sdk/go/dispatch
-    sdk/go/health
-    sdk/go/httpserver
-    sdk/go/manifest
-    sdk/go/asyncbuf
-    sdk/go/crunchrunner
-    sdk/go/stats
-    services/arv-git-httpd
-    services/crunchstat
-    services/health
-    services/keep-web
-    services/keepstore
-    sdk/go/keepclient
-    services/keep-balance
-    services/keepproxy
-    services/crunch-dispatch-local
-    services/crunch-dispatch-slurm
-    services/crunch-run
-    services/ws
-    tools/keep-block-check
-    tools/keep-exercise
-    tools/keep-rsync
-    tools/sync-groups
-)
+gostuff=($(git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
 
 install_apps/workbench() {
     cd "$WORKSPACE/apps/workbench" \
@@ -1043,10 +1013,31 @@ test_gofmt() {
     [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
 }
 
+test_govendor() {
+    (
+        set -e
+        cd "$GOPATH/src/git.curoverse.com/arvados.git"
+        # Remove cached source dirs in workdir. Otherwise, they will
+        # not qualify as +missing or +external below, and we won't be
+        # able to detect that they're missing from vendor/vendor.json.
+        rm -rf vendor/*/
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
+        if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then
+            echo >&2 "vendor/vendor.json has unused or missing dependencies -- try:
+
+(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
+
+"
+            return 1
+        fi
+    )
+}
+
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test TESTOPTS='-v -d' ${testargs[services/api]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]}
 }
 
 test_sdk/ruby() {
@@ -1067,6 +1058,10 @@ test_sdk/cli() {
         && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
 }
 
+test_sdk/java-v2() {
+    cd "$WORKSPACE/sdk/java-v2" && gradle test
+}
+
 test_services/login-sync() {
     cd "$WORKSPACE/services/login-sync" \
         && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
@@ -1079,27 +1074,27 @@ test_services/nodemanager_integration() {
 
 test_apps/workbench_units() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test:units TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test:functionals TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test:integration TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
 }
 
 test_apps/workbench_benchmark() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test:benchmark ${testargs[apps/workbench_benchmark]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
 }
 
 test_apps/workbench_profile() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test:profile ${testargs[apps/workbench_profile]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
 }
 
 install_deps() {
@@ -1159,11 +1154,13 @@ test_all() {
     fi
 
     do_test gofmt
+    do_test govendor
     do_test doc
     do_test sdk/ruby
     do_test sdk/R
     do_test sdk/cli
     do_test services/login-sync
+    do_test sdk/java-v2
     do_test services/nodemanager_integration
     for p in "${pythonstuff[@]}"
     do
@@ -1217,6 +1214,8 @@ for p in "${pythonstuff[@]}"; do
     testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/"
 done
 
+testfuncargs["sdk/cli"]="sdk/cli"
+
 if [[ -z ${interactive} ]]; then
     install_all
     test_all
@@ -1230,19 +1229,21 @@ else
             # assume emacs, or something, is offering a history buffer
             # and pre-populating the command will only cause trouble
             nextcmd=
-        elif [[ "$nextcmd" != "install deps" ]]; then
-            :
-        elif [[ -e "$VENVDIR/bin/activate" ]]; then
-            nextcmd="test lib/cmd"
-        else
+        elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
             nextcmd="install deps"
+        else
+            nextcmd=""
         fi
     }
     echo
     help_interactive
     nextcmd="install deps"
     setnextcmd
-    while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+    HISTFILE="$WORKSPACE/tmp/.history"
+    history -r
+    while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+        history -s "$nextcmd"
+        history -w
         read verb target opts <<<"${nextcmd}"
         target="${target%/}"
         target="${target/\/:/:}"