8460: Merge branch 'master' into 8460-websocket-go

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index c9cb2b2199a5b155b8334ce00182e14b276ded77..e0e1ce28524327851e6925d26284530ced1dd5a0 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -79,6 +79,7 @@ services/nodemanager
 services/crunch-run
 services/crunch-dispatch-local
 services/crunch-dispatch-slurm
+services/ws
 sdk/cli
 sdk/pam
 sdk/python
@@ -90,9 +91,11 @@ sdk/go/httpserver
 sdk/go/manifest
 sdk/go/blockdigest
 sdk/go/streamer
+sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
 tools/crunchstat-summary
+tools/keep-exercise
 tools/keep-rsync
 tools/keep-block-check
 
@@ -155,11 +158,17 @@ sanity_checks() {
     echo -n 'virtualenv: '
     virtualenv --version \
         || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
+    echo -n 'ruby: '
+    ruby -v \
+        || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)"
+    echo -n 'bundler: '
+    bundle version \
+        || fatal "No bundler. Try: gem install bundler"
     echo -n 'go: '
     go version \
         || fatal "No go binary. See http://golang.org/doc/install"
-    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 6 ]] \
-        || fatal "Go >= 1.6 required. See http://golang.org/doc/install"
+    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 7 ]] \
+        || fatal "Go >= 1.7 required. See http://golang.org/doc/install"
     echo -n 'gcc: '
     gcc --version | egrep ^gcc \
         || fatal "No gcc. Try: apt-get install build-essential"
@@ -263,15 +272,18 @@ start_api() {
         && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
+        && python sdk/python/tests/run_test_server.py start_ws \
+        && python sdk/python/tests/run_test_server.py start_nginx \
         && (env | egrep ^ARVADOS)
 }
 
 start_nginx_proxy_services() {
-    echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
+    echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     cd "$WORKSPACE" \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+        && python sdk/python/tests/run_test_server.py start_ws \
         && python sdk/python/tests/run_test_server.py start_nginx \
         && export ARVADOS_TEST_PROXY_SERVICES=1
 }
@@ -282,12 +294,15 @@ stop_services() {
         cd "$WORKSPACE" \
             && python sdk/python/tests/run_test_server.py stop_nginx \
             && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+            && python sdk/python/tests/run_test_server.py stop_ws \
             && python sdk/python/tests/run_test_server.py stop_keep-web \
             && python sdk/python/tests/run_test_server.py stop_keep_proxy
     fi
     if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
         unset ARVADOS_TEST_API_HOST
         cd "$WORKSPACE" \
+            && python sdk/python/tests/run_test_server.py stop_nginx \
+            && python sdk/python/tests/run_test_server.py stop_ws \
             && python sdk/python/tests/run_test_server.py stop
     fi
 }
@@ -520,6 +535,8 @@ do_test_once() {
             # mode makes Go show the wrong line numbers when reporting
             # compilation errors.
             go get -t "git.curoverse.com/arvados.git/$1" || return 1
+            cd "$WORKSPACE/$1" || return 1
+            gofmt -e -d . | egrep . && result=1
             if [[ -n "${testargs[$1]}" ]]
             then
                 # "go test -check.vv giturl" doesn't work, but this
@@ -530,7 +547,7 @@ do_test_once() {
                 # empty, so use this form in such cases:
                 go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
             fi
-            result="$?"
+            result=${result:-$?}
             if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
             then
                 go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
@@ -538,10 +555,22 @@ do_test_once() {
             fi
         elif [[ "$2" == "pip" ]]
         then
-            # $3 can name a path directory for us to use, including trailing
-            # slash; e.g., the bin/ subdirectory of a virtualenv.
-            cd "$WORKSPACE/$1" \
-                && "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+            tries=0
+            cd "$WORKSPACE/$1" && while :
+            do
+                tries=$((${tries}+1))
+                # $3 can name a path directory for us to use, including trailing
+                # slash; e.g., the bin/ subdirectory of a virtualenv.
+                "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+                result=$?
+                if [[ ${tries} < 3 && ${result} == 137 ]]
+                then
+                    printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1"
+                    continue
+                else
+                    break
+                fi
+            done
         elif [[ "$2" != "" ]]
         then
             "test_$2"
@@ -735,6 +764,7 @@ gostuff=(
     sdk/go/manifest
     sdk/go/streamer
     sdk/go/crunchrunner
+    sdk/go/stats
     lib/crunchstat
     services/arv-git-httpd
     services/crunchstat
@@ -743,15 +773,13 @@ gostuff=(
     sdk/go/keepclient
     services/keep-balance
     services/keepproxy
-    services/datamanager/summary
-    services/datamanager/collection
-    services/datamanager/keep
-    services/datamanager
     services/crunch-dispatch-local
     services/crunch-dispatch-slurm
     services/crunch-run
-    tools/keep-rsync
+    services/ws
     tools/keep-block-check
+    tools/keep-exercise
+    tools/keep-rsync
     )
 for g in "${gostuff[@]}"
 do