Merge branch '12020-flaky-py-test'

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 85504db0360e3d1921398a09b756e53f619fa2c2..a46f2ec7660590cd82a3969e22ebecaceafe1dc2 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -96,6 +96,7 @@ services/arv-git-httpd
 services/crunchstat
 services/dockercleaner
 services/fuse
+services/fuse:py3
 services/health
 services/keep-web
 services/keepproxy
@@ -226,6 +227,8 @@ sanity_checks() {
     echo -n 'Python3 pyconfig.h: '
     find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
         || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+    which netstat \
+        || fatal "No netstat. Try: apt-get install net-tools"
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
@@ -255,12 +258,6 @@ sanity_checks() {
     echo -n 'libpq libpq-fe.h: '
     find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
         || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
-    echo -n 'services/api/config/database.yml: '
-    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-           fatal "Please provide a database.yml file for the test suite"
-    else
-           echo "OK"
-    fi
     echo -n 'postgresql: '
     psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
     echo -n 'phantomjs: '
@@ -626,11 +623,30 @@ initialize() {
 
     unset http_proxy https_proxy no_proxy
 
-
     # Note: this must be the last time we change PATH, otherwise rvm will
     # whine a lot.
     setup_ruby_environment
 
+    if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+       cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
+       export ARVADOS_CONFIG="$temp/test-config.yml"
+    else
+       if [[ -s /etc/arvados/config.yml ]] ; then
+           python > "$temp/test-config.yml" <<EOF
+import yaml
+import json
+v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0]['PostgreSQL']
+v['Connection']['dbname'] = 'arvados_test'
+print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
+EOF
+           export ARVADOS_CONFIG="$temp/test-config.yml"
+       else
+           if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+               fatal "Please provide a database.yml file for the test suite"
+           fi
+       fi
+    fi
+
     echo "PATH is $PATH"
 }
 
@@ -656,6 +672,7 @@ install_env() {
         cd "$GOPATH/src/git.curoverse.com/arvados.git"
         go get -v -d ...
         "$GOPATH/bin/govendor" sync
+        which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports
     ) || fatal "Go setup failed"
 
     setup_virtualenv "$VENVDIR" --python python2.7
@@ -799,7 +816,10 @@ do_test_once() {
             tries=$((${tries}+1))
             # $3 can name a path directory for us to use, including trailing
             # slash; e.g., the bin/ subdirectory of a virtualenv.
-            "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+            if [[ -e "${3}activate" ]]; then
+                . "${3}activate"
+            fi
+            python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
             result=$?
             if [[ ${tries} < 3 && ${result} == 137 ]]
             then
@@ -924,6 +944,7 @@ install_services/login-sync() {
 }
 
 install_services/api() {
+    stop_services
     cd "$WORKSPACE/services/api" \
         && RAILS_ENV=test bundle_install_trylocal
 
@@ -982,6 +1003,7 @@ pythonstuff=(
     sdk/cwl:py3
     services/dockercleaner:py3
     services/fuse
+    services/fuse:py3
     services/nodemanager
     tools/crunchstat-summary
     tools/crunchstat-summary:py3