Merge branch '6388-centos-packages-wip'

[arvados-dev.git] / jenkins / run-tests.sh

diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 8e58d67e652b530cf9ab5b011c269ff670ca5216..38290f777fb0ca29498b3fdc37104b447c8d5f6c 100755 (executable)
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -111,7 +111,7 @@ clear_temp() {
 
 fatal() {
     clear_temp
-    echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+    echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
     exit 1
 }
 
@@ -135,6 +135,7 @@ report_outcomes() {
 
 exit_cleanly() {
     trap - INT
+    create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
     rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
     stop_services
     rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
@@ -144,47 +145,31 @@ exit_cleanly() {
 }
 
 sanity_checks() {
-  # Make sure WORKSPACE is set
-  if ! [[ -n "$WORKSPACE" ]]; then
-    echo >&2 "$helpmessage"
-    echo >&2
-    echo >&2 "Error: WORKSPACE environment variable not set"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure virtualenv is installed
-  `virtualenv --help >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: virtualenv could not be found"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure go is installed
-  `go env >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: go could not be found"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure gcc is installed
-  `gcc --help >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: gcc could not be found"
-    echo >&2
-    exit 1
-  fi
+    ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+        || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+    echo Checking dependencies:
+    echo -n 'virtualenv: '
+    virtualenv --version \
+        || fatal "No virtualenv. Try: apt-get install virtualenv"
+    echo -n 'go: '
+    go version \
+        || fatal "No go binary. See http://golang.org/doc/install"
+    echo -n 'gcc: '
+    gcc --version | egrep ^gcc \
+        || fatal "No gcc. Try: apt-get install build-essential"
+    echo -n 'fuse.h: '
+    find /usr/include -wholename '*fuse/fuse.h' \
+        || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+    echo -n 'pyconfig.h: '
+    find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+        || fatal "No pyconfig.h. Try: apt-get install python-dev"
+    echo -n 'nginx: '
+    PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+        || fatal "No nginx. Try: apt-get install nginx"
 }
 
 rotate_logfile() {
+  # i.e.  rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
   # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
   if [[ -f "$1/$2" ]]; then
     THEDATE=`date +%Y%m%d%H%M%S`
@@ -385,10 +370,6 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
 . "$VENVDIR/bin/activate"
 
-# When re-using $VENVDIR, upgrade any packages (except arvados) that are
-# already installed
-pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=`
-
 # Note: this must be the last time we change PATH, otherwise rvm will
 # whine a lot.
 setup_ruby_environment
@@ -401,8 +382,9 @@ then
 fi
 
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
-echo "pip install -q PyYAML"
-pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+    || pip install PyYAML >/dev/null \
+    || fatal "pip install PyYAML failed"
 
 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
 # Otherwise, skip dependent tests.
@@ -627,8 +609,9 @@ install_apiserver() {
     # Clear out any lingering postgresql connections to the test
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
-    test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
-    psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+    cd "$WORKSPACE/services/api" \
+        && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+        && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     cd "$WORKSPACE/services/api" \
         && RAILS_ENV=test bundle exec rake db:drop \
@@ -666,7 +649,7 @@ test_doclinkchecker() {
         ARVADOS_API_HOST=qr1hi.arvadosapi.com
         # Make sure python-epydoc is installed or the next line won't
         # do much good!
-        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
     )
 }
 do_test doc doclinkchecker