X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/acdfc466ec52fbbf394f0e0b00544e809e74cec8..ea0e43b2e928db9e7ef77977adc789cf44bc81c2:/jenkins/run-tests.sh?ds=inline

diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 8e58d67..a646e36 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -111,7 +111,7 @@ clear_temp() {
 
 fatal() {
     clear_temp
-    echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+    echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
     exit 1
 }
 
@@ -135,6 +135,7 @@ report_outcomes() {
 
 exit_cleanly() {
     trap - INT
+    create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
     rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
     stop_services
     rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
@@ -144,47 +145,31 @@ exit_cleanly() {
 }
 
 sanity_checks() {
-  # Make sure WORKSPACE is set
-  if ! [[ -n "$WORKSPACE" ]]; then
-    echo >&2 "$helpmessage"
-    echo >&2
-    echo >&2 "Error: WORKSPACE environment variable not set"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure virtualenv is installed
-  `virtualenv --help >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: virtualenv could not be found"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure go is installed
-  `go env >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: go could not be found"
-    echo >&2
-    exit 1
-  fi
-
-  # Make sure gcc is installed
-  `gcc --help >/dev/null 2>&1`
-
-  if [[ "$?" != "0" ]]; then
-    echo >&2
-    echo >&2 "Error: gcc could not be found"
-    echo >&2
-    exit 1
-  fi
+    ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+        || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+    echo Checking dependencies:
+    echo -n 'virtualenv: '
+    virtualenv --version \
+        || fatal "No virtualenv. Try: apt-get install virtualenv"
+    echo -n 'go: '
+    go version \
+        || fatal "No go binary. See http://golang.org/doc/install"
+    echo -n 'gcc: '
+    gcc --version | egrep ^gcc \
+        || fatal "No gcc. Try: apt-get install build-essential"
+    echo -n 'fuse.h: '
+    find /usr/include -wholename '*fuse/fuse.h' \
+        || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+    echo -n 'pyconfig.h: '
+    find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+        || fatal "No pyconfig.h. Try: apt-get install python-dev"
+    echo -n 'nginx: '
+    PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+        || fatal "No nginx. Try: apt-get install nginx"
 }
 
 rotate_logfile() {
+  # i.e.  rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
   # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
   if [[ -f "$1/$2" ]]; then
     THEDATE=`date +%Y%m%d%H%M%S`
@@ -302,8 +287,9 @@ for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
 do
     if [[ -n "${!tmpdir}" ]]; then
         leave_temp[$tmpdir]=1
+        mkdir -p "${!tmpdir}"
     else
-        eval $tmpdir=$(mktemp -d)
+        eval "$tmpdir"='$(mktemp -d)'
     fi
 done
 
@@ -373,7 +359,13 @@ with_test_gemset() {
     if [[ "$using_rvm" == true ]]; then
         "$@"
     else
-        GEM_HOME="$tmpdir_gem_home" "$@"
+        GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
+    fi
+}
+
+gem_uninstall_if_exists() {
+    if gem list "$1\$" | egrep '^\w'; then
+        gem uninstall --force --all --executables "$1"
     fi
 }
 
@@ -385,10 +377,6 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
 . "$VENVDIR/bin/activate"
 
-# When re-using $VENVDIR, upgrade any packages (except arvados) that are
-# already installed
-pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=`
-
 # Note: this must be the last time we change PATH, otherwise rvm will
 # whine a lot.
 setup_ruby_environment
@@ -401,8 +389,9 @@ then
 fi
 
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
-echo "pip install -q PyYAML"
-pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+    || pip install PyYAML >/dev/null \
+    || fatal "pip install PyYAML failed"
 
 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
 # Otherwise, skip dependent tests.
@@ -552,7 +541,7 @@ install_doc() {
 do_install doc
 
 install_ruby_sdk() {
-    with_test_gemset gem uninstall --force --all --executables arvados \
+    with_test_gemset gem_uninstall_if_exists arvados \
         && cd "$WORKSPACE/sdk/ruby" \
         && bundle_install_trylocal \
         && gem build arvados.gemspec \
@@ -561,7 +550,7 @@ install_ruby_sdk() {
 do_install sdk/ruby ruby_sdk
 
 install_cli() {
-    with_test_gemset gem uninstall --force --all --executables arvados-cli \
+    with_test_gemset gem_uninstall_if_exists arvados-cli \
         && cd "$WORKSPACE/sdk/cli" \
         && bundle_install_trylocal \
         && gem build arvados-cli.gemspec \
@@ -627,8 +616,9 @@ install_apiserver() {
     # Clear out any lingering postgresql connections to the test
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
-    test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
-    psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+    cd "$WORKSPACE/services/api" \
+        && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+        && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     cd "$WORKSPACE/services/api" \
         && RAILS_ENV=test bundle exec rake db:drop \
@@ -666,7 +656,7 @@ test_doclinkchecker() {
         ARVADOS_API_HOST=qr1hi.arvadosapi.com
         # Make sure python-epydoc is installed or the next line won't
         # do much good!
-        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
     )
 }
 do_test doc doclinkchecker