7451: Improve virtualenv setup in run-tests.sh.

The code to build the Python 3 virtualenv didn't check to see whether
the virtualenv already existed first.  When reusing a Python 3
virtualenv, pip would get downgraded and start failing.

Refactor the virtualenv setup code to avoid this duplication and these
sorts of bugs.

While I was at it: rather than trying to parse package versions
ourselves to figure out when to upgrade setuptools and pip, just tell
pip the versions we want.  'pip>=7' is an approximation of what we're
currently doing.  The main thing we care about is that we get a
version new enough to work with setuptools 18.

diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 0224f334852b6abf6524c03f156b6c79cb059cef..2e5e3208fc121b1ca310672f7fbfeecceeb3427d 100755 (executable)
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -384,6 +384,14 @@ gem_uninstall_if_exists() {
     fi
 }
 
+setup_virtualenv() {
+    local venvdest=$1; shift
+    if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
+        virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+    fi
+    "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7'
+}
+
 export PERLINSTALLBASE
 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
@@ -392,15 +400,9 @@ mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
-if ! [[ -e "$VENVDIR/bin/activate" ]] || ! [[ -e "$VENVDIR/bin/pip" ]]; then
-    virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
-fi
+setup_virtualenv "$VENVDIR" --python python2.7
 . "$VENVDIR/bin/activate"
 
-if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\  -f2 | cut -d. -f1)" -lt 18 ]; then
-    pip install --upgrade setuptools pip
-fi
-
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
 pip freeze 2>/dev/null | egrep ^PyYAML= \
     || pip install PyYAML >/dev/null \
@@ -419,17 +421,7 @@ deactivate
 # Otherwise, skip dependent tests.
 PYTHON3=$(which python3)
 if [ "0" = "$?" ]; then
-    virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
-       || fatal "python3 virtualenv $VENV3DIR failed"
-
-    . "$VENV3DIR/bin/activate"
-
-    if (pip install setuptools | grep setuptools-0) || [ "$($VENV3DIR/bin/easy_install --version | cut -d\  -f2 | cut -d. -f1)" -lt 18 ]; then
-       pip install --upgrade setuptools pip
-    fi
-
-    # Deactivate Python 3 virtualenv
-    deactivate
+    setup_virtualenv "$VENV3DIR" --python python3
 else
     PYTHON3=
     skip[services/dockercleaner]=1