4156: Fix install destination for gem install --user-install. Make tmp-gem-home disco...

[arvados-dev.git] / jenkins / run-tests.sh

diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index 742baebad07df6f3bd3825da5f6b211f998fd3c4..6d8c1c3af9d8027de8bdd9535bf757362cb52837 100755 (executable)
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -20,9 +20,10 @@ Options:
                from a previous invocation if you use this option.
 WORKSPACE=path Arvados source tree to test.
 CONFIGSRC=path Dir with api server config files to copy into source tree.
                from a previous invocation if you use this option.
 WORKSPACE=path Arvados source tree to test.
 CONFIGSRC=path Dir with api server config files to copy into source tree.

-apiserver_test="TEST=test/functional/arvados/v1/collections_test.rb"
+               (If none given, leave config files alone in source tree.)
+apiserver_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"

                Restrict apiserver tests to the given file
                Restrict apiserver tests to the given file

-python_sdk_test="--test-suite test.test_keep_locator"
+sdk/python_test="--test-suite test.test_keep_locator"

                Restrict Python SDK tests to the given class
 workbench_test="TEST=test/integration/pipeline_instances_test.rb"
                Restrict Workbench tests to the given file
                Restrict Python SDK tests to the given class
 workbench_test="TEST=test/integration/pipeline_instances_test.rb"
                Restrict Workbench tests to the given file


@@ -36,6 +37,9 @@ into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
 test suites depend on other components being installed, and installing
 everything tends to be quicker than debugging dependencies.
 
 test suites depend on other components being installed, and installing
 everything tends to be quicker than debugging dependencies.
 

+As a special concession to the current CI server config, CONFIGSRC
+defaults to $HOME/arvados-api-server if that directory exists.
+

 More information and background:
 
 https://arvados.org/projects/arvados/wiki/Running_tests
 More information and background:
 
 https://arvados.org/projects/arvados/wiki/Running_tests


@@ -55,12 +59,6 @@ GEMHOME=
 
 COLUMNS=80
 
 
 COLUMNS=80
 

-cli_test=
-workbench_test=
-apiserver_test=
-python_sdk_test=
-ruby_sdk_test=
-fuse_test=

 leave_temp=
 skip_install=
 
 leave_temp=
 skip_install=
 


@@ -84,7 +82,7 @@ clear_temp() {
             leaving+=" $var=\"${!var}\""
         fi
     done
             leaving+=" $var=\"${!var}\""
         fi
     done

-    if [[ -z "$leaving" ]]; then
+    if [[ -n "$leaving" ]]; then

         echo "Leaving behind temp dirs: $leaving"
     fi
 }
         echo "Leaving behind temp dirs: $leaving"
     fi
 }


@@ -114,9 +112,7 @@ report_outcomes() {
 }
 declare -a failures
 declare -A skip
 }
 declare -a failures
 declare -A skip

-
-# Always skip CLI tests. They don't know how to use run_test_server.py.
-skip[cli]=1
+declare -A testargs

 
 while [[ -n "$1" ]]
 do
 
 while [[ -n "$1" ]]
 do


@@ -142,8 +138,13 @@ do
             leave_temp[GOPATH]=1
             leave_temp[GEMHOME]=1
             ;;
             leave_temp[GOPATH]=1
             leave_temp[GEMHOME]=1
             ;;

+        *_test=*)
+            suite="${arg%%_test=*}"
+            args="${arg#*=}"
+            testargs["$suite"]="$args"
+            ;;

         *=*)
         *=*)

-            eval $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
+            eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"

             ;;
         *)
             echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
             ;;
         *)
             echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"


@@ -153,16 +154,25 @@ do
 done
 
 # Sanity check
 done
 
 # Sanity check

+if ! [[ -n "$WORKSPACE" ]]; then
+  echo >&2 "$helpmessage"
+  echo >&2
+  echo >&2 "Error: WORKSPACE environment variable not set"
+  echo >&2
+  exit 1
+fi
+

 echo "WORKSPACE=$WORKSPACE"
 echo "WORKSPACE=$WORKSPACE"

-[[ -n "$WORKSPACE" ]] || fatal "WORKSPACE not set"

 
 

-if [[ -n "$CONFIGSRC" ]]; then
-    if [[ -d "$HOME/arvados-api-server" ]]; then
-        # Jenkins expects us to use this by default.
-        CONFIGSRC="$HOME/arvados-api-server"
-    fi
+if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
+    # Jenkins expects us to use this by default.
+    CONFIGSRC="$HOME/arvados-api-server"

 fi
 
 fi
 

+# Clean up .pyc files that may exist in the workspace
+cd "$WORKSPACE"
+find -name '*.pyc' -delete
+

 # Set up temporary install dirs (unless existing dirs were supplied)
 for tmpdir in VENVDIR GOPATH GEMHOME
 do
 # Set up temporary install dirs (unless existing dirs were supplied)
 for tmpdir in VENVDIR GOPATH GEMHOME
 do


@@ -172,14 +182,48 @@ do
         eval $tmpdir=$(mktemp -d)
     fi
 done
         eval $tmpdir=$(mktemp -d)
     fi
 done

-PATH="$GEMHOME/.gem/ruby/2.1.0/bin:$PATH"
+
+# When our "bundle install"s need to install new gems to satisfy
+# dependencies, we want them to go where "gem install --user-install"
+# would put them. If rvm is in use (or something else has set
+# GEM_HOME) we assume "bundle install" already does something
+# reasonable.
+
+if [ -z "$GEM_HOME" ]; then
+    user_gempath="$(gem env gempath)"
+    export GEM_HOME="${user_gempath%%:*}"
+fi
+PATH="$(gem env gemdir)/bin:$PATH"
+
+# When we build and install our own gems, we install them in our
+# $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
+# PATH so integration tests prefer them over other versions that
+# happen to be installed in $user_gempath, system dirs, etc.
+
+tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
+PATH="${tmpdir_gem_home%%:*}/bin:$PATH"
+export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
+
+echo "PATH=$PATH"
+echo "Will install dependencies to $GEM_HOME"
+echo "Will install arvados gems to $tmpdir_gem_home"
+echo "Gem search path is GEM_PATH=$GEM_PATH"
+
+if ! which bundler >/dev/null
+then
+    gem install --user-install bundler || fatal 'Could not install bundler'
+fi
+

 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
 virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"

-PATH="$VENVDIR/bin:$PATH"
+. "$VENVDIR/bin/activate"
+
+# Needed for run_test_server.py which is used by certain (non-Python) tests.
+pip install PyYAML

 
 checkexit() {
     if [[ "$?" != "0" ]]; then
 
 checkexit() {
     if [[ "$?" != "0" ]]; then


@@ -206,6 +250,18 @@ do_test() {
         if [[ "$2" == "go" ]]
         then
             go test "git.curoverse.com/arvados.git/$1"
         if [[ "$2" == "go" ]]
         then
             go test "git.curoverse.com/arvados.git/$1"

+        elif [[ "$2" == "pip" ]]
+        then
+           # Other test suites can depend on tests_require
+           # dependencies of this package. For example, keepproxy runs
+           # run_test_server.py, which depends on the yaml package,
+           # which is in sdk/python's tests_require but not
+           # install_requires, and therefore does not get installed by
+           # setuptools until we run "setup.py test" *and* install the
+           # .egg files that setup.py downloads.
+           cd "$WORKSPACE/$1" \
+                && python setup.py test ${testargs[$1]} \
+                && (easy_install *.egg || true)

         else
             "test_$1"
         fi
         else
             "test_$1"
         fi


@@ -224,6 +280,11 @@ do_install() {
         if [[ "$2" == "go" ]]
         then
             go get -t "git.curoverse.com/arvados.git/$1"
         if [[ "$2" == "go" ]]
         then
             go get -t "git.curoverse.com/arvados.git/$1"

+        elif [[ "$2" == "pip" ]]
+        then
+            cd "$WORKSPACE/$1" \
+                && python setup.py sdist rotate --keep=1 --match .tar.gz \
+                && pip install --upgrade dist/*.tar.gz

         else
             "install_$1"
         fi
         else
             "install_$1"
         fi


@@ -241,54 +302,51 @@ title () {
 
 install_docs() {
     cd "$WORKSPACE/doc"
 
 install_docs() {
     cd "$WORKSPACE/doc"

-    HOME="$GEMHOME" bundle install --no-deployment
+    bundle install --no-deployment

     rm -rf .site
     # Make sure python-epydoc is installed or the next line won't do much good!
     ARVADOS_API_HOST=qr1hi.arvadosapi.com
     rm -rf .site
     # Make sure python-epydoc is installed or the next line won't do much good!
     ARVADOS_API_HOST=qr1hi.arvadosapi.com

-    PYTHONPATH=$WORKSPACE/sdk/python/ HOME="$GEMHOME" bundle exec rake generate baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+    PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake generate baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST

     unset ARVADOS_API_HOST
 }
 do_install docs
 
 install_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
     unset ARVADOS_API_HOST
 }
 do_install docs
 
 install_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \

-        && HOME="$GEMHOME" bundle install --no-deployment \
+        && bundle install --no-deployment \

         && gem build arvados.gemspec \
         && gem build arvados.gemspec \

-        && HOME="$GEMHOME" gem install --user-install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
+        && GEM_HOME="$tmpdir_gem_home" gem install --user-install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`

 }
 do_install ruby_sdk
 
 install_cli() {
     cd "$WORKSPACE/sdk/cli" \
 }
 do_install ruby_sdk
 
 install_cli() {
     cd "$WORKSPACE/sdk/cli" \

+        && bundle install --no-deployment \

         && gem build arvados-cli.gemspec \
         && gem build arvados-cli.gemspec \

-        && HOME="$GEMHOME" gem install --user-install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
+        && GEM_HOME="$tmpdir_gem_home" gem install --user-install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`

 }
 do_install cli
 
 }
 do_install cli
 

-install_python_sdk() {
-    # Install the Python SDK early. Various other test suites (like
-    # keepproxy) bring up run_test_server.py, which imports the arvados
-    # module. We can't actually *test* the Python SDK yet though, because
-    # its own test suite brings up some of those other programs (like
-    # keepproxy).
-
-    cd "$WORKSPACE/sdk/python" \
-        && python setup.py egg_info -b ".$(git log --first-parent --format=format:%ci.%h -n1 . | tr -dc 0-9a-f. | cut -c 1-14,19-)" sdist rotate --keep=1 --match .tar.gz \
-        && pip install dist/arvados-python-client-0.1.*.tar.gz
-}
-do_install python_sdk
-
-install_fuse() {
-    cd "$WORKSPACE/services/fuse" \
-        && python setup.py egg_info -b ".$(git log --first-parent --format=format:%ci.%h -n1 . | tr -dc 0-9a-f. | cut -c 1-14,19-)" sdist rotate --keep=1 --match .tar.gz \
-        && pip install dist/arvados_fuse-0.1.*.tar.gz
-}
-do_install fuse
+# Install the Python SDK early. Various other test suites (like
+# keepproxy) bring up run_test_server.py, which imports the arvados
+# module. We can't actually *test* the Python SDK yet though, because
+# its own test suite brings up some of those other programs (like
+# keepproxy).
+declare -a pythonstuff
+pythonstuff=(
+    sdk/python
+    services/fuse
+    services/nodemanager
+    )
+for p in "${pythonstuff[@]}"
+do
+    do_install "$p" pip
+done

 
 install_apiserver() {
     cd "$WORKSPACE/services/api"
     export RAILS_ENV=test
 
 install_apiserver() {
     cd "$WORKSPACE/services/api"
     export RAILS_ENV=test

-    HOME="$GEMHOME" bundle install --no-deployment
+    bundle install --no-deployment

 
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
 
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb


@@ -327,14 +385,15 @@ install_apiserver() {
     psql arvados_test -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'arvados_test';" 2>/dev/null
 
     cd "$WORKSPACE/services/api" \
     psql arvados_test -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'arvados_test';" 2>/dev/null
 
     cd "$WORKSPACE/services/api" \

-        && HOME="$GEMHOME" bundle exec rake db:drop \
-        && HOME="$GEMHOME" bundle exec rake db:create \
-        && HOME="$GEMHOME" bundle exec rake db:setup
+        && bundle exec rake db:drop \
+        && bundle exec rake db:create \
+        && bundle exec rake db:setup

 }
 do_install apiserver
 
 declare -a gostuff
 gostuff=(
 }
 do_install apiserver
 
 declare -a gostuff
 gostuff=(

+    services/crunchstat

     services/keepstore
     services/keepproxy
     sdk/go/arvadosclient
     services/keepstore
     services/keepproxy
     sdk/go/arvadosclient


@@ -348,56 +407,38 @@ done
 
 test_doclinkchecker() {
     cd "$WORKSPACE/doc"
 
 test_doclinkchecker() {
     cd "$WORKSPACE/doc"

-    HOME="$GEMHOME" bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/
+    bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/

 }
 do_test doclinkchecker
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
 }
 do_test doclinkchecker
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \

-        && HOME="$GEMHOME" bundle install --no-deployment \
-        && HOME="$GEMHOME" bundle exec rake test
+        && bundle install --no-deployment \
+        && bundle exec rake test ${testargs[sdk/ruby]}

 }
 do_test ruby_sdk
 
 test_cli() {
     title "Starting SDK CLI tests"
     cd "$WORKSPACE/sdk/cli" \
 }
 do_test ruby_sdk
 
 test_cli() {
     title "Starting SDK CLI tests"
     cd "$WORKSPACE/sdk/cli" \

-        && HOME="$GEMHOME" bundle install --no-deployment \
+        && bundle install --no-deployment \

         && mkdir -p /tmp/keep \
         && mkdir -p /tmp/keep \

-        && KEEP_LOCAL_STORE=/tmp/keep HOME="$GEMHOME" bundle exec rake test $cli_test
+        && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test ${testargs[sdk/cli]}

 }
 do_test cli
 
 test_apiserver() {
     cd "$WORKSPACE/services/api"
 }
 do_test cli
 
 test_apiserver() {
     cd "$WORKSPACE/services/api"

-    HOME="$GEMHOME" bundle exec rake test $apiserver_test
+    bundle exec rake test ${testargs[apiserver]}

 }
 do_test apiserver
 
 }
 do_test apiserver
 

-test_python_sdk() {
-    # Python SDK. We test this before testing keepproxy: keepproxy runs
-    # run_test_server.py, which depends on the yaml package, which is in
-    # tests_require but not install_requires, and therefore does not get
-    # installed by setuptools until we run "setup.py test" *and* install
-    # the .egg files that setup.py downloads.
-
-    cd "$WORKSPACE/sdk/python" \
-        && python setup.py test $python_sdk_test
-    r=$?
-    easy_install *.egg
-    return $r
-}
-do_test python_sdk
-
-test_fuse() {
-    # Install test dependencies here too, in case run_test_server needs them.
-    cd "$WORKSPACE/services/fuse" \
-        && python setup.py test $fuse_test
-    r=$?
-    easy_install *.egg
-    return $r
-}
-do_test fuse
+# We must test sdk/python before testing services/keepproxy, because
+# keepproxy depends on sdk/python's test dependencies.
+for p in "${pythonstuff[@]}"
+do
+    do_test "$p" pip
+done

 
 for g in "${gostuff[@]}"
 do
 
 for g in "${gostuff[@]}"
 do


@@ -406,8 +447,8 @@ done
 
 test_workbench() {
     cd "$WORKSPACE/apps/workbench" \
 
 test_workbench() {
     cd "$WORKSPACE/apps/workbench" \

-        && HOME="$GEMHOME" bundle install --no-deployment \
-        && HOME="$GEMHOME" bundle exec rake test $workbench_test
+        && bundle install --no-deployment \
+        && bundle exec rake test ${testargs[workbench]}

 }
 do_test workbench
 
 }
 do_test workbench