X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/848ccc7efab5dab22f7fedd920daffbd102d09a7..c6f9a3fc32babe5c0d26a1b23a89a5c7917af8db:/jenkins/run-tests.sh

diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index cd6e1d0..9ca70b8 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -60,9 +60,11 @@ services/dockercleaner
 services/fuse
 services/keepproxy
 services/keepstore
+services/login-sync
 services/nodemanager
 services/arv-git-httpd
 sdk/cli
+sdk/pam
 sdk/python
 sdk/ruby
 sdk/go/arvadosclient
@@ -83,6 +85,7 @@ VENVDIR=
 VENV3DIR=
 PYTHONPATH=
 GEMHOME=
+PERLINSTALLBASE=
 
 COLUMNS=80
 
@@ -92,7 +95,7 @@ skip_install=
 declare -A leave_temp
 clear_temp() {
     leaving=""
-    for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
+    for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME PERLINSTALLBASE
     do
         if [[ -z "${leave_temp[$var]}" ]]
         then
@@ -166,6 +169,17 @@ sanity_checks() {
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
+    echo -n 'perl: '
+    perl -v | grep version \
+        || fatal "No perl. Try: apt-get install perl"
+    for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
+        echo -n "perl $mod: "
+        perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
+            || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl"
+    done
+    echo -n 'gitolite: '
+    which gitolite \
+        || fatal "No gitolite. Try: apt-get install gitolite3"
 }
 
 rotate_logfile() {
@@ -211,6 +225,7 @@ do
             leave_temp[VENV3DIR]=1
             leave_temp[GOPATH]=1
             leave_temp[GEMHOME]=1
+            leave_temp[PERLINSTALLBASE]=1
             ;;
         --retry)
             retry=1
@@ -283,12 +298,13 @@ cd "$WORKSPACE"
 find -name '*.pyc' -delete
 
 # Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
 do
     if [[ -n "${!tmpdir}" ]]; then
         leave_temp[$tmpdir]=1
+        mkdir -p "${!tmpdir}"
     else
-        eval $tmpdir=$(mktemp -d)
+        eval "$tmpdir"='$(mktemp -d)'
     fi
 done
 
@@ -358,18 +374,33 @@ with_test_gemset() {
     if [[ "$using_rvm" == true ]]; then
         "$@"
     else
-        GEM_HOME="$tmpdir_gem_home" "$@"
+        GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
     fi
 }
 
+gem_uninstall_if_exists() {
+    if gem list "$1\$" | egrep '^\w'; then
+        gem uninstall --force --all --executables "$1"
+    fi
+}
+
+export PERLINSTALLBASE
+export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 
-virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
+if ! [[ -e "$VENVDIR/bin/activate" ]] || ! [[ -e "$VENVDIR/bin/pip" ]]; then
+    virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
+fi
 . "$VENVDIR/bin/activate"
 
+if (pip install setuptools | grep setuptools-0) || [ "$($VENVDIR/bin/easy_install --version | cut -d\  -f2 | cut -d. -f1)" -lt 18 ]; then
+    pip install --upgrade setuptools pip
+fi
+
 # Note: this must be the last time we change PATH, otherwise rvm will
 # whine a lot.
 setup_ruby_environment
@@ -383,9 +414,15 @@ fi
 
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
 pip freeze 2>/dev/null | egrep ^PyYAML= \
-    || pip install PyYAML \
+    || pip install PyYAML >/dev/null \
     || fatal "pip install PyYAML failed"
 
+# Preinstall forked version of libcloud, because nodemanager "pip install"
+# won't pick it up by default.
+pip freeze 2>/dev/null | egrep ^apache-libcloud==0.18.1.dev1 \
+    || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev1.zip >/dev/null \
+    || fatal "pip install apache-libcloud failed"
+
 # If Python 3 is available, set up its virtualenv in $VENV3DIR.
 # Otherwise, skip dependent tests.
 PYTHON3=$(which python3)
@@ -486,10 +523,10 @@ do_install() {
             # the source dist package to avoid a pip bug.
             # see https://arvados.org/issues/5766 for details.
 
-            # Also need to install twice, because if it belives the package is
+            # Also need to install twice, because if it believes the package is
             # already installed, pip it won't install it.  So the first "pip
             # install" ensures that the dependencies are met, the second "pip
-            # install" ensures that we've actually install the local package
+            # install" ensures that we've actually installed the local package
             # we just built.
             cd "$WORKSPACE/$1" \
                 && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
@@ -533,24 +570,38 @@ install_doc() {
 }
 do_install doc
 
-install_ruby_sdk() {
-    with_test_gemset gem uninstall --force --all --executables arvados \
-        && cd "$WORKSPACE/sdk/ruby" \
+install_gem() {
+    gemname=$1
+    srcpath=$2
+    with_test_gemset gem_uninstall_if_exists "$gemname" \
+        && cd "$WORKSPACE/$srcpath" \
         && bundle_install_trylocal \
-        && gem build arvados.gemspec \
-        && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
+        && gem build "$gemname.gemspec" \
+        && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+}
+
+install_ruby_sdk() {
+    install_gem arvados sdk/ruby
 }
 do_install sdk/ruby ruby_sdk
 
+install_perl_sdk() {
+    cd "$WORKSPACE/sdk/perl" \
+        && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
+        && make install INSTALLDIRS=perl
+}
+do_install sdk/perl perl_sdk
+
 install_cli() {
-    with_test_gemset gem uninstall --force --all --executables arvados-cli \
-        && cd "$WORKSPACE/sdk/cli" \
-        && bundle_install_trylocal \
-        && gem build arvados-cli.gemspec \
-        && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
+    install_gem arvados-cli sdk/cli
 }
 do_install sdk/cli cli
 
+install_login-sync() {
+    install_gem arvados-login-sync services/login-sync
+}
+do_install services/login-sync login-sync
+
 # Install the Python SDK early. Various other test suites (like
 # keepproxy) bring up run_test_server.py, which imports the arvados
 # module. We can't actually *test* the Python SDK yet though, because
@@ -558,6 +609,7 @@ do_install sdk/cli cli
 # keepproxy).
 declare -a pythonstuff
 pythonstuff=(
+    sdk/pam
     sdk/python
     services/fuse
     services/nodemanager
@@ -626,6 +678,8 @@ gostuff=(
     services/crunchstat
     services/keepstore
     services/keepproxy
+    services/datamanager/summary
+    services/datamanager/collection
     sdk/go/arvadosclient
     sdk/go/keepclient
     sdk/go/streamer
@@ -649,7 +703,7 @@ test_doclinkchecker() {
         ARVADOS_API_HOST=qr1hi.arvadosapi.com
         # Make sure python-epydoc is installed or the next line won't
         # do much good!
-        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+        PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
     )
 }
 do_test doc doclinkchecker
@@ -684,6 +738,12 @@ test_cli() {
 }
 do_test sdk/cli cli
 
+test_login-sync() {
+    cd "$WORKSPACE/services/login-sync" \
+        && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
+}
+do_test services/login-sync login-sync
+
 for p in "${pythonstuff[@]}"
 do
     do_test "$p" pip