Added listAll utility function which will always fetch all resources.

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 57ce41e3a52b1e57bf1ec9e2afbb5c2f173de9ae..9081d99bb545cac5ff14272e63e8f842292585da 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -74,10 +74,10 @@ doc
 lib/cli
 lib/cmd
 lib/crunchstat
+lib/dispatchcloud
 services/api
 services/arv-git-httpd
 services/crunchstat
-services/dispatchcloud
 services/dockercleaner
 services/fuse
 services/health
@@ -109,6 +109,7 @@ sdk/go/asyncbuf
 sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
+sdk/R
 tools/sync-groups
 tools/crunchstat-summary
 tools/keep-exercise
@@ -133,6 +134,7 @@ VENV3DIR=
 PYTHONPATH=
 GEMHOME=
 PERLINSTALLBASE=
+R_LIBS=
 
 short=
 only_install=
@@ -240,6 +242,18 @@ sanity_checks() {
     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
+
+    if [[ "$NEED_SDK_R" = true ]]; then
+      # R SDK stuff
+      echo -n 'R: '
+      which R || fatal "No R. Try: apt-get install r-base"
+      echo -n 'testthat: '
+      R -q -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat"
+      # needed for roxygen2, needed for devtools, needed for R sdk
+      pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev"
+      # needed for pkgdown, builds R SDK doc pages
+      which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
+    fi
 }
 
 rotate_logfile() {
@@ -309,6 +323,18 @@ do
     esac
 done
 
+# R SDK installation is very slow (~360s in a clean environment) and only
+# required when testing it. Skip that step if it is not needed.
+NEED_SDK_R=true
+
+if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then
+  NEED_SDK_R=false
+fi
+
+if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then
+  NEED_SDK_R=false
+fi
+
 start_services() {
     echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
@@ -368,7 +394,7 @@ if [[ -z "$temp" ]]; then
 fi
 
 # Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
 do
     if [[ -z "${!tmpdir}" ]]; then
         eval "$tmpdir"="$temp/$tmpdir"
@@ -477,6 +503,7 @@ setup_virtualenv() {
 export PERLINSTALLBASE
 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
+export R_LIBS
 
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
@@ -513,13 +540,13 @@ setup_virtualenv "$VENVDIR" --python python2.7
 
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
 pip freeze 2>/dev/null | egrep ^PyYAML= \
-    || pip install PyYAML >/dev/null \
+    || pip install --no-cache-dir PyYAML >/dev/null \
     || fatal "pip install PyYAML failed"
 
-# Preinstall forked version of libcloud, because nodemanager "pip install"
+# Preinstall libcloud, because nodemanager "pip install"
 # won't pick it up by default.
 pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
-    || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
+    || pip install --pre --ignore-installed --no-cache-dir apache-libcloud>=$LIBCLOUD_PIN >/dev/null \
     || fatal "pip install apache-libcloud failed"
 
 # We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks.
@@ -688,11 +715,17 @@ do_test_once() {
 }
 
 do_install() {
-    if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
-        retry do_install_once ${@}
-    else
-        title "Skipping $1 install"
-    fi
+  skipit=false
+
+  if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+      retry do_install_once ${@}
+  else
+      skipit=true
+  fi
+
+  if [[ "$skipit" = true ]]; then
+    title "Skipping $1 install"
+  fi
 }
 
 do_install_once() {
@@ -766,6 +799,14 @@ install_ruby_sdk() {
 }
 do_install sdk/ruby ruby_sdk
 
+install_R_sdk() {
+  if [[ "$NEED_SDK_R" = true ]]; then
+    cd "$WORKSPACE/sdk/R" \
+       && R --quiet --vanilla --file=install_deps.R
+  fi
+}
+do_install sdk/R R_sdk
+
 install_perl_sdk() {
     cd "$WORKSPACE/sdk/perl" \
         && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
@@ -858,6 +899,7 @@ gostuff=(
     lib/cli
     lib/cmd
     lib/crunchstat
+    lib/dispatchcloud
     sdk/go/arvados
     sdk/go/arvadosclient
     sdk/go/blockdigest
@@ -870,7 +912,6 @@ gostuff=(
     sdk/go/stats
     services/arv-git-httpd
     services/crunchstat
-    services/dispatchcloud
     services/health
     services/keep-web
     services/keepstore
@@ -937,6 +978,15 @@ test_ruby_sdk() {
 }
 do_test sdk/ruby ruby_sdk
 
+test_R_sdk() {
+  if [[ "$NEED_SDK_R" = true ]]; then
+    cd "$WORKSPACE/sdk/R" \
+        && R --quiet --file=run_test.R
+  fi
+}
+
+do_test sdk/R R_sdk
+
 test_cli() {
     cd "$WORKSPACE/sdk/cli" \
         && mkdir -p /tmp/keep \