12199: Merge branch 'master' into 12199-dispatch-to-node-type

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 46418be63a77763b0320075d6327c47a8cb768a2..1ef15cd3dbe9974f2fe37619f06845b8964a4ecf 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -77,6 +77,7 @@ lib/crunchstat
 services/api
 services/arv-git-httpd
 services/crunchstat
+services/dispatchcloud
 services/dockercleaner
 services/fuse
 services/health
@@ -108,6 +109,7 @@ sdk/go/asyncbuf
 sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
+sdk/R
 tools/sync-groups
 tools/crunchstat-summary
 tools/keep-exercise
@@ -132,6 +134,7 @@ VENV3DIR=
 PYTHONPATH=
 GEMHOME=
 PERLINSTALLBASE=
+R_LIBS=
 
 short=
 only_install=
@@ -239,6 +242,16 @@ sanity_checks() {
     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
+
+    # R SDK stuff
+    echo -n 'R: '
+    which R || fatal "No R. Try: apt-get install r-base"
+    echo -n 'testthat: '
+    R -q -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat"
+    # needed for roxygen2, needed for devtools, needed for R sdk
+    pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev"
+    # needed for pkgdown, builds R SDK doc pages
+    which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
 }
 
 rotate_logfile() {
@@ -367,7 +380,7 @@ if [[ -z "$temp" ]]; then
 fi
 
 # Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
 do
     if [[ -z "${!tmpdir}" ]]; then
         eval "$tmpdir"="$temp/$tmpdir"
@@ -476,13 +489,37 @@ setup_virtualenv() {
 export PERLINSTALLBASE
 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
+export R_LIBS
+
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+    "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+    "$GOPATH/src/git.curoverse.com/arvados.git"; do
+    [[ -d "$d" ]] && rmdir "$d"
+    [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 go get -v github.com/kardianos/govendor \
     || fatal "govendor install failed"
+cd "$GOPATH/src/git.curoverse.com/arvados.git" \
+    || fatal
+# Remove cached source dirs in workdir. Otherwise, they won't qualify
+# as +missing or +external below, and we won't be able to detect that
+# they're missing from vendor/vendor.json.
+rm -r vendor/*/
+go get -v -d ...
+"$GOPATH/bin/govendor" sync \
+    || fatal "govendor sync failed"
+[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+    || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+* govendor remove +unused
+* govendor add +missing +external
+"
+cd "$WORKSPACE"
+
 
 setup_virtualenv "$VENVDIR" --python python2.7
 . "$VENVDIR/bin/activate"
@@ -742,6 +779,24 @@ install_ruby_sdk() {
 }
 do_install sdk/ruby ruby_sdk
 
+install_R_sdk() {
+    cd "$WORKSPACE/sdk/R" \
+       && R --quiet --vanilla <<EOF
+options(repos=structure(c(CRAN="http://cran.wustl.edu/")))
+if (!requireNamespace("devtools")) {
+  install.packages("devtools")
+}
+if (!requireNamespace("roxygen2")) {
+  install.packages("roxygen2")
+}
+if (!requireNamespace("pkgdown")) {
+  devtools::install_github("hadley/pkgdown")
+}
+devtools::install_dev_deps()
+EOF
+}
+do_install sdk/R R_sdk
+
 install_perl_sdk() {
     cd "$WORKSPACE/sdk/perl" \
         && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
@@ -828,9 +883,6 @@ install_apiserver() {
 }
 do_install services/api apiserver
 
-cd "$GOPATH/src/git.curoverse.com/arvados.git" && \
-    "$GOPATH/bin/govendor" sync -v || \
-        fatal "govendor sync failed"
 declare -a gostuff
 gostuff=(
     cmd/arvados-client
@@ -849,6 +901,7 @@ gostuff=(
     sdk/go/stats
     services/arv-git-httpd
     services/crunchstat
+    services/dispatchcloud
     services/health
     services/keep-web
     services/keepstore
@@ -915,6 +968,12 @@ test_ruby_sdk() {
 }
 do_test sdk/ruby ruby_sdk
 
+test_R_sdk() {
+    cd "$WORKSPACE/sdk/R" \
+        && R --quiet --file=run_test.R
+}
+do_test sdk/R R_sdk
+
 test_cli() {
     cd "$WORKSPACE/sdk/cli" \
         && mkdir -p /tmp/keep \