X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5d164d74505c232cfa232d69a56273da2d02b72a..66d07bb91f91b0cd4c92c7ffa913f7181a3d4942:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index ec3c463770..2fb82554a5 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -70,10 +70,13 @@ apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
 cmd/arvados-client
+cmd/arvados-server
 doc
 lib/cli
 lib/cmd
+lib/controller
 lib/crunchstat
+lib/dispatchcloud
 services/api
 services/arv-git-httpd
 services/crunchstat
@@ -98,6 +101,7 @@ sdk/python:py3
 sdk/ruby
 sdk/go/arvados
 sdk/go/arvadosclient
+sdk/go/auth
 sdk/go/dispatch
 sdk/go/keepclient
 sdk/go/health
@@ -181,8 +185,8 @@ sanity_checks() {
     echo -n 'go: '
     go version \
         || fatal "No go binary. See http://golang.org/doc/install"
-    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \
-        || fatal "Go >= 1.8 required. See http://golang.org/doc/install"
+    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
+        || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
     echo -n 'gcc: '
     gcc --version | egrep ^gcc \
         || fatal "No gcc. Try: apt-get install build-essential"
@@ -241,16 +245,20 @@ sanity_checks() {
     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
-
-    # R SDK stuff
-    echo -n 'R: '
-    which R || fatal "No R. Try: apt-get install r-base"
-    echo -n 'testthat: '
-    R -q -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat"
-    # needed for roxygen2, needed for devtools, needed for R sdk
-    pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev"
-    # needed for pkgdown, builds R SDK doc pages
-    which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
+    echo -n 'geckodriver: '
+    geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver"
+
+    if [[ "$NEED_SDK_R" = true ]]; then
+      # R SDK stuff
+      echo -n 'R: '
+      which Rscript || fatal "No Rscript. Try: apt-get install r-base"
+      echo -n 'testthat: '
+      Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat"
+      # needed for roxygen2, needed for devtools, needed for R sdk
+      pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev"
+      # needed for pkgdown, builds R SDK doc pages
+      which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
+    fi
 }
 
 rotate_logfile() {
@@ -265,8 +273,11 @@ rotate_logfile() {
 
 declare -a failures
 declare -A skip
+declare -A only
 declare -A testargs
 skip[apps/workbench_profile]=1
+# nodemanager_integration tests are not reliable, see #12061.
+skip[services/nodemanager_integration]=1
 
 while [[ -n "$1" ]]
 do
@@ -281,7 +292,7 @@ do
             skip[$1]=1; shift
             ;;
         --only)
-            only="$1"; skip[$1]=""; shift
+            only[$1]=1; skip[$1]=""; shift
             ;;
         --short)
             short=1
@@ -320,6 +331,23 @@ do
     esac
 done
 
+# R SDK installation is very slow (~360s in a clean environment) and only
+# required when testing it. Skip that step if it is not needed.
+NEED_SDK_R=true
+
+if [[ ${#only[@]} -ne 0 ]] &&
+   [[ -z "${only['sdk/R']}" && -z "${only['doc']}" ]]; then
+  NEED_SDK_R=false
+fi
+
+if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then
+  NEED_SDK_R=false
+fi
+
+if [[ $NEED_SDK_R == false ]]; then
+	echo "R SDK not needed, it will not be installed."
+fi
+
 start_services() {
     echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
@@ -330,15 +358,19 @@ start_services() {
 	rm -f "$WORKSPACE/tmp/api.pid"
     fi
     cd "$WORKSPACE" \
-        && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+        && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
+        && python sdk/python/tests/run_test_server.py start_controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
-        && python sdk/python/tests/run_test_server.py start_nginx \
+        && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \
         && (env | egrep ^ARVADOS)
+    if [[ -n "$fail" ]]; then
+       return 1
+    fi
 }
 
 stop_services() {
@@ -352,6 +384,7 @@ stop_services() {
         && python sdk/python/tests/run_test_server.py stop_ws \
         && python sdk/python/tests/run_test_server.py stop_keep-web \
         && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+        && python sdk/python/tests/run_test_server.py stop_controller \
         && python sdk/python/tests/run_test_server.py stop
 }
 
@@ -389,6 +422,8 @@ do
     fi
 done
 
+rm -vf "${WORKSPACE}/tmp/*.log"
+
 setup_ruby_environment() {
     if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
         source "$HOME/.rvm/scripts/rvm"
@@ -474,15 +509,17 @@ setup_virtualenv() {
     local venvdest="$1"; shift
     if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
         virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+    elif [[ -n "$short" ]]; then
+        return
     fi
     if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then
         # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy
-        "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8'
+        "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8'
     else
-        "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7'
+        "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
     fi
     # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
-    "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
+    "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
 }
 
 export PERLINSTALLBASE
@@ -491,64 +528,59 @@ export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 export R_LIBS
 
 export GOPATH
-mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-for d in \
-    "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
-    "$GOPATH/src/git.curoverse.com/arvados.git"; do
-    [[ -d "$d" ]] && rmdir "$d"
-    [[ -h "$d" ]] && rm "$d"
-done
-ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
-    || fatal "symlink failed"
-go get -v github.com/kardianos/govendor \
-    || fatal "govendor install failed"
-cd "$GOPATH/src/git.curoverse.com/arvados.git" \
-    || fatal
-# Remove cached source dirs in workdir. Otherwise, they won't qualify
-# as +missing or +external below, and we won't be able to detect that
-# they're missing from vendor/vendor.json.
-rm -r vendor/*/
-go get -v -d ...
-"$GOPATH/bin/govendor" sync \
-    || fatal "govendor sync failed"
-[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
-    || fatal "vendor/vendor.json has unused or missing dependencies -- try:
-* govendor remove +unused
-* govendor add +missing +external
-"
-cd "$WORKSPACE"
-
+(
+    set -e
+    mkdir -p "$GOPATH/src/git.curoverse.com"
+    rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH"
+    if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then
+        for d in \
+            "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
+                "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
+                "$GOPATH/src/git.curoverse.com/arvados.git"; do
+            [[ -d "$d" ]] && rmdir "$d"
+        done
+    fi
+    for d in \
+        "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
+        "$GOPATH/src/git.curoverse.com/arvados.git"; do
+        [[ -h "$d" ]] && rm "$d"
+    done
+    ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+    go get -v github.com/kardianos/govendor
+    cd "$GOPATH/src/git.curoverse.com/arvados.git"
+    if [[ -n "$short" ]]; then
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
+    else
+        # Remove cached source dirs in workdir. Otherwise, they will
+        # not qualify as +missing or +external below, and we won't be
+        # able to detect that they're missing from vendor/vendor.json.
+        rm -rf vendor/*/
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
+        [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+            || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+
+(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
+
+";
+    fi
+) || fatal "Go setup failed"
 
 setup_virtualenv "$VENVDIR" --python python2.7
 . "$VENVDIR/bin/activate"
 
 # Needed for run_test_server.py which is used by certain (non-Python) tests.
-pip freeze 2>/dev/null | egrep ^PyYAML= \
-    || pip install PyYAML >/dev/null \
+pip install --no-cache-dir PyYAML \
     || fatal "pip install PyYAML failed"
 
-# Preinstall forked version of libcloud, because nodemanager "pip install"
-# won't pick it up by default.
-pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
-    || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
-    || fatal "pip install apache-libcloud failed"
-
-# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks.
-pip freeze | grep -x llfuse==1.2.0 || (
-    set -e
-    yes | pip uninstall llfuse || true
-    cython --version || fatal "no cython; try sudo apt-get install cython"
-    cd "$temp"
-    (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse)
-    cd python-llfuse
-    git checkout 620722fd990ea642ddb8e7412676af482c090c0c
-    git checkout setup.py
-    sed -i -e "s:'1\\.2':'1.2.0':" setup.py
-    python setup.py build_cython
-    python setup.py install --force
-) || fatal "llfuse fork failed"
-pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???"
+# Preinstall libcloud if using a fork; otherwise nodemanager "pip
+# install" won't pick it up by default.
+if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then
+    pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
+        || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \
+        || fatal "pip install apache-libcloud failed"
+fi
 
 # Deactivate Python 2 virtualenv
 deactivate
@@ -593,6 +625,12 @@ then
     gem install --user-install bundler || fatal 'Could not install bundler'
 fi
 
+# Jenkins config requires that glob tmp/*.log match something. Ensure
+# that happens even if we don't end up running services that set up
+# logging.
+mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp"
+touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log"
+
 retry() {
     remain="${repeat}"
     while :
@@ -629,8 +667,9 @@ do_test() {
             ;;
     esac
     if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
-                (-z "${only}" || "${only}" == "${suite}" || \
-                 "${only}" == "${1}") ]]; then
+              (${#only[@]} -eq 0 || ${only[$suite]} -eq 1 || \
+                   ${only[$1]} -eq 1) ||
+                  ${only[$2]} -eq 1 ]]; then
         retry do_test_once ${@}
     else
         title "Skipping ${1} tests"
@@ -650,7 +689,7 @@ do_test_once() {
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
         # compilation errors.
-        go get -t "git.curoverse.com/arvados.git/$1" && \
+        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
             [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
             if [[ -n "${testargs[$1]}" ]]
@@ -700,11 +739,17 @@ do_test_once() {
 }
 
 do_install() {
-    if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
-        retry do_install_once ${@}
-    else
-        title "Skipping $1 install"
-    fi
+  skipit=false
+
+  if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
+      retry do_install_once ${@}
+  else
+      skipit=true
+  fi
+
+  if [[ "$skipit" = true ]]; then
+    title "Skipping $1 install"
+  fi
 }
 
 do_install_once() {
@@ -712,7 +757,7 @@ do_install_once() {
     timer_reset
     if [[ "$2" == "go" ]]
     then
-        go get -t "git.curoverse.com/arvados.git/$1"
+        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1"
     elif [[ "$2" == "pip" ]]
     then
         # $3 can name a path directory for us to use, including trailing
@@ -730,8 +775,8 @@ do_install_once() {
         cd "$WORKSPACE/$1" \
             && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
             && cd "$WORKSPACE" \
-            && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
-            && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+            && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+            && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
     elif [[ "$2" != "" ]]
     then
         "install_$2"
@@ -779,20 +824,10 @@ install_ruby_sdk() {
 do_install sdk/ruby ruby_sdk
 
 install_R_sdk() {
+  if [[ "$NEED_SDK_R" = true ]]; then
     cd "$WORKSPACE/sdk/R" \
-       && R --quiet --vanilla <<EOF
-options(repos=structure(c(CRAN="http://cran.wustl.edu/")))
-if (!requireNamespace("devtools")) {
-  install.packages("devtools")
-}
-if (!requireNamespace("roxygen2")) {
-  install.packages("roxygen2")
-}
-if (!requireNamespace("pkgdown")) {
-  devtools::install_github("hadley/pkgdown")
-}
-devtools::install_dev_deps()
-EOF
+       && Rscript --vanilla install_deps.R
+  fi
 }
 do_install sdk/R R_sdk
 
@@ -852,7 +887,7 @@ install_apiserver() {
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
         && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
-        && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+        && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
 
@@ -885,11 +920,15 @@ do_install services/api apiserver
 declare -a gostuff
 gostuff=(
     cmd/arvados-client
+    cmd/arvados-server
     lib/cli
     lib/cmd
+    lib/controller
     lib/crunchstat
+    lib/dispatchcloud
     sdk/go/arvados
     sdk/go/arvadosclient
+    sdk/go/auth
     sdk/go/blockdigest
     sdk/go/dispatch
     sdk/go/health
@@ -967,9 +1006,12 @@ test_ruby_sdk() {
 do_test sdk/ruby ruby_sdk
 
 test_R_sdk() {
+  if [[ "$NEED_SDK_R" = true ]]; then
     cd "$WORKSPACE/sdk/R" \
-        && R --quiet --file=run_test.R
+        && Rscript --vanilla run_test.R
+  fi
 }
+
 do_test sdk/R R_sdk
 
 test_cli() {