14360: Merge branch 'master'

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7d3646c9fbf2dfc0c4c4d21cf23e8c0c4dc5348b..cb44372566f8eb36ec6163f9108c97e464d6fca8 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -70,11 +70,17 @@ apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
 cmd/arvados-client
+cmd/arvados-server
 doc
 lib/cli
 lib/cmd
+lib/controller
 lib/crunchstat
 lib/dispatchcloud
+lib/dispatchcloud/container
+lib/dispatchcloud/scheduler
+lib/dispatchcloud/ssh_executor
+lib/dispatchcloud/worker
 services/api
 services/arv-git-httpd
 services/crunchstat
@@ -99,6 +105,7 @@ sdk/python:py3
 sdk/ruby
 sdk/go/arvados
 sdk/go/arvadosclient
+sdk/go/auth
 sdk/go/dispatch
 sdk/go/keepclient
 sdk/go/health
@@ -242,6 +249,8 @@ sanity_checks() {
     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
+    echo -n 'geckodriver: '
+    geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver"
 
     if [[ "$NEED_SDK_R" = true ]]; then
       # R SDK stuff
@@ -268,6 +277,7 @@ rotate_logfile() {
 
 declare -a failures
 declare -A skip
+declare -A only
 declare -A testargs
 skip[apps/workbench_profile]=1
 # nodemanager_integration tests are not reliable, see #12061.
@@ -286,7 +296,7 @@ do
             skip[$1]=1; shift
             ;;
         --only)
-            only="$1"; skip[$1]=""; shift
+            only[$1]=1; skip[$1]=""; shift
             ;;
         --short)
             short=1
@@ -329,14 +339,19 @@ done
 # required when testing it. Skip that step if it is not needed.
 NEED_SDK_R=true
 
-if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then
+if [[ ${#only[@]} -ne 0 ]] &&
+   [[ -z "${only['sdk/R']}" && -z "${only['doc']}" ]]; then
   NEED_SDK_R=false
 fi
 
-if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then
+if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then
   NEED_SDK_R=false
 fi
 
+if [[ $NEED_SDK_R == false ]]; then
+       echo "R SDK not needed, it will not be installed."
+fi
+
 start_services() {
     echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
@@ -347,15 +362,19 @@ start_services() {
        rm -f "$WORKSPACE/tmp/api.pid"
     fi
     cd "$WORKSPACE" \
-        && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+        && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo fail=1) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
+        && python sdk/python/tests/run_test_server.py start_controller \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
-        && python sdk/python/tests/run_test_server.py start_nginx \
+        && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo fail=1) \
         && (env | egrep ^ARVADOS)
+    if [[ -n "$fail" ]]; then
+       return 1
+    fi
 }
 
 stop_services() {
@@ -369,6 +388,7 @@ stop_services() {
         && python sdk/python/tests/run_test_server.py stop_ws \
         && python sdk/python/tests/run_test_server.py stop_keep-web \
         && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+        && python sdk/python/tests/run_test_server.py stop_controller \
         && python sdk/python/tests/run_test_server.py stop
 }
 
@@ -406,6 +426,8 @@ do
     fi
 done
 
+rm -vf "${WORKSPACE}/tmp/*.log"
+
 setup_ruby_environment() {
     if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
         source "$HOME/.rvm/scripts/rvm"
@@ -607,6 +629,12 @@ then
     gem install --user-install bundler || fatal 'Could not install bundler'
 fi
 
+# Jenkins config requires that glob tmp/*.log match something. Ensure
+# that happens even if we don't end up running services that set up
+# logging.
+mkdir -p "${WORKSPACE}/tmp/" || fatal "could not mkdir ${WORKSPACE}/tmp"
+touch "${WORKSPACE}/tmp/controller.log" || fatal "could not touch ${WORKSPACE}/tmp/controller.log"
+
 retry() {
     remain="${repeat}"
     while :
@@ -643,8 +671,9 @@ do_test() {
             ;;
     esac
     if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
-                (-z "${only}" || "${only}" == "${suite}" || \
-                 "${only}" == "${1}") ]]; then
+              (${#only[@]} -eq 0 || ${only[$suite]} -eq 1 || \
+                   ${only[$1]} -eq 1) ||
+                  ${only[$2]} -eq 1 ]]; then
         retry do_test_once ${@}
     else
         title "Skipping ${1} tests"
@@ -664,7 +693,7 @@ do_test_once() {
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
         # compilation errors.
-        go get -t "git.curoverse.com/arvados.git/$1" && \
+        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
             [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
             if [[ -n "${testargs[$1]}" ]]
@@ -716,7 +745,7 @@ do_test_once() {
 do_install() {
   skipit=false
 
-  if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+  if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
       retry do_install_once ${@}
   else
       skipit=true
@@ -732,7 +761,7 @@ do_install_once() {
     timer_reset
     if [[ "$2" == "go" ]]
     then
-        go get -t "git.curoverse.com/arvados.git/$1"
+        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
     elif [[ "$2" == "pip" ]]
     then
         # $3 can name a path directory for us to use, including trailing
@@ -895,12 +924,19 @@ do_install services/api apiserver
 declare -a gostuff
 gostuff=(
     cmd/arvados-client
+    cmd/arvados-server
     lib/cli
     lib/cmd
+    lib/controller
     lib/crunchstat
     lib/dispatchcloud
+    lib/dispatchcloud/container
+    lib/dispatchcloud/scheduler
+    lib/dispatchcloud/ssh_executor
+    lib/dispatchcloud/worker
     sdk/go/arvados
     sdk/go/arvadosclient
+    sdk/go/auth
     sdk/go/blockdigest
     sdk/go/dispatch
     sdk/go/health