X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6e8c3cc1fe4328b708d5979e27632835f74261fb..f9e1447e5ab21dbc0fdb08328c31811fca2fd327:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 3ae0b16c8e..57ce41e3a5 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -69,10 +69,15 @@ apps/workbench_functionals (*)
 apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
+cmd/arvados-client
 doc
+lib/cli
+lib/cmd
+lib/crunchstat
 services/api
 services/arv-git-httpd
 services/crunchstat
+services/dispatchcloud
 services/dockercleaner
 services/fuse
 services/health
@@ -100,10 +105,11 @@ sdk/go/health
 sdk/go/httpserver
 sdk/go/manifest
 sdk/go/blockdigest
-sdk/go/streamer
+sdk/go/asyncbuf
 sdk/go/stats
 sdk/go/crunchrunner
 sdk/cwl
+tools/sync-groups
 tools/crunchstat-summary
 tools/keep-exercise
 tools/keep-rsync
@@ -211,6 +217,29 @@ sanity_checks() {
     echo -n 'cadaver: '
     cadaver --version | grep -w cadaver \
           || fatal "No cadaver. Try: apt-get install cadaver"
+    echo -n 'libattr1 xattr.h: '
+    find /usr/include -path '*/attr/xattr.h' | egrep --max-count=1 . \
+        || fatal "No libattr1 xattr.h. Try: apt-get install libattr1-dev"
+    echo -n 'libcurl curl.h: '
+    find /usr/include -path '*/curl/curl.h' | egrep --max-count=1 . \
+        || fatal "No libcurl curl.h. Try: apt-get install libcurl4-gnutls-dev"
+    echo -n 'libpq libpq-fe.h: '
+    find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
+        || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
+    echo -n 'services/api/config/database.yml: '
+    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+	    fatal "Please provide a database.yml file for the test suite"
+    else
+	    echo "OK"
+    fi
+    echo -n 'postgresql: '
+    psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
+    echo -n 'phantomjs: '
+    phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs"
+    echo -n 'xvfb: '
+    which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
+    echo -n 'graphviz: '
+    dot -V || fatal "No graphviz. Try: apt-get install graphviz"
 }
 
 rotate_logfile() {
@@ -280,45 +309,39 @@ do
     esac
 done
 
-start_api() {
-    echo 'Starting API server...'
+start_services() {
+    echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
+    if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
+	mkdir -p "$WORKSPACE/services/api/log"
+    fi
+    # Remove empty api.pid file if it exists
+    if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
+	rm -f "$WORKSPACE/tmp/api.pid"
+    fi
     cd "$WORKSPACE" \
         && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
         && export ARVADOS_TEST_API_INSTALLED="$$" \
-        && python sdk/python/tests/run_test_server.py start_ws \
-        && python sdk/python/tests/run_test_server.py start_nginx \
-        && (env | egrep ^ARVADOS)
-}
-
-start_nginx_proxy_services() {
-    echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
-    cd "$WORKSPACE" \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
         && python sdk/python/tests/run_test_server.py start_keep-web \
         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
         && python sdk/python/tests/run_test_server.py start_ws \
         && python sdk/python/tests/run_test_server.py start_nginx \
-        && export ARVADOS_TEST_PROXY_SERVICES=1
+        && (env | egrep ^ARVADOS)
 }
 
 stop_services() {
-    if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
-        unset ARVADOS_TEST_PROXY_SERVICES
-        cd "$WORKSPACE" \
-            && python sdk/python/tests/run_test_server.py stop_nginx \
-            && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
-            && python sdk/python/tests/run_test_server.py stop_ws \
-            && python sdk/python/tests/run_test_server.py stop_keep-web \
-            && python sdk/python/tests/run_test_server.py stop_keep_proxy
-    fi
-    if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
-        unset ARVADOS_TEST_API_HOST
-        cd "$WORKSPACE" \
-            && python sdk/python/tests/run_test_server.py stop_nginx \
-            && python sdk/python/tests/run_test_server.py stop_ws \
-            && python sdk/python/tests/run_test_server.py stop
+    if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then
+        return
     fi
+    unset ARVADOS_TEST_API_HOST
+    cd "$WORKSPACE" \
+        && python sdk/python/tests/run_test_server.py stop_nginx \
+        && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+        && python sdk/python/tests/run_test_server.py stop_ws \
+        && python sdk/python/tests/run_test_server.py stop_keep-web \
+        && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+        && python sdk/python/tests/run_test_server.py stop
 }
 
 interrupt() {
@@ -454,12 +477,36 @@ setup_virtualenv() {
 export PERLINSTALLBASE
 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
+
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+    "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+    "$GOPATH/src/git.curoverse.com/arvados.git"; do
+    [[ -d "$d" ]] && rmdir "$d"
+    [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 go get -v github.com/kardianos/govendor \
     || fatal "govendor install failed"
+cd "$GOPATH/src/git.curoverse.com/arvados.git" \
+    || fatal
+# Remove cached source dirs in workdir. Otherwise, they won't qualify
+# as +missing or +external below, and we won't be able to detect that
+# they're missing from vendor/vendor.json.
+rm -r vendor/*/
+go get -v -d ...
+"$GOPATH/bin/govendor" sync \
+    || fatal "govendor sync failed"
+[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+    || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+* govendor remove +unused
+* govendor add +missing +external
+"
+cd "$WORKSPACE"
+
 
 setup_virtualenv "$VENVDIR" --python python2.7
 . "$VENVDIR/bin/activate"
@@ -779,6 +826,25 @@ install_apiserver() {
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
 
+    cert="$WORKSPACE/services/api/tmp/self-signed"
+    if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
+        (
+            dir="$WORKSPACE/services/api/tmp"
+            set -ex
+            openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
+            openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
+        ) || return 1
+    fi
+
+    cd "$WORKSPACE/services/api" \
+        && rm -rf tmp/git \
+        && mkdir -p tmp/git \
+        && cd tmp/git \
+        && tar xf ../../test/test.git.tar \
+        && mkdir -p internal.git \
+        && git --git-dir internal.git init \
+            || return 1
+
     cd "$WORKSPACE/services/api" \
         && RAILS_ENV=test bundle exec rake db:drop \
         && RAILS_ENV=test bundle exec rake db:setup \
@@ -786,11 +852,12 @@ install_apiserver() {
 }
 do_install services/api apiserver
 
-cd "$GOPATH/src/git.curoverse.com/arvados.git" && \
-    "$GOPATH/bin/govendor" sync -v || \
-        fatal "govendor sync failed"
 declare -a gostuff
 gostuff=(
+    cmd/arvados-client
+    lib/cli
+    lib/cmd
+    lib/crunchstat
     sdk/go/arvados
     sdk/go/arvadosclient
     sdk/go/blockdigest
@@ -798,12 +865,12 @@ gostuff=(
     sdk/go/health
     sdk/go/httpserver
     sdk/go/manifest
-    sdk/go/streamer
+    sdk/go/asyncbuf
     sdk/go/crunchrunner
     sdk/go/stats
-    lib/crunchstat
     services/arv-git-httpd
     services/crunchstat
+    services/dispatchcloud
     services/health
     services/keep-web
     services/keepstore
@@ -817,6 +884,7 @@ gostuff=(
     tools/keep-block-check
     tools/keep-exercise
     tools/keep-rsync
+    tools/sync-groups
 )
 for g in "${gostuff[@]}"
 do
@@ -861,7 +929,7 @@ if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
   exit_cleanly
 fi
 
-start_api || { stop_services; fatal "start_api"; }
+start_services || { stop_services; fatal "start_services"; }
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
@@ -908,37 +976,32 @@ do
 done
 
 test_workbench_units() {
-    start_nginx_proxy_services \
-        && cd "$WORKSPACE/apps/workbench" \
+    cd "$WORKSPACE/apps/workbench" \
         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
 }
 do_test apps/workbench_units workbench_units
 
 test_workbench_functionals() {
-    start_nginx_proxy_services \
-        && cd "$WORKSPACE/apps/workbench" \
+    cd "$WORKSPACE/apps/workbench" \
         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
 }
 do_test apps/workbench_functionals workbench_functionals
 
 test_workbench_integration() {
-    start_nginx_proxy_services \
-        && cd "$WORKSPACE/apps/workbench" \
+    cd "$WORKSPACE/apps/workbench" \
         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
 }
 do_test apps/workbench_integration workbench_integration
 
 
 test_workbench_benchmark() {
-    start_nginx_proxy_services \
-        && cd "$WORKSPACE/apps/workbench" \
+    cd "$WORKSPACE/apps/workbench" \
         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
 }
 do_test apps/workbench_benchmark workbench_benchmark
 
 test_workbench_profile() {
-    start_nginx_proxy_services \
-        && cd "$WORKSPACE/apps/workbench" \
+    cd "$WORKSPACE/apps/workbench" \
         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
 }
 do_test apps/workbench_profile workbench_profile