X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d6f2d88621fb4f70674e669fe92df23f3e866dc4..05c87aaf4572a6420a61e16495420e99ece2cef6:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 43c5769482..3cf96f3b4a 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -77,9 +77,14 @@ doc
 lib/cli
 lib/cmd
 lib/controller
+lib/controller/federation
+lib/controller/railsproxy
+lib/controller/router
+lib/controller/rpc
 lib/crunchstat
 lib/cloud
 lib/cloud/azure
+lib/cloud/cloudtest
 lib/dispatchcloud
 lib/dispatchcloud/container
 lib/dispatchcloud/scheduler
@@ -205,8 +210,8 @@ sanity_checks() {
     echo -n 'go: '
     go version \
         || fatal "No go binary. See http://golang.org/doc/install"
-    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
-        || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
+    [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \
+        || fatal "Go >= 1.12 required. See http://golang.org/doc/install"
     echo -n 'gcc: '
     gcc --version | egrep ^gcc \
         || fatal "No gcc. Try: apt-get install build-essential"
@@ -222,6 +227,8 @@ sanity_checks() {
     echo -n 'Python3 pyconfig.h: '
     find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
         || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+    which netstat \
+        || fatal "No netstat. Try: apt-get install net-tools"
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
@@ -251,12 +258,6 @@ sanity_checks() {
     echo -n 'libpq libpq-fe.h: '
     find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
         || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
-    echo -n 'services/api/config/database.yml: '
-    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-	    fatal "Please provide a database.yml file for the test suite"
-    else
-	    echo "OK"
-    fi
     echo -n 'postgresql: '
     psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
     echo -n 'phantomjs: '
@@ -571,8 +572,6 @@ setup_virtualenv() {
     else
         "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
     fi
-    # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
-    "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
 }
 
 initialize() {
@@ -612,6 +611,9 @@ initialize() {
     export R_LIBS
 
     export GOPATH
+    # Make sure our compiled binaries under test override anything
+    # else that might be in the environment.
+    export PATH=$GOPATH/bin:$PATH
 
     # Jenkins config requires that glob tmp/*.log match something. Ensure
     # that happens even if we don't end up running services that set up
@@ -621,11 +623,30 @@ initialize() {
 
     unset http_proxy https_proxy no_proxy
 
-
     # Note: this must be the last time we change PATH, otherwise rvm will
     # whine a lot.
     setup_ruby_environment
 
+    if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+	cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
+	export ARVADOS_CONFIG="$temp/test-config.yml"
+    else
+	if [[ -s /etc/arvados/config.yml ]] ; then
+	    python > "$temp/test-config.yml" <<EOF
+import yaml
+import json
+v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0]['PostgreSQL']
+v['Connection']['dbname'] = 'arvados_test'
+print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
+EOF
+	    export ARVADOS_CONFIG="$temp/test-config.yml"
+	else
+	    if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+		fatal "Please provide a database.yml file for the test suite"
+	    fi
+	fi
+    fi
+
     echo "PATH is $PATH"
 }
 
@@ -649,23 +670,8 @@ install_env() {
         ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
         go get -v github.com/kardianos/govendor
         cd "$GOPATH/src/git.curoverse.com/arvados.git"
-        if [[ -n "$short" ]]; then
-            go get -v -d ...
-            "$GOPATH/bin/govendor" sync
-        else
-            # Remove cached source dirs in workdir. Otherwise, they will
-            # not qualify as +missing or +external below, and we won't be
-            # able to detect that they're missing from vendor/vendor.json.
-            rm -rf vendor/*/
-            go get -v -d ...
-            "$GOPATH/bin/govendor" sync
-            [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
-                || fatal "vendor/vendor.json has unused or missing dependencies -- try:
-
-(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
-
-";
-        fi
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
     ) || fatal "Go setup failed"
 
     setup_virtualenv "$VENVDIR" --python python2.7
@@ -750,7 +756,7 @@ do_test() {
         services/api)
             stop_services
             ;;
-        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+        gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
             # don't care whether services are running
             ;;
         *)
@@ -782,7 +788,7 @@ do_test_once() {
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
         # compilation errors.
-        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
+        go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
             if [[ -n "${testargs[$1]}" ]]
         then
@@ -809,7 +815,10 @@ do_test_once() {
             tries=$((${tries}+1))
             # $3 can name a path directory for us to use, including trailing
             # slash; e.g., the bin/ subdirectory of a virtualenv.
-            "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+            if [[ -e "${3}activate" ]]; then
+                . "${3}activate"
+            fi
+            python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
             result=$?
             if [[ ${tries} < 3 && ${result} == 137 ]]
             then
@@ -848,7 +857,7 @@ do_install_once() {
         result=1
     elif [[ "$2" == "go" ]]
     then
-        go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
+        go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
     elif [[ "$2" == "pip" ]]
     then
         # $3 can name a path directory for us to use, including trailing
@@ -999,51 +1008,7 @@ pythonstuff=(
 )
 
 declare -a gostuff
-gostuff=(
-    cmd/arvados-client
-    cmd/arvados-server
-    lib/cli
-    lib/cmd
-    lib/controller
-    lib/crunchstat
-    lib/cloud
-    lib/cloud/azure
-    lib/cloud/ec2
-    lib/config
-    lib/dispatchcloud
-    lib/dispatchcloud/container
-    lib/dispatchcloud/scheduler
-    lib/dispatchcloud/ssh_executor
-    lib/dispatchcloud/worker
-    lib/service
-    sdk/go/arvados
-    sdk/go/arvadosclient
-    sdk/go/auth
-    sdk/go/blockdigest
-    sdk/go/dispatch
-    sdk/go/health
-    sdk/go/httpserver
-    sdk/go/manifest
-    sdk/go/asyncbuf
-    sdk/go/crunchrunner
-    sdk/go/stats
-    services/arv-git-httpd
-    services/crunchstat
-    services/health
-    services/keep-web
-    services/keepstore
-    sdk/go/keepclient
-    services/keep-balance
-    services/keepproxy
-    services/crunch-dispatch-local
-    services/crunch-dispatch-slurm
-    services/crunch-run
-    services/ws
-    tools/keep-block-check
-    tools/keep-exercise
-    tools/keep-rsync
-    tools/sync-groups
-)
+gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
 
 install_apps/workbench() {
     cd "$WORKSPACE/apps/workbench" \
@@ -1069,6 +1034,27 @@ test_gofmt() {
     [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
 }
 
+test_govendor() {
+    (
+        set -e
+        cd "$GOPATH/src/git.curoverse.com/arvados.git"
+        # Remove cached source dirs in workdir. Otherwise, they will
+        # not qualify as +missing or +external below, and we won't be
+        # able to detect that they're missing from vendor/vendor.json.
+        rm -rf vendor/*/
+        go get -v -d ...
+        "$GOPATH/bin/govendor" sync
+        if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then
+            echo >&2 "vendor/vendor.json has unused or missing dependencies -- try:
+
+(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
+
+"
+            return 1
+        fi
+    )
+}
+
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
@@ -1108,28 +1094,48 @@ test_services/nodemanager_integration() {
 }
 
 test_apps/workbench_units() {
+    local TASK="test:units"
+    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then
+        TASK="test"
+    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
+    local TASK="test:functionals"
+    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then
+        TASK="test"
+    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
+    local TASK="test:integration"
+    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then
+        TASK="test"
+    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
 }
 
 test_apps/workbench_benchmark() {
+    local TASK="test:benchmark"
+    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then
+        TASK="test"
+    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
 }
 
 test_apps/workbench_profile() {
+    local TASK="test:profile"
+    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then
+        TASK="test"
+    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]}
 }
 
 install_deps() {
@@ -1169,11 +1175,11 @@ install_all() {
             fi
         fi
     done
-    do_install services/api
     for g in "${gostuff[@]}"
     do
         do_install "$g" go
     done
+    do_install services/api
     do_install apps/workbench
 }
 
@@ -1189,6 +1195,7 @@ test_all() {
     fi
 
     do_test gofmt
+    do_test govendor
     do_test doc
     do_test sdk/ruby
     do_test sdk/R
@@ -1249,6 +1256,13 @@ for p in "${pythonstuff[@]}"; do
 done
 
 testfuncargs["sdk/cli"]="sdk/cli"
+testfuncargs["sdk/R"]="sdk/R"
+testfuncargs["sdk/java-v2"]="sdk/java-v2"
+testfuncargs["apps/workbench_units"]="apps/workbench_units"
+testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals"
+testfuncargs["apps/workbench_integration"]="apps/workbench_integration"
+testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark"
+testfuncargs["apps/workbench_profile"]="apps/workbench_profile"
 
 if [[ -z ${interactive} ]]; then
     install_all
@@ -1263,19 +1277,21 @@ else
             # assume emacs, or something, is offering a history buffer
             # and pre-populating the command will only cause trouble
             nextcmd=
-        elif [[ "$nextcmd" != "install deps" ]]; then
-            :
-        elif [[ -e "$VENVDIR/bin/activate" ]]; then
-            nextcmd="test lib/cmd"
-        else
+        elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
             nextcmd="install deps"
+        else
+            nextcmd=""
         fi
     }
     echo
     help_interactive
     nextcmd="install deps"
     setnextcmd
-    while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+    HISTFILE="$WORKSPACE/tmp/.history"
+    history -r
+    while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+        history -s "$nextcmd"
+        history -w
         read verb target opts <<<"${nextcmd}"
         target="${target%/}"
         target="${target/\/:/:}"