18874: Fix run-tests.sh test targets.

[arvados.git] / build / run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index d517e7e2c4d921bbc91d8edd31f1ecfcee9bb881..d6415e239d9f021581011596c9a74942d0b7a2f3 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -58,7 +58,7 @@ defaults to $HOME/arvados-api-server if that directory exists.
 
 More information and background:
 
-https://arvados.org/projects/arvados/wiki/Running_tests
+https://dev.arvados.org/projects/arvados/wiki/Running_tests
 
 Available tests:
 
@@ -69,6 +69,7 @@ apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
 cmd/arvados-client
+cmd/arvados-package
 cmd/arvados-server
 doc
 lib/cli
@@ -93,7 +94,6 @@ lib/pam
 lib/service
 services/api
 services/githttpd
-services/crunchstat
 services/dockercleaner
 services/fuse
 services/fuse:py3
@@ -105,6 +105,8 @@ services/keep-balance
 services/login-sync
 services/crunch-dispatch-local
 services/crunch-dispatch-slurm
+services/workbench2_units
+services/workbench2_integration
 services/ws
 sdk/cli
 sdk/python
@@ -149,7 +151,6 @@ VENVDIR=
 VENV3DIR=
 PYTHONPATH=
 GEMHOME=
-PERLINSTALLBASE=
 R_LIBS=
 export LANG=en_US.UTF-8
 
@@ -158,6 +159,8 @@ only_install=
 temp=
 temp_preserve=
 
+ignore_sigint=
+
 clear_temp() {
     if [[ -z "$temp" ]]; then
         # we did not even get as far as making a temp dir
@@ -229,14 +232,6 @@ sanity_checks() {
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
-    echo -n 'perl: '
-    perl -v | grep version \
-        || fatal "No perl. Try: apt-get install perl"
-    for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
-        echo -n "perl $mod: "
-        perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
-            || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
-    done
     echo -n 'gitolite: '
     which gitolite \
         || fatal "No gitolite. Try: apt-get install gitolite3"
@@ -267,7 +262,13 @@ sanity_checks() {
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
     echo -n 'geckodriver: '
-    geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver"
+    geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install"
+    echo -n 'singularity: '
+    singularity --version || fatal "No singularity. Try: arvados-server install"
+    echo -n 'docker client: '
+    docker --version || echo "No docker client. Try: arvados-server install"
+    echo -n 'docker server: '
+    docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install"
 
     if [[ "$NEED_SDK_R" = true ]]; then
       # R SDK stuff
@@ -473,6 +474,10 @@ stop_services() {
 }
 
 interrupt() {
+    if [[ -n "$ignore_sigint" ]]; then
+        echo >&2 "ignored SIGINT"
+        return
+    fi
     failures+=("($(basename $0) interrupted)")
     exit_cleanly
 }
@@ -608,7 +613,7 @@ initialize() {
     fi
 
     # Set up temporary install dirs (unless existing dirs were supplied)
-    for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
+    for tmpdir in VENV3DIR GOPATH GEMHOME R_LIBS
     do
         if [[ -z "${!tmpdir}" ]]; then
             eval "$tmpdir"="$temp/$tmpdir"
@@ -620,9 +625,6 @@ initialize() {
 
     rm -vf "${WORKSPACE}/tmp/*.log"
 
-    export PERLINSTALLBASE
-    export PERL5LIB="$PERLINSTALLBASE/lib/perl5${PERL5LIB:+:$PERL5LIB}"
-
     export R_LIBS
 
     export GOPATH
@@ -653,7 +655,7 @@ install_env() {
     . "$VENV3DIR/bin/activate"
 
     # Needed for run_test_server.py which is used by certain (non-Python) tests.
-    # pdoc3 needed to generate the Python SDK documentation.
+    # pdoc needed to generate the Python SDK documentation.
     (
         set -e
         "${VENV3DIR}/bin/pip3" install wheel
@@ -664,7 +666,7 @@ install_env() {
         "${VENV3DIR}/bin/pip3" install ciso8601
         "${VENV3DIR}/bin/pip3" install pycurl
         "${VENV3DIR}/bin/pip3" install ws4py
-        "${VENV3DIR}/bin/pip3" install pdoc3
+        "${VENV3DIR}/bin/pip3" install pdoc
         cd "$WORKSPACE/sdk/python"
         python3 setup.py install
     ) || fatal "installing PyYAML and sdk/python failed"
@@ -698,6 +700,9 @@ do_test() {
         apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
             suite=apps/workbench
             ;;
+        services/workbench2_units | services/workbench2_integration)
+            suite=services/workbench2
+            ;;
         *)
             suite="${1}"
             ;;
@@ -712,7 +717,7 @@ do_test() {
             stop_services
             check_arvados_config "$1"
             ;;
-        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker)
+        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker | services/workbench2_units | services/workbench2_integration)
             check_arvados_config "$1"
             # don't care whether services are running
             ;;
@@ -730,7 +735,7 @@ do_test() {
 
 go_ldflags() {
     version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}
-    echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}"
+    echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version} -s -w"
 }
 
 do_test_once() {
@@ -748,6 +753,10 @@ do_test_once() {
     then
         covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
         coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+        testflags=()
+        if [[ "$1" == "cmd/arvados-package" ]]; then
+            testflags+=("-timeout" "20m")
+        fi
         # We do "go install" here to catch compilation errors
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
@@ -758,11 +767,11 @@ do_test_once() {
         then
             # "go test -check.vv giturl" doesn't work, but this
             # does:
-            go test ${short:+-short} ${testargs[$1]}
+            go test ${short:+-short} ${testflags[@]} ${testargs[$1]}
         else
             # The above form gets verbose even when testargs is
             # empty, so use this form in such cases:
-            go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
+            go test ${short:+-short} ${testflags[@]} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
         fi
         result=${result:-$?}
         if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
@@ -911,17 +920,12 @@ install_sdk/R() {
   fi
 }
 
-install_sdk/perl() {
-    cd "$WORKSPACE/sdk/perl" \
-        && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
-        && make install INSTALLDIRS=perl
-}
-
 install_sdk/cli() {
     install_gem arvados-cli sdk/cli
 }
 
 install_services/login-sync() {
+    install_gem arvados sdk/ruby
     install_gem arvados-login-sync services/login-sync
 }
 
@@ -994,6 +998,11 @@ install_apps/workbench() {
         && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install
 }
 
+install_services/workbench2() {
+    cd "$WORKSPACE/services/workbench2" \
+        && make yarn-install ARVADOS_DIRECTORY="${WORKSPACE}"
+}
+
 test_doc() {
     (
         set -e
@@ -1044,6 +1053,14 @@ test_services/login-sync() {
         && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
 }
 
+test_services/workbench2_units() {
+    cd "$WORKSPACE/services/workbench2" && make unit-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
+}
+
+test_services/workbench2_integration() {
+    cd "$WORKSPACE/services/workbench2" && make integration-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
+}
+
 test_apps/workbench_units() {
     local TASK="test:units"
     cd "$WORKSPACE/apps/workbench" \
@@ -1079,7 +1096,6 @@ install_deps() {
     do_install env
     do_install cmd/arvados-server go
     do_install sdk/cli
-    do_install sdk/perl
     do_install sdk/python pip "${VENV3DIR}/bin/"
     do_install sdk/ruby
     do_install services/api
@@ -1092,7 +1108,6 @@ install_all() {
     do_install doc
     do_install sdk/ruby
     do_install sdk/R
-    do_install sdk/perl
     do_install sdk/cli
     do_install services/login-sync
     for p in "${pythonstuff[@]}"
@@ -1108,6 +1123,7 @@ install_all() {
     done
     do_install services/api
     do_install apps/workbench
+    do_install services/workbench2
 }
 
 test_all() {
@@ -1145,6 +1161,8 @@ test_all() {
     do_test apps/workbench_integration
     do_test apps/workbench_benchmark
     do_test apps/workbench_profile
+    do_test services/workbench2_units
+    do_test services/workbench2_integration
 }
 
 test_go() {
@@ -1216,6 +1234,7 @@ else
     setnextcmd
     HISTFILE="$WORKSPACE/tmp/.history"
     history -r
+    ignore_sigint=1
     while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
         history -s "$nextcmd"
         history -w