X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0dab89df8040f203a33bc1922df0ff893791def7..95c835b12a32a5fa6dc94799d6fea16e5b20080c:/build/run-tests.sh

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 0f996f77e9..b8d2081e6e 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -38,8 +38,6 @@ services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.r
                Restrict apiserver tests to the given file
 sdk/python_test="--test-suite tests.test_keep_locator"
                Restrict Python SDK tests to the given class
-apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
-               Restrict Workbench tests to the given file
 services/githttpd_test="-check.vv"
                Show all log messages, even when tests pass (also works
                with services/keepstore_test etc.)
@@ -58,17 +56,12 @@ defaults to $HOME/arvados-api-server if that directory exists.
 
 More information and background:
 
-https://arvados.org/projects/arvados/wiki/Running_tests
+https://dev.arvados.org/projects/arvados/wiki/Running_tests
 
 Available tests:
 
-apps/workbench (*)
-apps/workbench_units (*)
-apps/workbench_functionals (*)
-apps/workbench_integration (*)
-apps/workbench_benchmark
-apps/workbench_profile
 cmd/arvados-client
+cmd/arvados-package
 cmd/arvados-server
 doc
 lib/cli
@@ -93,7 +86,6 @@ lib/pam
 lib/service
 services/api
 services/githttpd
-services/crunchstat
 services/dockercleaner
 services/fuse
 services/fuse:py3
@@ -105,10 +97,13 @@ services/keep-balance
 services/login-sync
 services/crunch-dispatch-local
 services/crunch-dispatch-slurm
+services/workbench2_units
+services/workbench2_integration
 services/ws
 sdk/cli
 sdk/python
 sdk/python:py3
+sdk/ruby-google-api-client
 sdk/ruby
 sdk/go/arvados
 sdk/go/arvadosclient
@@ -132,9 +127,6 @@ tools/keep-exercise
 tools/keep-rsync
 tools/keep-block-check
 
-(*) apps/workbench is shorthand for apps/workbench_units +
-    apps/workbench_functionals + apps/workbench_integration
-
 EOF
 
 # First make sure to remove any ARVADOS_ variables from the calling
@@ -149,7 +141,6 @@ VENVDIR=
 VENV3DIR=
 PYTHONPATH=
 GEMHOME=
-PERLINSTALLBASE=
 R_LIBS=
 export LANG=en_US.UTF-8
 
@@ -182,10 +173,6 @@ fatal() {
 
 exit_cleanly() {
     trap - INT
-    if which create-plot-data-from-log.sh >/dev/null; then
-        create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
-    fi
-    rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
     stop_services
     rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
     report_outcomes
@@ -221,7 +208,8 @@ sanity_checks() {
     find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \
         || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev"
     echo -n 'virtualenv: '
-    python3 -m venv -h | egrep --max-count=1 . \
+    python3 -m venv --help | grep -q '^usage: venv ' \
+        && echo "venv module found" \
         || fatal "No virtualenv. Try: apt-get install python3-venv"
     echo -n 'Python3 pyconfig.h: '
     find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
@@ -231,20 +219,12 @@ sanity_checks() {
     echo -n 'nginx: '
     PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
         || fatal "No nginx. Try: apt-get install nginx"
-    echo -n 'perl: '
-    perl -v | grep version \
-        || fatal "No perl. Try: apt-get install perl"
-    for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
-        echo -n "perl $mod: "
-        perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
-            || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
-    done
     echo -n 'gitolite: '
     which gitolite \
         || fatal "No gitolite. Try: apt-get install gitolite3"
     echo -n 'npm: '
     npm --version \
-        || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/"
+        || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v12.22.12-linux-x64/bin/{node,npm} /usr/local/bin/"
     echo -n 'cadaver: '
     cadaver --version | grep -w cadaver \
           || fatal "No cadaver. Try: apt-get install cadaver"
@@ -262,14 +242,16 @@ sanity_checks() {
         || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev"
     echo -n 'postgresql: '
     psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
-    echo -n 'phantomjs: '
-    phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs"
     echo -n 'xvfb: '
     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
     echo -n 'graphviz: '
     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
-    echo -n 'geckodriver: '
-    geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver"
+    echo -n 'singularity: '
+    singularity --version || fatal "No singularity. Try: arvados-server install"
+    echo -n 'docker client: '
+    docker --version || echo "No docker client. Try: arvados-server install"
+    echo -n 'docker server: '
+    docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install"
 
     if [[ "$NEED_SDK_R" = true ]]; then
       # R SDK stuff
@@ -289,7 +271,7 @@ sanity_checks() {
 }
 
 rotate_logfile() {
-  # i.e.  rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+  # i.e.  rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
   # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
   if [[ -f "$1/$2" ]]; then
     THEDATE=`date +%Y%m%d%H%M%S`
@@ -302,7 +284,6 @@ declare -a failures
 declare -A skip
 declare -A only
 declare -A testargs
-skip[apps/workbench_profile]=1
 
 while [[ -n "$1" ]]
 do
@@ -592,7 +573,7 @@ setup_virtualenv() {
     elif [[ -n "$short" ]]; then
         return
     fi
-    "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
+    "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=68' 'pip>=20'
 }
 
 initialize() {
@@ -614,7 +595,7 @@ initialize() {
     fi
 
     # Set up temporary install dirs (unless existing dirs were supplied)
-    for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
+    for tmpdir in VENV3DIR GOPATH GEMHOME R_LIBS
     do
         if [[ -z "${!tmpdir}" ]]; then
             eval "$tmpdir"="$temp/$tmpdir"
@@ -626,9 +607,6 @@ initialize() {
 
     rm -vf "${WORKSPACE}/tmp/*.log"
 
-    export PERLINSTALLBASE
-    export PERL5LIB="$PERLINSTALLBASE/lib/perl5${PERL5LIB:+:$PERL5LIB}"
-
     export R_LIBS
 
     export GOPATH
@@ -658,22 +636,16 @@ install_env() {
     setup_virtualenv "$VENV3DIR"
     . "$VENV3DIR/bin/activate"
 
-    # Needed for run_test_server.py which is used by certain (non-Python) tests.
-    # pdoc3 needed to generate the Python SDK documentation.
-    (
-        set -e
-        "${VENV3DIR}/bin/pip3" install wheel
-        "${VENV3DIR}/bin/pip3" install PyYAML
-        "${VENV3DIR}/bin/pip3" install httplib2
-        "${VENV3DIR}/bin/pip3" install future
-        "${VENV3DIR}/bin/pip3" install google-api-python-client
-        "${VENV3DIR}/bin/pip3" install ciso8601
-        "${VENV3DIR}/bin/pip3" install pycurl
-        "${VENV3DIR}/bin/pip3" install ws4py
-        "${VENV3DIR}/bin/pip3" install pdoc3
-        cd "$WORKSPACE/sdk/python"
-        python3 setup.py install
-    ) || fatal "installing PyYAML and sdk/python failed"
+    # wheel modernizes the venv (as of early 2024) and makes it more closely
+    # match our package build environment.
+    # PyYAML is a test requirement used by run_test_server.py and needed for
+    # other, non-Python tests.
+    # pdoc is needed to build PySDK documentation.
+    # We run `setup.py build` first to generate _version.py.
+    pip install PyYAML pdoc wheel \
+        && env -C "$WORKSPACE/sdk/python" python3 setup.py build \
+        && pip install "$WORKSPACE/sdk/python" \
+        || fatal "installing Python SDK and related dependencies failed"
 }
 
 retry() {
@@ -701,8 +673,8 @@ retry() {
 
 do_test() {
     case "${1}" in
-        apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
-            suite=apps/workbench
+        services/workbench2_units | services/workbench2_integration)
+            suite=services/workbench2
             ;;
         *)
             suite="${1}"
@@ -718,7 +690,22 @@ do_test() {
             stop_services
             check_arvados_config "$1"
             ;;
-        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker)
+        gofmt \
+            | arvados_version.py \
+            | cmd/arvados-package \
+            | doc \
+            | lib/boot \
+            | lib/cli \
+            | lib/cloud/azure \
+            | lib/cloud/cloudtest \
+            | lib/cloud/ec2 \
+            | lib/cmd \
+            | lib/dispatchcloud/sshexecutor \
+            | lib/dispatchcloud/worker \
+            | lib/install \
+            | services/workbench2_integration \
+            | services/workbench2_units \
+            )
             check_arvados_config "$1"
             # don't care whether services are running
             ;;
@@ -736,7 +723,7 @@ do_test() {
 
 go_ldflags() {
     version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}
-    echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}"
+    echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version} -s -w"
 }
 
 do_test_once() {
@@ -754,6 +741,10 @@ do_test_once() {
     then
         covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
         coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+        testflags=()
+        if [[ "$1" == "cmd/arvados-package" ]]; then
+            testflags+=("-timeout" "20m")
+        fi
         # We do "go install" here to catch compilation errors
         # before trying "go test". Otherwise, coverage-reporting
         # mode makes Go show the wrong line numbers when reporting
@@ -764,11 +755,11 @@ do_test_once() {
         then
             # "go test -check.vv giturl" doesn't work, but this
             # does:
-            go test ${short:+-short} ${testargs[$1]}
+            go test ${short:+-short} ${testflags[@]} ${testargs[$1]}
         else
             # The above form gets verbose even when testargs is
             # empty, so use this form in such cases:
-            go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
+            go test ${short:+-short} ${testflags[@]} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
         fi
         result=${result:-$?}
         if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
@@ -910,6 +901,10 @@ install_sdk/ruby() {
     install_gem arvados sdk/ruby
 }
 
+install_sdk/ruby-google-api-client() {
+    install_gem arvados-google-api-client sdk/ruby-google-api-client
+}
+
 install_sdk/R() {
   if [[ "$NEED_SDK_R" = true ]]; then
     cd "$WORKSPACE/sdk/R" \
@@ -917,17 +912,13 @@ install_sdk/R() {
   fi
 }
 
-install_sdk/perl() {
-    cd "$WORKSPACE/sdk/perl" \
-        && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
-        && make install INSTALLDIRS=perl
-}
-
 install_sdk/cli() {
     install_gem arvados-cli sdk/cli
 }
 
 install_services/login-sync() {
+    install_gem arvados-google-api-client sdk/ruby-google-api-client
+    install_gem arvados sdk/ruby
     install_gem arvados-login-sync services/login-sync
 }
 
@@ -983,38 +974,56 @@ install_services/api() {
 
 declare -a pythonstuff
 pythonstuff=(
+    # The ordering of sdk/python, tools/crunchstat-summary, and
+    # sdk/cwl here is significant. See
+    # https://dev.arvados.org/issues/19744#note-26
     sdk/python:py3
+    tools/crunchstat-summary:py3
     sdk/cwl:py3
     services/dockercleaner:py3
     services/fuse:py3
-    tools/crunchstat-summary:py3
 )
 
 declare -a gostuff
 gostuff=($(cd "$WORKSPACE" && git ls-files | grep '\.go$' | sed -e 's/\/[^\/]*$//' | sort -u))
 
-install_apps/workbench() {
-    cd "$WORKSPACE/apps/workbench" \
-        && mkdir -p tmp/cache \
-        && RAILS_ENV=test bundle_install_trylocal \
-        && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install
+install_services/workbench2() {
+    cd "$WORKSPACE/services/workbench2" \
+        && make yarn-install ARVADOS_DIRECTORY="${WORKSPACE}"
 }
 
 test_doc() {
-    (
-        set -e
-        cd "$WORKSPACE/doc"
-        ARVADOS_API_HOST=pirca.arvadosapi.com
-        # Make sure python-epydoc is installed or the next line won't
-        # do much good!
-        PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
-    )
+    local arvados_api_host=pirca.arvadosapi.com && \
+        env -C "$WORKSPACE/doc" \
+        "$bundle" exec rake linkchecker \
+        arvados_api_host="$arvados_api_host" \
+        arvados_workbench_host="https://workbench.$arvados_api_host" \
+        baseurl="file://$WORKSPACE/doc/.site/" \
+        ${testargs[doc]}
 }
 
 test_gofmt() {
     cd "$WORKSPACE" || return 1
     dirs=$(ls -d */ | egrep -v 'vendor|tmp')
     [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+    go vet -composites=false ./...
+}
+
+test_arvados_version.py() {
+    local orig_fn=""
+    local fail_count=0
+    while read -d "" fn; do
+        if [[ -z "$orig_fn" ]]; then
+            orig_fn="$fn"
+        elif ! cmp "$orig_fn" "$fn"; then
+            fail_count=$(( $fail_count + 1 ))
+            printf "FAIL: %s and %s are not identical\n" "$orig_fn" "$fn"
+        fi
+    done < <(git -C "$WORKSPACE" ls-files -z | grep -z '/arvados_version\.py$')
+    case "$orig_fn" in
+        "") return 66 ;;  # EX_NOINPUT
+        *) return "$fail_count" ;;
+    esac
 }
 
 test_services/api() {
@@ -1028,6 +1037,11 @@ test_sdk/ruby() {
         && "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
 }
 
+test_sdk/ruby-google-api-client() {
+    echo "*** note \`test sdk/ruby-google-api-client\` does not actually run any tests, see https://dev.arvados.org/issues/20993 ***"
+    true
+}
+
 test_sdk/R() {
   if [[ "$NEED_SDK_R" = true ]]; then
     cd "$WORKSPACE/sdk/R" \
@@ -1050,45 +1064,26 @@ test_services/login-sync() {
         && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
 }
 
-test_apps/workbench_units() {
-    local TASK="test:units"
-    cd "$WORKSPACE/apps/workbench" \
-        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+test_services/workbench2_units() {
+    cd "$WORKSPACE/services/workbench2" && make unit-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
 }
 
-test_apps/workbench_functionals() {
-    local TASK="test:functionals"
-    cd "$WORKSPACE/apps/workbench" \
-        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
-}
-
-test_apps/workbench_integration() {
-    local TASK="test:integration"
-    cd "$WORKSPACE/apps/workbench" \
-        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
-}
-
-test_apps/workbench_benchmark() {
-    local TASK="test:benchmark"
-    cd "$WORKSPACE/apps/workbench" \
-        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
-}
-
-test_apps/workbench_profile() {
-    local TASK="test:profile"
-    cd "$WORKSPACE/apps/workbench" \
-        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_profile]}
+test_services/workbench2_integration() {
+    cd "$WORKSPACE/services/workbench2" && make integration-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
 }
 
 install_deps() {
     # Install parts needed by test suites
     do_install env
     do_install cmd/arvados-server go
-    do_install sdk/cli
-    do_install sdk/perl
-    do_install sdk/python pip "${VENV3DIR}/bin/"
+    do_install tools/crunchstat-summary pip "${VENV3DIR}/bin/"
+    do_install sdk/ruby-google-api-client
     do_install sdk/ruby
+    do_install sdk/cli
     do_install services/api
+    # lib/controller integration tests depend on arv-mount to run
+    # containers.
+    do_install services/fuse pip "${VENV3DIR}/bin/"
     do_install services/keepproxy go
     do_install services/keep-web go
 }
@@ -1096,9 +1091,9 @@ install_deps() {
 install_all() {
     do_install env
     do_install doc
+    do_install sdk/ruby-google-api-client
     do_install sdk/ruby
     do_install sdk/R
-    do_install sdk/perl
     do_install sdk/cli
     do_install services/login-sync
     for p in "${pythonstuff[@]}"
@@ -1113,7 +1108,7 @@ install_all() {
         do_install "$g" go
     done
     do_install services/api
-    do_install apps/workbench
+    do_install services/workbench2
 }
 
 test_all() {
@@ -1128,7 +1123,9 @@ test_all() {
     fi
 
     do_test gofmt
+    do_test arvados_version.py
     do_test doc
+    do_test sdk/ruby-google-api-client
     do_test sdk/ruby
     do_test sdk/R
     do_test sdk/cli
@@ -1146,11 +1143,8 @@ test_all() {
     do
         do_test "$g" go
     done
-    do_test apps/workbench_units
-    do_test apps/workbench_functionals
-    do_test apps/workbench_integration
-    do_test apps/workbench_benchmark
-    do_test apps/workbench_profile
+    do_test services/workbench2_units
+    do_test services/workbench2_integration
 }
 
 test_go() {
@@ -1191,11 +1185,6 @@ done
 testfuncargs["sdk/cli"]="sdk/cli"
 testfuncargs["sdk/R"]="sdk/R"
 testfuncargs["sdk/java-v2"]="sdk/java-v2"
-testfuncargs["apps/workbench_units"]="apps/workbench_units"
-testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals"
-testfuncargs["apps/workbench_integration"]="apps/workbench_integration"
-testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark"
-testfuncargs["apps/workbench_profile"]="apps/workbench_profile"
 
 if [[ -z ${interactive} ]]; then
     install_all