Restrict Python SDK tests to the given class
apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
Restrict Workbench tests to the given file
-services/githttp_test="-check.vv"
+services/githttpd_test="-check.vv"
Show all log messages, even when tests pass (also works
with services/keepstore_test etc.)
ARVADOS_DEBUG=1
More information and background:
-https://arvados.org/projects/arvados/wiki/Running_tests
+https://dev.arvados.org/projects/arvados/wiki/Running_tests
Available tests:
apps/workbench_benchmark
apps/workbench_profile
cmd/arvados-client
+cmd/arvados-package
cmd/arvados-server
doc
lib/cli
lib/pam
lib/service
services/api
-services/githttp
-services/crunchstat
+services/githttpd
services/dockercleaner
services/fuse
services/fuse:py3
services/login-sync
services/crunch-dispatch-local
services/crunch-dispatch-slurm
+services/workbench2_units
+services/workbench2_integration
services/ws
sdk/cli
sdk/python
sdk/python:py3
+sdk/ruby-google-api-client
sdk/ruby
sdk/go/arvados
sdk/go/arvadosclient
VENV3DIR=
PYTHONPATH=
GEMHOME=
-PERLINSTALLBASE=
R_LIBS=
export LANG=en_US.UTF-8
temp=
temp_preserve=
+ignore_sigint=
+
clear_temp() {
if [[ -z "$temp" ]]; then
# we did not even get as far as making a temp dir
echo -n 'nginx: '
PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
|| fatal "No nginx. Try: apt-get install nginx"
- echo -n 'perl: '
- perl -v | grep version \
- || fatal "No perl. Try: apt-get install perl"
- for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
- echo -n "perl $mod: "
- perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
- || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
- done
echo -n 'gitolite: '
which gitolite \
|| fatal "No gitolite. Try: apt-get install gitolite3"
echo -n 'graphviz: '
dot -V || fatal "No graphviz. Try: apt-get install graphviz"
echo -n 'geckodriver: '
- geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: wget -O- https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz | sudo tar -C /usr/local/bin -xzf - geckodriver"
+ geckodriver --version | grep ^geckodriver || echo "No geckodriver. Try: arvados-server install"
+ echo -n 'singularity: '
+ singularity --version || fatal "No singularity. Try: arvados-server install"
+ echo -n 'docker client: '
+ docker --version || echo "No docker client. Try: arvados-server install"
+ echo -n 'docker server: '
+ docker info --format='{{.ServerVersion}}' || echo "No docker server. Try: arvados-server install"
if [[ "$NEED_SDK_R" = true ]]; then
# R SDK stuff
return 0
fi
. "$VENV3DIR/bin/activate"
- echo 'Starting API, controller, keepproxy, keep-web, githttp, ws, and nginx ssl proxy...'
+ echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& python3 sdk/python/tests/run_test_server.py start_keep-web \
&& checkpidfile keep-web \
&& checkhealth WebDAV \
- && python3 sdk/python/tests/run_test_server.py start_githttp \
- && checkpidfile githttp \
+ && python3 sdk/python/tests/run_test_server.py start_githttpd \
+ && checkpidfile githttpd \
&& checkhealth GitHTTP \
&& python3 sdk/python/tests/run_test_server.py start_ws \
&& checkpidfile ws \
. "$VENV3DIR/bin/activate" || return
cd "$WORKSPACE" \
&& python3 sdk/python/tests/run_test_server.py stop_nginx \
- && python3 sdk/python/tests/run_test_server.py stop_githttp \
+ && python3 sdk/python/tests/run_test_server.py stop_githttpd \
&& python3 sdk/python/tests/run_test_server.py stop_ws \
&& python3 sdk/python/tests/run_test_server.py stop_keep-web \
&& python3 sdk/python/tests/run_test_server.py stop_keep_proxy \
}
interrupt() {
+ if [[ -n "$ignore_sigint" ]]; then
+ echo >&2 "ignored SIGINT"
+ return
+ fi
failures+=("($(basename $0) interrupted)")
exit_cleanly
}
elif [[ -n "$short" ]]; then
return
fi
- "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
+ "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=68' 'pip>=20'
}
initialize() {
fi
# Set up temporary install dirs (unless existing dirs were supplied)
- for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
+ for tmpdir in VENV3DIR GOPATH GEMHOME R_LIBS
do
if [[ -z "${!tmpdir}" ]]; then
eval "$tmpdir"="$temp/$tmpdir"
rm -vf "${WORKSPACE}/tmp/*.log"
- export PERLINSTALLBASE
- export PERL5LIB="$PERLINSTALLBASE/lib/perl5${PERL5LIB:+:$PERL5LIB}"
-
export R_LIBS
export GOPATH
setup_virtualenv "$VENV3DIR"
. "$VENV3DIR/bin/activate"
- # Needed for run_test_server.py which is used by certain (non-Python) tests.
- # pdoc3 needed to generate the Python SDK documentation.
- (
- set -e
- "${VENV3DIR}/bin/pip3" install wheel
- "${VENV3DIR}/bin/pip3" install PyYAML
- "${VENV3DIR}/bin/pip3" install httplib2
- "${VENV3DIR}/bin/pip3" install future
- "${VENV3DIR}/bin/pip3" install google-api-python-client
- "${VENV3DIR}/bin/pip3" install ciso8601
- "${VENV3DIR}/bin/pip3" install pycurl
- "${VENV3DIR}/bin/pip3" install ws4py
- "${VENV3DIR}/bin/pip3" install pdoc3
- cd "$WORKSPACE/sdk/python"
- python3 setup.py install
- ) || fatal "installing PyYAML and sdk/python failed"
+ # PyYAML is a test requirement used by run_test_server.py and needed for
+ # other, non-Python tests.
+ # pdoc is needed to build PySDK documentation.
+ # We run `setup.py build` first to generate _version.py.
+ env -C "$WORKSPACE/sdk/python" python3 setup.py build \
+ && python3 -m pip install "$WORKSPACE/sdk/python" \
+ && python3 -m pip install PyYAML pdoc \
+ || fatal "installing Python SDK and related dependencies failed"
}
retry() {
apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
suite=apps/workbench
;;
+ services/workbench2_units | services/workbench2_integration)
+ suite=services/workbench2
+ ;;
*)
suite="${1}"
;;
stop_services
check_arvados_config "$1"
;;
- gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/sshexecutor | lib/dispatchcloud/worker)
+ gofmt \
+ | cmd/arvados-package \
+ | doc \
+ | lib/boot \
+ | lib/cli \
+ | lib/cloud/azure \
+ | lib/cloud/cloudtest \
+ | lib/cloud/ec2 \
+ | lib/cmd \
+ | lib/dispatchcloud/sshexecutor \
+ | lib/dispatchcloud/worker \
+ | lib/install \
+ | services/workbench2_integration \
+ | services/workbench2_units \
+ )
check_arvados_config "$1"
# don't care whether services are running
;;
go_ldflags() {
version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}
- echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}"
+ echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version} -s -w"
}
do_test_once() {
then
covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ testflags=()
+ if [[ "$1" == "cmd/arvados-package" ]]; then
+ testflags+=("-timeout" "20m")
+ fi
# We do "go install" here to catch compilation errors
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
then
# "go test -check.vv giturl" doesn't work, but this
# does:
- go test ${short:+-short} ${testargs[$1]}
+ go test ${short:+-short} ${testflags[@]} ${testargs[$1]}
else
# The above form gets verbose even when testargs is
# empty, so use this form in such cases:
- go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
+ go test ${short:+-short} ${testflags[@]} ${coverflags[@]} "git.arvados.org/arvados.git/$1"
fi
result=${result:-$?}
if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
install_gem arvados sdk/ruby
}
+install_sdk/ruby-google-api-client() {
+ install_gem arvados-google-api-client sdk/ruby-google-api-client
+}
+
install_sdk/R() {
if [[ "$NEED_SDK_R" = true ]]; then
cd "$WORKSPACE/sdk/R" \
fi
}
-install_sdk/perl() {
- cd "$WORKSPACE/sdk/perl" \
- && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
- && make install INSTALLDIRS=perl
-}
-
install_sdk/cli() {
install_gem arvados-cli sdk/cli
}
install_services/login-sync() {
+ install_gem arvados-google-api-client sdk/ruby-google-api-client
+ install_gem arvados sdk/ruby
install_gem arvados-login-sync services/login-sync
}
&& RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install
}
+install_services/workbench2() {
+ cd "$WORKSPACE/services/workbench2" \
+ && make yarn-install ARVADOS_DIRECTORY="${WORKSPACE}"
+}
+
test_doc() {
(
set -e
&& "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
}
+test_sdk/ruby-google-api-client() {
+ echo "*** note \`test sdk/ruby-google-api-client\` does not actually run any tests, see https://dev.arvados.org/issues/20993 ***"
+ true
+}
+
test_sdk/R() {
if [[ "$NEED_SDK_R" = true ]]; then
cd "$WORKSPACE/sdk/R" \
&& "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
}
+test_services/workbench2_units() {
+ cd "$WORKSPACE/services/workbench2" && make unit-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
+}
+
+test_services/workbench2_integration() {
+ cd "$WORKSPACE/services/workbench2" && make integration-tests ARVADOS_DIRECTORY="${WORKSPACE}" WORKSPACE="$(pwd)" ${testargs[services/workbench2]}
+}
+
test_apps/workbench_units() {
local TASK="test:units"
cd "$WORKSPACE/apps/workbench" \
do_install env
do_install cmd/arvados-server go
do_install sdk/cli
- do_install sdk/perl
do_install sdk/python pip "${VENV3DIR}/bin/"
+ do_install sdk/ruby-google-api-client
do_install sdk/ruby
do_install services/api
do_install services/keepproxy go
install_all() {
do_install env
do_install doc
+ do_install sdk/ruby-google-api-client
do_install sdk/ruby
do_install sdk/R
- do_install sdk/perl
do_install sdk/cli
do_install services/login-sync
for p in "${pythonstuff[@]}"
done
do_install services/api
do_install apps/workbench
+ do_install services/workbench2
}
test_all() {
do_test gofmt
do_test doc
+ do_test sdk/ruby-google-api-client
do_test sdk/ruby
do_test sdk/R
do_test sdk/cli
do_test apps/workbench_integration
do_test apps/workbench_benchmark
do_test apps/workbench_profile
+ do_test services/workbench2_units
+ do_test services/workbench2_integration
}
test_go() {
setnextcmd
HISTFILE="$WORKSPACE/tmp/.history"
history -r
+ ignore_sigint=1
while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
history -s "$nextcmd"
history -w
projects / arvados.git / blobdiff