apps/workbench_integration (*)
apps/workbench_benchmark
apps/workbench_profile
+cmd/arvados-client
doc
+lib/cli
+lib/cmd
+lib/crunchstat
+lib/dispatchcloud
services/api
services/arv-git-httpd
services/crunchstat
sdk/go/stats
sdk/go/crunchrunner
sdk/cwl
-tools/arv-sync-groups
+sdk/R
+tools/sync-groups
tools/crunchstat-summary
tools/keep-exercise
tools/keep-rsync
PYTHONPATH=
GEMHOME=
PERLINSTALLBASE=
+R_LIBS=
short=
only_install=
which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
echo -n 'graphviz: '
dot -V || fatal "No graphviz. Try: apt-get install graphviz"
+
+ if [[ "$NEED_SDK_R" = true ]]; then
+ # R SDK stuff
+ echo -n 'R: '
+ which Rscript || fatal "No Rscript. Try: apt-get install r-base"
+ echo -n 'testthat: '
+ Rscript -e "library('testthat')" || fatal "No testthat. Try: apt-get install r-cran-testthat"
+ # needed for roxygen2, needed for devtools, needed for R sdk
+ pkg-config --exists libxml-2.0 || fatal "No libxml2. Try: apt-get install libxml2-dev"
+ # needed for pkgdown, builds R SDK doc pages
+ which pandoc || fatal "No pandoc. Try: apt-get install pandoc"
+ fi
}
rotate_logfile() {
esac
done
-start_api() {
- echo 'Starting API server...'
+# R SDK installation is very slow (~360s in a clean environment) and only
+# required when testing it. Skip that step if it is not needed.
+NEED_SDK_R=true
+
+if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then
+ NEED_SDK_R=false
+fi
+
+if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then
+ NEED_SDK_R=false
+fi
+
+start_services() {
+ echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
- && python sdk/python/tests/run_test_server.py start_ws \
- && python sdk/python/tests/run_test_server.py start_nginx \
- && (env | egrep ^ARVADOS)
-}
-
-start_nginx_proxy_services() {
- echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
- cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
&& python sdk/python/tests/run_test_server.py start_nginx \
- && export ARVADOS_TEST_PROXY_SERVICES=1
+ && (env | egrep ^ARVADOS)
}
stop_services() {
- if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
- unset ARVADOS_TEST_PROXY_SERVICES
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop_keep-web \
- && python sdk/python/tests/run_test_server.py stop_keep_proxy
- fi
- if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
- unset ARVADOS_TEST_API_HOST
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop
+ if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then
+ return
fi
+ unset ARVADOS_TEST_API_HOST
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_ws \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+ && python sdk/python/tests/run_test_server.py stop
}
interrupt() {
fi
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS
do
if [[ -z "${!tmpdir}" ]]; then
eval "$tmpdir"="$temp/$tmpdir"
fi
if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then
# pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy
- "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8'
+ "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8'
else
- "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7'
+ "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
fi
# ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
+ "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
}
export PERLINSTALLBASE
export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+export R_LIBS
+
export GOPATH
mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+ "$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -d "$d" ]] && rmdir "$d"
+ [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
|| fatal "symlink failed"
go get -v github.com/kardianos/govendor \
|| fatal "govendor install failed"
+cd "$GOPATH/src/git.curoverse.com/arvados.git" \
+ || fatal
+# Remove cached source dirs in workdir. Otherwise, they won't qualify
+# as +missing or +external below, and we won't be able to detect that
+# they're missing from vendor/vendor.json.
+rm -r vendor/*/
+go get -v -d ...
+"$GOPATH/bin/govendor" sync \
+ || fatal "govendor sync failed"
+[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+ || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+* govendor remove +unused
+* govendor add +missing +external
+"
+cd "$WORKSPACE"
+
setup_virtualenv "$VENVDIR" --python python2.7
. "$VENVDIR/bin/activate"
# Needed for run_test_server.py which is used by certain (non-Python) tests.
pip freeze 2>/dev/null | egrep ^PyYAML= \
- || pip install PyYAML >/dev/null \
+ || pip install --no-cache-dir PyYAML >/dev/null \
|| fatal "pip install PyYAML failed"
-# Preinstall forked version of libcloud, because nodemanager "pip install"
+# Preinstall libcloud, because nodemanager "pip install"
# won't pick it up by default.
pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
- || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
+ || pip install --pre --ignore-installed --no-cache-dir apache-libcloud>=$LIBCLOUD_PIN >/dev/null \
|| fatal "pip install apache-libcloud failed"
# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks.
}
do_install() {
- if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
- retry do_install_once ${@}
- else
- title "Skipping $1 install"
- fi
+ skipit=false
+
+ if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+ retry do_install_once ${@}
+ else
+ skipit=true
+ fi
+
+ if [[ "$skipit" = true ]]; then
+ title "Skipping $1 install"
+ fi
}
do_install_once() {
cd "$WORKSPACE/$1" \
&& "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
- && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
- && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
}
do_install sdk/ruby ruby_sdk
+install_R_sdk() {
+ if [[ "$NEED_SDK_R" = true ]]; then
+ cd "$WORKSPACE/sdk/R" \
+ && Rscript --vanilla install_deps.R
+ fi
+}
+do_install sdk/R R_sdk
+
install_perl_sdk() {
cd "$WORKSPACE/sdk/perl" \
&& perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
mkdir -p "$WORKSPACE/services/api/tmp/pids"
cert="$WORKSPACE/services/api/tmp/self-signed"
- if ! [[ -e "$cert.key" ]]; then
- dir="$WORKSPACE/services/api/tmp"
- openssl req -new -x509 -nodes -out "$cert.pem" -keyout "$cert.key" -days 3650 -subj /CN=0.0.0.0 -extfile <(printf 'subjectAltName=DNS:127.0.0.1,DNS:localhost,DNS:::1')
+ if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
+ (
+ dir="$WORKSPACE/services/api/tmp"
+ set -ex
+ openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
+ openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
+ ) || return 1
fi
+ cd "$WORKSPACE/services/api" \
+ && rm -rf tmp/git \
+ && mkdir -p tmp/git \
+ && cd tmp/git \
+ && tar xf ../../test/test.git.tar \
+ && mkdir -p internal.git \
+ && git --git-dir internal.git init \
+ || return 1
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
}
do_install services/api apiserver
-cd "$GOPATH/src/git.curoverse.com/arvados.git" && \
- "$GOPATH/bin/govendor" sync -v || \
- fatal "govendor sync failed"
declare -a gostuff
gostuff=(
+ cmd/arvados-client
+ lib/cli
+ lib/cmd
+ lib/crunchstat
+ lib/dispatchcloud
sdk/go/arvados
sdk/go/arvadosclient
sdk/go/blockdigest
sdk/go/asyncbuf
sdk/go/crunchrunner
sdk/go/stats
- lib/crunchstat
services/arv-git-httpd
services/crunchstat
services/health
tools/keep-block-check
tools/keep-exercise
tools/keep-rsync
- tools/arv-sync-groups
+ tools/sync-groups
)
for g in "${gostuff[@]}"
do
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_services || { stop_services; fatal "start_services"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
}
do_test sdk/ruby ruby_sdk
+test_R_sdk() {
+ if [[ "$NEED_SDK_R" = true ]]; then
+ cd "$WORKSPACE/sdk/R" \
+ && Rscript --vanilla run_test.R
+ fi
+}
+
+do_test sdk/R R_sdk
+
test_cli() {
cd "$WORKSPACE/sdk/cli" \
&& mkdir -p /tmp/keep \
done
test_workbench_units() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_units workbench_units
test_workbench_functionals() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_functionals workbench_functionals
test_workbench_integration() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_integration workbench_integration
test_workbench_benchmark() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
}
do_test apps/workbench_benchmark workbench_benchmark
test_workbench_profile() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
do_test apps/workbench_profile workbench_profile
projects / arvados.git / blobdiff