lib/cli
lib/cmd
lib/controller
+ lib/controller/federation
+ lib/controller/railsproxy
+ lib/controller/router
+ lib/controller/rpc
lib/crunchstat
lib/cloud
lib/cloud/azure
+ lib/cloud/cloudtest
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
services/crunchstat
services/dockercleaner
services/fuse
+services/fuse:py3
services/health
services/keep-web
services/keepproxy
echo -n 'go: '
go version \
|| fatal "No go binary. See http://golang.org/doc/install"
- [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
- || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
+ [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \
+ || fatal "Go >= 1.12 required. See http://golang.org/doc/install"
echo -n 'gcc: '
gcc --version | egrep ^gcc \
|| fatal "No gcc. Try: apt-get install build-essential"
echo -n 'Python3 pyconfig.h: '
find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
|| fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+ which netstat \
+ || fatal "No netstat. Try: apt-get install net-tools"
echo -n 'nginx: '
PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
|| fatal "No nginx. Try: apt-get install nginx"
echo -n 'libpq libpq-fe.h: '
find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
|| fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
- echo -n 'services/api/config/database.yml: '
- if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
- fatal "Please provide a database.yml file for the test suite"
- else
- echo "OK"
- fi
echo -n 'postgresql: '
psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
echo -n 'phantomjs: '
else
"$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
fi
- # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
}
initialize() {
export R_LIBS
export GOPATH
+ # Make sure our compiled binaries under test override anything
+ # else that might be in the environment.
+ export PATH=$GOPATH/bin:$PATH
# Jenkins config requires that glob tmp/*.log match something. Ensure
# that happens even if we don't end up running services that set up
unset http_proxy https_proxy no_proxy
-
# Note: this must be the last time we change PATH, otherwise rvm will
# whine a lot.
setup_ruby_environment
+ if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+ cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
+ export ARVADOS_CONFIG="$temp/test-config.yml"
+ else
+ if [[ -s /etc/arvados/config.yml ]] ; then
+ python > "$temp/test-config.yml" <<EOF
+ import yaml
+ import json
+ v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0]['PostgreSQL']
+ v['Connection']['dbname'] = 'arvados_test'
+ print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
+ EOF
+ export ARVADOS_CONFIG="$temp/test-config.yml"
+ else
+ if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+ fatal "Please provide a database.yml file for the test suite"
+ fi
+ fi
+ fi
+
echo "PATH is $PATH"
}
services/api)
stop_services
;;
- gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
# don't care whether services are running
;;
*)
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
# compilation errors.
- go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
+ go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
if [[ -n "${testargs[$1]}" ]]
then
tries=$((${tries}+1))
# $3 can name a path directory for us to use, including trailing
# slash; e.g., the bin/ subdirectory of a virtualenv.
- "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+ if [[ -e "${3}activate" ]]; then
+ . "${3}activate"
+ fi
+ python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
result=$?
if [[ ${tries} < 3 && ${result} == 137 ]]
then
result=1
elif [[ "$2" == "go" ]]
then
- go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
+ go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
# $3 can name a path directory for us to use, including trailing
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
+ services/fuse:py3
services/nodemanager
tools/crunchstat-summary
tools/crunchstat-summary:py3
)
declare -a gostuff
- gostuff=(
- cmd/arvados-client
- cmd/arvados-server
- lib/cli
- lib/cmd
- lib/controller
- lib/crunchstat
- lib/cloud
- lib/cloud/azure
- lib/cloud/ec2
- lib/config
- lib/dispatchcloud
- lib/dispatchcloud/container
- lib/dispatchcloud/scheduler
- lib/dispatchcloud/ssh_executor
- lib/dispatchcloud/worker
- lib/service
- sdk/go/arvados
- sdk/go/arvadosclient
- sdk/go/auth
- sdk/go/blockdigest
- sdk/go/dispatch
- sdk/go/health
- sdk/go/httpserver
- sdk/go/manifest
- sdk/go/asyncbuf
- sdk/go/crunchrunner
- sdk/go/stats
- services/arv-git-httpd
- services/crunchstat
- services/health
- services/keep-web
- services/keepstore
- sdk/go/keepclient
- services/keep-balance
- services/keepproxy
- services/crunch-dispatch-local
- services/crunch-dispatch-slurm
- services/crunch-run
- services/ws
- tools/keep-block-check
- tools/keep-exercise
- tools/keep-rsync
- tools/sync-groups
- )
+ gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
install_apps/workbench() {
cd "$WORKSPACE/apps/workbench" \
}
test_apps/workbench_units() {
+ local TASK="test:units"
+ if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then
+ TASK="test"
+ fi
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
}
test_apps/workbench_functionals() {
+ local TASK="test:functionals"
+ if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then
+ TASK="test"
+ fi
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
}
test_apps/workbench_integration() {
+ local TASK="test:integration"
+ if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then
+ TASK="test"
+ fi
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
}
test_apps/workbench_benchmark() {
+ local TASK="test:benchmark"
+ if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then
+ TASK="test"
+ fi
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
}
test_apps/workbench_profile() {
+ local TASK="test:profile"
+ if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then
+ TASK="test"
+ fi
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]}
}
install_deps() {
fi
fi
done
- do_install services/api
for g in "${gostuff[@]}"
do
do_install "$g" go
done
+ do_install services/api
do_install apps/workbench
}
done
testfuncargs["sdk/cli"]="sdk/cli"
+ testfuncargs["sdk/R"]="sdk/R"
+ testfuncargs["sdk/java-v2"]="sdk/java-v2"
+ testfuncargs["apps/workbench_units"]="apps/workbench_units"
+ testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals"
+ testfuncargs["apps/workbench_integration"]="apps/workbench_integration"
+ testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark"
+ testfuncargs["apps/workbench_profile"]="apps/workbench_profile"
if [[ -z ${interactive} ]]; then
install_all
# assume emacs, or something, is offering a history buffer
# and pre-populating the command will only cause trouble
nextcmd=
- elif [[ "$nextcmd" != "install deps" ]]; then
- :
- elif [[ -e "$VENVDIR/bin/activate" ]]; then
- nextcmd="test lib/cmd"
- else
+ elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
nextcmd="install deps"
+ else
+ nextcmd=""
fi
}
echo
help_interactive
nextcmd="install deps"
setnextcmd
- while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+ HISTFILE="$WORKSPACE/tmp/.history"
+ history -r
+ while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+ history -s "$nextcmd"
+ history -w
read verb target opts <<<"${nextcmd}"
target="${target%/}"
target="${target/\/:/:}"
in seconds), print a warning on stderr before returning.
"""
try:
- subprocess.check_output(['which', 'lsof'])
+ subprocess.check_output(['which', 'netstat'])
except subprocess.CalledProcessError:
- print("WARNING: No `lsof` -- cannot wait for port to listen. "+
+ print("WARNING: No `netstat` -- cannot wait for port to listen. "+
"Sleeping 0.5 and hoping for the best.",
file=sys.stderr)
time.sleep(0.5)
return
deadline = time.time() + timeout
while time.time() < deadline:
- try:
- subprocess.check_output(
- ['lsof', '-t', '-i', 'tcp:'+str(port)])
- except subprocess.CalledProcessError:
- time.sleep(0.1)
- continue
- return True
+ if re.search(r'\ntcp.*:'+str(port)+' .* LISTEN *\n', subprocess.check_output(['netstat', '-Wln']).decode()):
+ return True
+ time.sleep(0.1)
if warn:
print(
"WARNING: Nothing is listening on port {} (waited {} seconds).".
f.write("""
Clusters:
zzzzz:
+ EnableBetaController14287: {beta14287}
ManagementToken: e687950a23c3a9bceec28c6223a06c79
API:
RequestTimeout: 30s
+ Logging:
+ Level: "{loglevel}"
+ HTTPRequestTimeout: 30s
PostgreSQL:
ConnectionPool: 32
Connection:
InternalURLs:
"https://localhost:{railsport}": {{}}
""".format(
+ beta14287=('true' if '14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '') else 'false'),
+ loglevel=('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
dbhost=_dbconfig('host'),
- dbname=_dbconfig('database'),
- dbuser=_dbconfig('username'),
+ dbname=_dbconfig('dbname'),
+ dbuser=_dbconfig('user'),
dbpass=_dbconfig('password'),
controllerport=port,
railsport=rails_api_port,
port,
('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
_dbconfig('host'),
- _dbconfig('database'),
- _dbconfig('username'),
+ _dbconfig('dbname'),
+ _dbconfig('user'),
_dbconfig('password')))
logf = open(_logfilename('ws'), 'a')
ws = subprocess.Popen(
for arg, val in keep_args.items():
keep_cmd.append("{}={}".format(arg, val))
- logf = open(_logfilename('keep{}'.format(n)), 'a')
- kp0 = subprocess.Popen(
- keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ with open(_logfilename('keep{}'.format(n)), 'a') as logf:
+ with open('/dev/null') as _stdin:
+ kp0 = subprocess.Popen(
+ keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
with open(_pidfile('keep{}'.format(n)), 'w') as f:
f.write(str(kp0.pid))
pidfile = _pidfile('keepproxy')
if os.path.exists(pidfile):
try:
- os.kill(int(open(pidfile).read()), signal.SIGHUP)
+ with open(pidfile) as pid:
+ os.kill(int(pid.read()), signal.SIGHUP)
except OSError:
os.remove(pidfile)
# Returns 9 if program is not up.
def _getport(program):
try:
- return int(open(_portfile(program)).read())
+ with open(_portfile(program)) as prog:
+ return int(prog.read())
except IOError:
return 9
def _dbconfig(key):
global _cached_db_config
if not _cached_db_config:
- _cached_db_config = yaml.safe_load(open(os.path.join(
- SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
- return _cached_db_config['test'][key]
+ if "ARVADOS_CONFIG" in os.environ:
+ _cached_db_config = list(yaml.safe_load(open(os.environ["ARVADOS_CONFIG"]))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
+ else:
+ _cached_db_config = yaml.safe_load(open(os.path.join(
+ SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))["test"]
+ _cached_db_config["dbname"] = _cached_db_config["database"]
+ _cached_db_config["user"] = _cached_db_config["username"]
+ return _cached_db_config[key]
def _apiconfig(key):
global _cached_config
projects / arvados.git / commitdiff