GEMHOME=
PERLINSTALLBASE=
R_LIBS=
+export LANG=en_US.UTF-8
short=
only_install=
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
echo Checking dependencies:
+ echo "locale: ${LANG}"
+ [[ "$(locale charmap)" = "UTF-8" ]] \
+ || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen"
echo -n 'virtualenv: '
virtualenv --version \
|| fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
echo "R SDK not needed, it will not be installed."
fi
+checkpidfile() {
+ svc="$1"
+ pid="$(cat "$WORKSPACE/tmp/${svc}.pid")"
+ if [[ -z "$pid" ]] || ! kill -0 "$pid"; then
+ tail $WORKSPACE/tmp/${1}*.log
+ echo "${svc} pid ${pid} not running"
+ return 1
+ fi
+ echo "${svc} pid ${pid} ok"
+}
+
+checkdiscoverydoc() {
+ dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
+ if ! (set -o pipefail; curl -fsk "$dd" | jq . >/dev/null); then
+ echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd"
+ tail -v $WORKSPACE/services/api/log/test.log
+ return 1
+ fi
+ echo "${dd} ok"
+}
+
start_services() {
if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
return 0
&& eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo "fail=1; false") \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
+ && checkpidfile api \
+ && checkdiscoverydoc $ARVADOS_API_HOST \
&& python sdk/python/tests/run_test_server.py start_controller \
+ && checkpidfile controller \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && checkpidfile keepproxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
+ && checkpidfile keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && checkpidfile arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
+ && checkpidfile ws \
&& eval $(python sdk/python/tests/run_test_server.py start_nginx || echo "fail=1; false") \
+ && checkdiscoverydoc $ARVADOS_API_HOST \
+ && checkpidfile nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1 \
&& (env | egrep ^ARVADOS) \
|| fail=1
deactivate
+ if [[ $fail = 0 ]] && ! kill -0 "$(cat "$WORKSPACE/tmp/nginx.pid")"; then
+ echo >&2 "ERROR: nginx seems to have died already"
+ fail=1
+ fi
if [[ $fail != 0 ]]; then
unset ARVADOS_TEST_API_HOST
fi
if [[ -n "$all_services_stopped" ]]; then
return
fi
- unset ARVADOS_TEST_API_HOST
+ unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES
. "$VENVDIR/bin/activate" || return
cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py stop_nginx \
services/api)
stop_services
;;
- doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
# don't care whether services are running
;;
*)
# compilation errors.
go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
- [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
if [[ -n "${testargs[$1]}" ]]
then
# "go test -check.vv giturl" doesn't work, but this
go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
rm "$WORKSPACE/tmp/.$covername.tmp"
fi
+ [[ $result = 0 ]] && gofmt -e -d *.go
elif [[ "$2" == "pip" ]]
then
tries=0
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
cd "$WORKSPACE/services/api" \
- && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+ && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
&& psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
mkdir -p "$WORKSPACE/services/api/tmp/pids"
)
}
+test_gofmt() {
+ cd "$WORKSPACE" || return 1
+ dirs=$(ls -d */ | egrep -v 'vendor|tmp')
+ [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+}
+
test_services/api() {
rm -f "$WORKSPACE/services/api/git-commit.version"
cd "$WORKSPACE/services/api" \
test_apps/workbench_units() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
}
test_apps/workbench_functionals() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
}
test_apps/workbench_integration() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
}
test_apps/workbench_benchmark() {
exit_cleanly
fi
+ do_test gofmt
do_test doc
do_test sdk/ruby
do_test sdk/R
read verb target opts <<<"${nextcmd}"
target="${target%/}"
target="${target/\/:/:}"
- if [[ -z "${target}" ]]; then
- verb=help
- fi
case "${verb}" in
- "" | "help")
- help_interactive
- ;;
"exit" | "quit")
exit_cleanly
;;
"reset")
stop_services
;;
- *)
- testargs["$target"]="${opts}"
+ "test" | "install")
case "$target" in
+ "")
+ help_interactive
+ ;;
all | deps)
${verb}_${target}
;;
*)
+ testargs["$target"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
do_$verb $tt
;;
esac
;;
+ "" | "help" | *)
+ help_interactive
+ ;;
esac
if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then
report_outcomes
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
client_body_temp_path "{{TMPDIR}}";
+ proxy_temp_path "{{TMPDIR}}";
+ fastcgi_temp_path "{{TMPDIR}}";
+ uwsgi_temp_path "{{TMPDIR}}";
+ scgi_temp_path "{{TMPDIR}}";
upstream arv-git-http {
server localhost:{{GITPORT}};
}
def run_keep_proxy():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(_getport('keepproxy'))
return
stop_keep_proxy()
def _dbconfig(key):
global _cached_db_config
if not _cached_db_config:
- _cached_db_config = yaml.load(open(os.path.join(
+ _cached_db_config = yaml.safe_load(open(os.path.join(
SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
return _cached_db_config['test'][key]
fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
if not required and not os.path.exists(fullpath):
return {}
- return yaml.load(fullpath)
+ return yaml.safe_load(fullpath)
cdefault = _load('application.default.yml')
csite = _load('application.yml', required=False)
_cached_config = {}
yaml_file = yaml_file[0:trim_index]
except ValueError:
pass
- return yaml.load(yaml_file)
+ return yaml.safe_load(yaml_file)
def auth_token(token_name):
return fixture("api_client_authorizations")[token_name]["api_token"]
if not mandatory and not os.path.exists(path):
continue
with open(path) as f:
- rails_config = yaml.load(f.read())
+ rails_config = yaml.safe_load(f.read())
for config_section in ['test', 'common']:
try:
key = rails_config[config_section]["blob_signing_key"]
adapter: postgresql
template: template0
encoding: utf8
+ collation: en_US.utf8
database: arvados_test
username: arvados
password: xxxxxxxx
username: arvados
password: xxxxxxxx
host: localhost
- # For the websockets server, prefer a larger database connection pool size since it
- # multithreaded and can serve a large number of long-lived clients. See also
- # websocket_max_connections configuration option.
pool: 50
APIHost: arvadostest.APIHost(),
Insecure: true,
},
- Listen: ":0",
+ Listen: "localhost:0",
GitCommand: "/usr/share/gitolite3/gitolite-shell",
GitoliteHome: s.gitoliteHome,
RepoRoot: s.tmpRepoRoot,
APIHost: arvadostest.APIHost(),
Insecure: true,
},
- Listen: ":0",
+ Listen: "localhost:0",
GitCommand: "/usr/bin/git",
RepoRoot: s.tmpRepoRoot,
ManagementToken: arvadostest.ManagementToken,
go func() {
_, err := io.Copy(response.Conn, stdinRdr)
if err != nil {
- runner.CrunchLog.Print("While writing stdin collection to docker container %q", err)
+ runner.CrunchLog.Printf("While writing stdin collection to docker container %q", err)
runner.stop(nil)
}
stdinRdr.Close()
go func() {
_, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
if err != nil {
- runner.CrunchLog.Print("While writing stdin json to docker container %q", err)
+ runner.CrunchLog.Printf("While writing stdin json to docker container %q", err)
runner.stop(nil)
}
response.CloseWrite()
c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
cmd := exec.Command("curl", curlArgs...)
stdout, err := cmd.StdoutPipe()
- c.Assert(err, check.Equals, nil)
- cmd.Stderr = cmd.Stdout
- go cmd.Start()
+ c.Assert(err, check.IsNil)
+ cmd.Stderr = os.Stderr
+ err = cmd.Start()
+ c.Assert(err, check.IsNil)
buf := make([]byte, 2<<27)
n, err := io.ReadFull(stdout, buf)
// Discard (but measure size of) anything past 128 MiB.
if err == io.ErrUnexpectedEOF {
buf = buf[:n]
} else {
- c.Assert(err, check.Equals, nil)
+ c.Assert(err, check.IsNil)
discarded, err = io.Copy(ioutil.Discard, stdout)
- c.Assert(err, check.Equals, nil)
+ c.Assert(err, check.IsNil)
}
err = cmd.Wait()
// Without "-f", curl exits 0 as long as it gets a valid HTTP
}
err = f.Sync()
if err != nil {
- log.Fatal("sync(%s): %s", cfg.PIDFile, err)
+ log.Fatalf("sync(%s): %s", cfg.PIDFile, err)
}
}
gemspec
group :test, :performance do
gem 'minitest', '>= 5.0.0'
- gem 'mocha', require: false
+ gem 'mocha', '>= 1.5.0', require: false
gem 'rake'
end
#
# SPDX-License-Identifier: AGPL-3.0
-if not File.exists?('/usr/bin/git') then
+if not File.exist?('/usr/bin/git') then
STDERR.puts "\nGit binary not found, aborting. Please install git and run gem build from a checked out copy of the git repository.\n\n"
exit
end
"-G", groups.join(","),
l[:username],
out: devnull)
- STDERR.puts "Account creation failed for #{l[:username]}: $?"
+ STDERR.puts "Account creation failed for #{l[:username]}: #{$?}"
next
end
begin
@homedir = pwnam[l[:username]].dir
userdotssh = File.join(@homedir, ".ssh")
- Dir.mkdir(userdotssh) if !File.exists?(userdotssh)
+ Dir.mkdir(userdotssh) if !File.exist?(userdotssh)
newkeys = "###\n###\n" + keys[l[:username]].join("\n") + "\n###\n###\n"
keysfile = File.join(userdotssh, "authorized_keys")
- if File.exists?(keysfile)
+ if File.exist?(keysfile)
oldkeys = IO::read(keysfile)
else
oldkeys = ""
# SPDX-License-Identifier: AGPL-3.0
require 'etc'
-require 'mocha/mini_test'
+require 'mocha/minitest'
require 'ostruct'
module Stubs
- # These Etc mocks help only when we run arvados-login-sync in-process.
-
- def setup
- super
- ENV['ARVADOS_VIRTUAL_MACHINE_UUID'] = 'testvm2.shell'
- Etc.stubs(:to_enum).with(:passwd).returns stubpasswd.map { |x| OpenStruct.new x }
- Etc.stubs(:to_enum).with(:group).returns stubgroup.map { |x| OpenStruct.new x }
- end
-
def stubpasswd
[{name: 'root', uid: 0}]
end
[{name: 'root', gid: 0}]
end
- # These child-ENV tricks help only when we run arvados-login-sync as a subprocess.
def setup
super
+
+ # These Etc mocks help only when we run arvados-login-sync in-process.
+ ENV['ARVADOS_VIRTUAL_MACHINE_UUID'] = 'testvm2.shell'
+ Etc.stubs(:to_enum).with(:passwd).returns stubpasswd.map { |x| OpenStruct.new x }
+ Etc.stubs(:to_enum).with(:group).returns stubgroup.map { |x| OpenStruct.new x }
+
+ # These child-ENV tricks help only when we run arvados-login-sync as a subprocess.
@env_was = Hash[ENV]
@tmpdir = Dir.mktmpdir
end
include Stubs
def test_useradd_error
+ valid_groups = %w(docker admin fuse).select { |g| Etc.getgrnam(g) rescue false }
# binstub_new_user/useradd will exit non-zero because its args
# won't match any line in this empty file:
File.open(@tmpdir+'/succeed', 'w') do |f| end
invoke_sync binstubs: ['new_user']
spied = File.read(@tmpdir+'/spy')
assert_match %r{useradd -m -c active -s /bin/bash -G (fuse)? active}, spied
- # BUG(TC): This assertion succeeds only if docker and fuse groups
- # exist on the host, but is insensitive to the admin group (groups
- # are quietly ignored by login-sync if they don't exist on the
- # current host).
- assert_match %r{useradd -m -c adminroot -s /bin/bash -G (docker)?(,admin)?(,fuse)? adminroot}, spied
+ assert_match %r{useradd -m -c adminroot -s /bin/bash -G #{valid_groups.join(',')} adminroot}, spied
end
def test_useradd_success
projects / arvados.git / commitdiff