ubuntu1204,ubuntu1404|requests|2.4.3|2|python|all
ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
ubuntu1204,centos7|isodate|0.5.4|2|python|all
-centos7|daemon|2.1.1|2|python|all
+centos7|python-daemon|2.1.2|1|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
Run package install tests
--debug
Output debug information (default: false)
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--only-test)
ONLY_TEST="$1 $2"; shift
;;
+ --build-version)
+ ARVADOS_BUILDING_VERSION="$2"; shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" --build-version "$ARVADOS_BUILDING_VERSION" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
true
else
FINAL_EXITCODE=$?
Build only a specific package
--only-test <package>
Test only a specific package
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test:,only-build: \
+ help,debug,test-packages,target:,command:,only-test:,only-build:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--test-packages)
test_packages=1
;;
+ --build-version)
+ if [[ -z "$2" ]]; then
+ :
+ elif ! [[ "$2" =~ (.*)-(.*) ]]; then
+ echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
+ exit 1
+ else
+ ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+ ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
+ fi
+ shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
set -e
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
+
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
set +e
# Build packages
if docker run --rm \
"${docker_volume_args[@]}" \
+ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
+ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
"$IMAGE" $COMMAND
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
-arvados_cwl_runner_iteration=3
+declare -a iterargs=()
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ arvados_cwl_runner_iteration=3
+ iterargs+=(--iteration $arvados_cwl_runner_iteration)
+else
+ arvados_cwl_runner_iteration=
+fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 2
fi
# The PAM module
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-dockercleaner_iteration=3
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
if [[ "$?" == "0" ]]; then
rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
- # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
LIBCLOUD_DIR=$(mktemp -d)
(
cd $LIBCLOUD_DIR
handle_python_package
DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
)
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
rm -rf $LIBCLOUD_DIR
fi
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+ bundle install --system >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
# older packages.
LICENSE_PACKAGE_TS=20151208015500
-RAILS_PACKAGE_ITERATION=8
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ RAILS_PACKAGE_ITERATION=8
+else
+ RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
+fi
debug_echo () {
echo "$@" >"$STDOUT_IF_DEBUG"
}
version_from_git() {
- # Generates a version number from the git log for the current working
- # directory, and writes it to stdout.
- local git_ts git_hash prefix
- if [[ -n "$1" ]] ; then
- prefix="$1"
- else
- prefix="0.1"
- fi
+ # Output the version being built, or if we're building a
+ # dev/prerelease, output a version number based on the git log for
+ # the current working directory.
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
+
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
- declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
version_from_git $1 | cut -d. -f1-3
}
go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
- declare -a switches=()
+ local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
}
default_iteration() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_ITERATION"
+ return
+ fi
local package_name="$1"; shift
local package_version="$1"; shift
local package_type="$1"; shift
cd $tmppwd
- test_package_presence $pkgname $version rails $RAILS_PACKAGE_ITERATION
+ test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
test_package_presence() {
fi
if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
fi
if [[ "$arch" == "" ]]; then
fi
if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname=$pkgname"_"$version"-"$iteration"_"$deb_architecture".deb"
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
else
- local complete_pkgname="$pkgname-$version-$iteration.$rpm_architecture.rpm"
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
fi
# See if we can skip building the package, only if it already exists in the
return 0
fi
local srcdir="$1"; shift
+ cd "$srcdir"
local license_path="$1"; shift
+ local version="$(version_from_git)"
local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
- local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ (
set -e
_build_rails_package_scripts "$pkgname" "$scripts_dir"
cd "$srcdir"
mkdir -p tmp
- version_from_git >"$version_file"
git rev-parse HEAD >git-commit.version
bundle package --all
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
EXITCODE=1
return 1
fi
local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
- "$(cat "$version_file")")
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
local license_arg="$license_path=$railsdir/$(basename "$license_path")"
- local -a switches=(--iteration=$RAILS_PACKAGE_ITERATION
- --after-install "$scripts_dir/postinst"
+ local -a switches=(--after-install "$scripts_dir/postinst"
--before-remove "$scripts_dir/prerm"
--after-remove "$scripts_dir/postrm")
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ switches+=(--iteration $RAILS_PACKAGE_ITERATION)
+ fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
# .git and packages are for the SSO server, which is built from its
switches+=(-x "$exclude_root/$exclude")
done
fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
}
# Build packages for everything
if [[ "$VERSION" != "" ]]; then
COMMAND_ARR+=('-v' "$VERSION")
fi
- # We can always add an --iteration here. If another one is specified in $@,
- # that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ -n "$default_iteration_value" ]]; then
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$default_iteration_value")
+ fi
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
mkdir -p "$WORKSPACE/services/api/tmp/pids"
+ cert="$WORKSPACE/services/api/tmp/self-signed"
+ if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
+ (
+ dir="$WORKSPACE/services/api/tmp"
+ set -ex
+ openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
+ openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
+ ) || return 1
+ fi
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
-
-versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
-try:
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%H', gittaggers.choose_version_from()]).strip()
- with open(versionfile, "w") as f:
- f.write("__version__ = '%s'\n" % gitinfo)
-except Exception as e:
- # When installing from package, it won't be part of a git repository, and
- # check_output() will raise an exception. But the package should include the
- # version file, so we can proceed.
- if not os.path.exists(versionfile):
- raise
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "1.0"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-cwl-runner',
- version='1.0',
+ version=version,
description='Arvados Common Workflow Language runner',
long_description=open(README).read(),
author='Arvados',
README = os.path.join(SETUP_DIR, 'README.rst')
tagger = egg_info_cmd.egg_info
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except (ImportError, OSError):
- pass
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-pam',
- version='0.1',
+ version=version,
description='Arvados PAM module',
long_description=open(README).read(),
author='Arvados',
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados-python-client',
- version='0.1',
+ version=version,
description='Arvados client library',
long_description=open(README).read(),
author='Arvados',
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
- if not os.path.exists('tmp/self-signed.pem'):
- # We assume here that either passenger reports its listening
- # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
- # then the certificate won't match the host and reset() will
- # fail certificate verification. If it reports "localhost",
- # clients (notably Python SDK's websocket client) might
- # resolve localhost as ::1 and then fail to connect.
- subprocess.check_call([
- 'openssl', 'req', '-new', '-x509', '-nodes',
- '-out', 'tmp/self-signed.pem',
- '-keyout', 'tmp/self-signed.key',
- '-days', '3650',
- '-subj', '/CN=0.0.0.0'],
- stdout=sys.stderr)
-
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
gem 'arvados', '>= 0.1.20150615153458'
gem 'arvados-cli', '>= 0.1.20161017193526'
+gem 'httpclient'
gem 'sshkey'
gem 'safe_yaml'
hashie (3.5.5)
highline (1.7.8)
hike (1.2.3)
+ httpclient (2.8.3)
i18n (0.9.0)
concurrent-ruby (~> 1.0)
jquery-rails (4.2.2)
database_cleaner
factory_girl_rails
faye-websocket
+ httpclient
jquery-rails
lograge
logstash-event
.all
end
@read_auths.select! { |auth| auth.scopes_allow_request? request }
- @read_users = @read_auths.map { |auth| auth.user }.uniq
+ @read_users = @read_auths.map(&:user).uniq
end
def require_login
def index
expires_in 24.hours, public: true
- discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
+ send_json discovery_doc
+ end
+
+ protected
+
+ def discovery_doc
+ Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
discovery = {
kind: "discovery#restDescription",
crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
+ remoteHosts: Rails.configuration.remote_hosts,
+ remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
end
discovery
end
- send_json discovery
end
end
return redirect_to login_failure_url
end
- user = User.find_by_identity_url(omniauth['info']['identity_url'])
+ # Only local users can create sessions, hence uuid_like_pattern
+ # here.
+ user = User.where('identity_url = ? and uuid like ?',
+ omniauth['info']['identity_url'],
+ User.uuid_like_pattern).first
if not user
# Check for permission to log in to an existing User record with
# a different identity_url
end
def call env
- # First, clean up just in case we have a multithreaded server and thread
- # local variables are still set from a prior request. Also useful for
- # tests that call this code to set up the environment.
- Thread.current[:api_client_ip_address] = nil
- Thread.current[:api_client_authorization] = nil
- Thread.current[:api_client_uuid] = nil
- Thread.current[:api_client] = nil
- Thread.current[:user] = nil
-
request = Rack::Request.new(env)
params = request.params
remote_ip = env["action_dispatch.remote_ip"]
Thread.current[:request_starttime] = Time.now
- user = nil
- api_client = nil
- api_client_auth = nil
- if request.get? || params["_method"] == 'GET'
+
+ remote = false
+ reader_tokens = nil
+ if params["remote"] && request.get? && (
+ request.path.start_with?('/arvados/v1/groups') ||
+ request.path.start_with?('/arvados/v1/users/current'))
+ # Request from a remote API server, asking to validate a salted
+ # token.
+ remote = params["remote"]
+ elsif request.get? || params["_method"] == 'GET'
reader_tokens = params["reader_tokens"]
if reader_tokens.is_a? String
reader_tokens = SafeJSON.load(reader_tokens)
end
- else
- reader_tokens = nil
end
# Set current_user etc. based on the primary session token if a
# valid one is present. Otherwise, use the first valid token in
# reader_tokens.
+ auth = nil
[params["api_token"],
params["oauth_token"],
- env["HTTP_AUTHORIZATION"].andand.match(/OAuth2 ([a-zA-Z0-9]+)/).andand[1],
+ env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([-\/a-zA-Z0-9]+)/).andand[2],
*reader_tokens,
].each do |supplied|
next if !supplied
try_auth = ApiClientAuthorization.
- includes(:api_client, :user).
- where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', supplied).
- first
+ validate(token: supplied, remote: remote)
if try_auth.andand.user
- api_client_auth = try_auth
- user = api_client_auth.user
- api_client = api_client_auth.api_client
+ auth = try_auth
break
end
end
+
Thread.current[:api_client_ip_address] = remote_ip
- Thread.current[:api_client_authorization] = api_client_auth
- Thread.current[:api_client_uuid] = api_client.andand.uuid
- Thread.current[:api_client] = api_client
- Thread.current[:user] = user
+ Thread.current[:api_client_authorization] = auth
+ Thread.current[:api_client_uuid] = auth.andand.api_client.andand.uuid
+ Thread.current[:api_client] = auth.andand.api_client
+ Thread.current[:user] = auth.andand.user
@app.call env if @app
end
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ extend CurrentApiClient
belongs_to :api_client
belongs_to :user
t.add :owner_uuid
t.add :user_id
t.add :api_client_id
+ # NB the "api_token" db column is a misnomer in that it's only the
+ # "secret" part of a token: a v1 token is just the secret, but a
+ # v2 token is "v2/uuid/secret".
t.add :api_token
t.add :created_by_ip_address
t.add :default_owner_uuid
["#{table_name}.id desc"]
end
+ def self.remote_host(uuid_prefix:)
+ Rails.configuration.remote_hosts[uuid_prefix] ||
+ (Rails.configuration.remote_hosts_via_dns &&
+ uuid_prefix+".arvadosapi.com")
+ end
+
+ def self.validate(token:, remote:)
+ return nil if !token
+ remote ||= Rails.configuration.uuid_prefix
+
+ case token[0..2]
+ when 'v2/'
+ _, uuid, secret = token.split('/')
+ unless uuid.andand.length == 27 && secret.andand.length.andand > 0
+ return nil
+ end
+
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('uuid=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', uuid).
+ first
+ if auth && auth.user &&
+ (secret == auth.api_token ||
+ secret == OpenSSL::HMAC.hexdigest('sha1', auth.api_token, remote))
+ return auth
+ end
+
+ uuid_prefix = uuid[0..4]
+ if uuid_prefix == Rails.configuration.uuid_prefix
+ # If the token were valid, we would have validated it above
+ return nil
+ elsif uuid_prefix.length != 5
+ # malformed
+ return nil
+ end
+
+ host = remote_host(uuid_prefix: uuid_prefix)
+ if !host
+ Rails.logger.warn "remote authentication rejected: no host for #{uuid_prefix.inspect}"
+ return nil
+ end
+
+ # Token was issued by a different cluster. If it's expired or
+ # missing in our database, ask the originating cluster to
+ # [re]validate it.
+ begin
+ clnt = HTTPClient.new
+ if Rails.configuration.sso_insecure
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ end
+ remote_user = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/users/current',
+ {'remote' => Rails.configuration.uuid_prefix},
+ {'Authorization' => 'Bearer ' + token}))
+ rescue => e
+ Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}"
+ return nil
+ end
+ if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String) || remote_user['uuid'][0..4] != uuid[0..4]
+ Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}"
+ return nil
+ end
+ act_as_system_user do
+ # Add/update user and token in our database so we can
+ # validate subsequent requests faster.
+
+ user = User.find_or_create_by(uuid: remote_user['uuid']) do |user|
+ # (this block runs for the "create" case, not for "find")
+ user.is_admin = false
+ user.email = remote_user['email']
+ if remote_user['username'].andand.length.andand > 0
+ user.set_initial_username(requested: remote_user['username'])
+ end
+ end
+
+ if Rails.configuration.new_users_are_active
+ # Update is_active to whatever it is at the remote end
+ user.is_active = remote_user['is_active']
+ elsif !remote_user['is_active']
+ # Remote user is inactive; our mirror should be, too.
+ user.is_active = false
+ end
+
+ %w[first_name last_name email prefs].each do |attr|
+ user.send(attr+'=', remote_user[attr])
+ end
+
+ user.save!
+
+ auth = ApiClientAuthorization.find_or_create_by(uuid: uuid) do |auth|
+ auth.user = user
+ auth.api_token = secret
+ auth.api_client_id = 0
+ end
+
+ # Accept this token (and don't reload the user record) for
+ # 5 minutes. TODO: Request the actual api_client_auth
+ # record from the remote server in case it wants the token
+ # to expire sooner.
+ auth.update_attributes!(expires_at: Time.now + 5.minutes)
+ end
+ return auth
+ else
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token).
+ first
+ if auth && auth.user
+ return auth
+ end
+ end
+ return nil
+ end
+
protected
def permission_to_create
# original job reuse behavior, and is still the default).
reuse_job_if_outputs_differ: false
+ ###
+ ### Federation support.
+ ###
+
+ # You can enable use of this cluster by users who are authenticated
+ # by a remote Arvados site. Control which remote hosts are trusted
+ # to authenticate which user IDs by configuring remote_hosts,
+ # remote_hosts_via_dns, or both. The default configuration disables
+ # remote authentication.
+
+ # Map known prefixes to hosts. For example, if user IDs beginning
+ # with "zzzzz-" should be authenticated by the Arvados server at
+ # "zzzzz.example.com", use:
+ #
+ # remote_hosts:
+ # zzzzz: zzzzz.example.com
+ remote_hosts: {}
+
+ # Use {prefix}.arvadosapi.com for any prefix not given in
+ # remote_hosts above.
+ remote_hosts_via_dns: false
+
###
### Remaining assorted configuration options.
###
arvados_theme: default
- # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
- # On (sso) server. Should only be enabled during development when the SSO
- # server is using a self-signed cert.
+ # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the
+ # Single Sign On (sso) server and remote Arvados sites. Should only
+ # be enabled during development when the SSO server is using a
+ # self-signed cert.
sso_insecure: false
## Set Time.zone default to the specified zone and make Active
assert_response :success
assert_not_nil Group.readable_by(users(auth)).where(uuid: groups(:trashed_subproject).uuid).first
end
-
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'webrick'
+require 'webrick/https'
+require 'test_helper'
+require 'helpers/users_test_helper'
+
+class RemoteUsersTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
+
+ def salted_active_token(remote:)
+ salt_token(fixture: :active, remote: remote).sub('/zzzzz-', '/'+remote+'-')
+ end
+
+ def auth(remote:)
+ token = salted_active_token(remote: remote)
+ {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ end
+
+ # For remote authentication tests, we bring up a simple stub server
+ # (on a port chosen by webrick) and configure the SUT so the stub is
+ # responsible for clusters "zbbbb" (a well-behaved cluster) and
+ # "zbork" (a misbehaving cluster).
+ #
+ # Test cases can override the stub's default response to
+ # .../users/current by changing @stub_status and @stub_content.
+ setup do
+ clnt = HTTPClient.new
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ HTTPClient.stubs(:new).returns clnt
+
+ @controller = Arvados::V1::UsersController.new
+ ready = Thread::Queue.new
+ srv = WEBrick::HTTPServer.new(
+ Port: 0,
+ Logger: WEBrick::Log.new(
+ Rails.root.join("log", "webrick.log").to_s,
+ WEBrick::Log::INFO),
+ AccessLog: [[File.open(Rails.root.join(
+ "log", "webrick_access.log").to_s, 'a+'),
+ WEBrick::AccessLog::COMBINED_LOG_FORMAT]],
+ SSLEnable: true,
+ SSLVerifyClient: OpenSSL::SSL::VERIFY_NONE,
+ SSLPrivateKey: OpenSSL::PKey::RSA.new(
+ File.open(Rails.root.join("tmp", "self-signed.key")).read),
+ SSLCertificate: OpenSSL::X509::Certificate.new(
+ File.open(Rails.root.join("tmp", "self-signed.pem")).read),
+ SSLCertName: [["CN", WEBrick::Utils::getservername]],
+ StartCallback: lambda { ready.push(true) })
+ srv.mount_proc '/discovery/v1/apis/arvados/v1/rest' do |req, res|
+ Rails.cache.delete 'arvados_v1_rest_discovery'
+ res.body = Arvados::V1::SchemaController.new.send(:discovery_doc).to_json
+ end
+ srv.mount_proc '/arvados/v1/users/current' do |req, res|
+ res.status = @stub_status
+ res.body = @stub_content.is_a?(String) ? @stub_content : @stub_content.to_json
+ end
+ Thread.new do
+ srv.start
+ end
+ ready.pop
+ @remote_server = srv
+ @remote_host = "127.0.0.1:#{srv.config[:Port]}"
+ Rails.configuration.remote_hosts['zbbbb'] = @remote_host
+ Rails.configuration.remote_hosts['zbork'] = @remote_host
+ Arvados::V1::SchemaController.any_instance.stubs(:root_url).returns "https://#{@remote_host}"
+ @stub_status = 200
+ @stub_content = {
+ uuid: 'zbbbb-tpzed-000000000000000',
+ email: 'foo@example.com',
+ username: 'barney',
+ is_admin: true,
+ is_active: true,
+ }
+ end
+
+ teardown do
+ @remote_server.andand.stop
+ end
+
+ test 'authenticate with remote token' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
+ assert_equal false, json_response['is_admin']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+
+ # revoke original token
+ @stub_status = 401
+
+ # re-authorize before cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+
+ # simulate cache expiry
+ ApiClientAuthorization.where(
+ uuid: salted_active_token(remote: 'zbbbb').split('/')[1]).
+ update_all(expires_at: db_current_time - 1.minute)
+
+ # re-authorize after cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+
+ # revive original token and re-authorize
+ @stub_status = 200
+ @stub_content[:username] = 'blarney'
+ @stub_content[:email] = 'blarney@example.com'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'barney', json_response['username'], 'local username should not change once assigned'
+ assert_equal 'blarney@example.com', json_response['email']
+ end
+
+ test 'authenticate with remote token, remote username conflicts with local' do
+ @stub_content[:username] = 'active'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'active2', json_response['username']
+ end
+
+ test 'authenticate with remote token, remote username is nil' do
+ @stub_content.delete :username
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'foo', json_response['username']
+ end
+
+ test 'authenticate with remote token from misbhehaving remote cluster' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbork')
+ assert_response 401
+ end
+
+ test 'authenticate with remote token that fails validate' do
+ @stub_status = 401
+ @stub_content = {
+ error: 'not authorized',
+ }
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['v2',
+ 'v2/',
+ 'v2//',
+ 'v2///',
+ "v2/'; delete from users where 1=1; commit; select '/lol",
+ 'v2/foo/bar',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1/',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2//3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v8/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '"v2/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"',
+ '/',
+ '//',
+ '///',
+ ].each do |token|
+ test "authenticate with malformed remote token #{token}" do
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response 401
+ end
+ end
+
+ test "ignore extra fields in remote token" do
+ token = salted_active_token(remote: 'zbbbb') + '/foo/bar/baz/*'
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response :success
+ end
+
+ test 'remote api server is not an api server' do
+ @stub_status = 200
+ @stub_content = '<html>bad</html>'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['zbbbb', 'z0000'].each do |token_valid_for|
+ test "validate #{token_valid_for}-salted token for remote cluster zbbbb" do
+ salted_token = salt_token(fixture: :active, remote: token_valid_for)
+ get '/arvados/v1/users/current', {format: 'json', remote: 'zbbbb'}, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ if token_valid_for == 'zbbbb'
+ assert_response 200
+ assert_equal(users(:active).uuid, json_response['uuid'])
+ else
+ assert_response 401
+ end
+ end
+ end
+
+ test "list readable groups with salted token" do
+ salted_token = salt_token(fixture: :active, remote: 'zbbbb')
+ get '/arvados/v1/groups', {
+ format: 'json',
+ remote: 'zbbbb',
+ limit: 10000,
+ }, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ assert_response 200
+ group_uuids = json_response['items'].collect { |i| i['uuid'] }
+ assert_includes(group_uuids, 'zzzzz-j7d0g-fffffffffffffff')
+ refute_includes(group_uuids, 'zzzzz-j7d0g-000000000000000')
+ assert_includes(group_uuids, groups(:aproject).uuid)
+ refute_includes(group_uuids, groups(:trashed_project).uuid)
+ refute_includes(group_uuids, groups(:testusergroup_admins).uuid)
+ end
+end
"HTTP_AUTHORIZATION" => "OAuth2 #{t}")
end
+ def salt_token(fixture:, remote:)
+ auth = api_client_authorizations(fixture)
+ uuid = auth.uuid
+ token = auth.api_token
+ hmac = OpenSSL::HMAC.hexdigest('sha1', token, remote)
+ return "v2/#{uuid}/#{hmac}"
+ end
+
def self.skip_slow_tests?
!(ENV['RAILS_TEST_SHORT'] || '').empty?
end
from setuptools import setup, find_packages
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name="arvados-docker-cleaner",
- version="0.1",
+ version=version,
description="Arvados Docker cleaner",
author="Arvados",
author_email="info@arvados.org",
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados_fuse',
- version='0.1',
+ version=version,
description='Arvados FUSE driver',
long_description=open(README).read(),
author='Arvados',
SETUP_DIR = os.path.dirname(__file__) or "."
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-node-manager',
- version='0.1',
+ version=version,
description='Arvados compute node manager',
long_description=open(README).read(),
author='Arvados',
yarn install
if test "$1" != "--only-deps" ; then
- echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/arvados-configuration.yml
+ echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
exec ng serve --host 0.0.0.0 --port 4200 --env=webdev
fi
SETUP_DIR = os.path.dirname(__file__) or '.'
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='crunchstat_summary',
- version='0.1',
+ version=version,
description='read crunch log files and summarize resource usage',
author='Arvados',
author_email='info@arvados.org',
projects / arvados.git / commitdiff