sdk/java/log
tmp
sdk/cli/binstubs/
-sdk/cwl/arvados_cwl/_version.py
services/api/config/arvados-clients.yml
*#*
.DS_Store
ubuntu1204,ubuntu1404|requests|2.4.3|2|python|all
ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
ubuntu1204,centos7|isodate|0.5.4|2|python|all
-centos7|daemon|2.1.1|2|python|all
+centos7|python-daemon|2.1.2|1|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
Run package install tests
--debug
Output debug information (default: false)
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--only-test)
ONLY_TEST="$1 $2"; shift
;;
+ --build-version)
+ ARVADOS_BUILDING_VERSION="$2"; shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" --build-version "$ARVADOS_BUILDING_VERSION" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
true
else
FINAL_EXITCODE=$?
Build only a specific package
--only-test <package>
Test only a specific package
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test:,only-build: \
+ help,debug,test-packages,target:,command:,only-test:,only-build:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--test-packages)
test_packages=1
;;
+ --build-version)
+ if [[ -z "$2" ]]; then
+ :
+ elif ! [[ "$2" =~ (.*)-(.*) ]]; then
+ echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
+ exit 1
+ else
+ ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+ ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
+ fi
+ shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
set -e
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
+
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
set +e
# Build packages
if docker run --rm \
"${docker_volume_args[@]}" \
+ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
+ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
"$IMAGE" $COMMAND
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
-arvados_cwl_runner_iteration=3
+declare -a iterargs=()
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ arvados_cwl_runner_iteration=3
+ iterargs+=(--iteration $arvados_cwl_runner_iteration)
+else
+ arvados_cwl_runner_iteration=
+fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 2
fi
# The PAM module
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-dockercleaner_iteration=3
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
if [[ "$?" == "0" ]]; then
rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
- # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
LIBCLOUD_DIR=$(mktemp -d)
(
cd $LIBCLOUD_DIR
handle_python_package
DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
)
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
rm -rf $LIBCLOUD_DIR
fi
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+ bundle install --system >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
# older packages.
LICENSE_PACKAGE_TS=20151208015500
-RAILS_PACKAGE_ITERATION=8
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ RAILS_PACKAGE_ITERATION=8
+else
+ RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
+fi
debug_echo () {
echo "$@" >"$STDOUT_IF_DEBUG"
}
version_from_git() {
- # Generates a version number from the git log for the current working
- # directory, and writes it to stdout.
- local git_ts git_hash prefix
- if [[ -n "$1" ]] ; then
- prefix="$1"
- else
- prefix="0.1"
- fi
+ # Output the version being built, or if we're building a
+ # dev/prerelease, output a version number based on the git log for
+ # the current working directory.
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
+
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
- declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
version_from_git $1 | cut -d. -f1-3
}
go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
- declare -a switches=()
+ local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
}
default_iteration() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_ITERATION"
+ return
+ fi
local package_name="$1"; shift
local package_version="$1"; shift
local package_type="$1"; shift
cd $tmppwd
- test_package_presence $pkgname $version rails $RAILS_PACKAGE_ITERATION
+ test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
test_package_presence() {
fi
if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
fi
if [[ "$arch" == "" ]]; then
fi
if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname=$pkgname"_"$version"-"$iteration"_"$deb_architecture".deb"
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
else
- local complete_pkgname="$pkgname-$version-$iteration.$rpm_architecture.rpm"
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
fi
# See if we can skip building the package, only if it already exists in the
return 0
fi
local srcdir="$1"; shift
+ cd "$srcdir"
local license_path="$1"; shift
+ local version="$(version_from_git)"
local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
- local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ (
set -e
_build_rails_package_scripts "$pkgname" "$scripts_dir"
cd "$srcdir"
mkdir -p tmp
- version_from_git >"$version_file"
git rev-parse HEAD >git-commit.version
bundle package --all
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
EXITCODE=1
return 1
fi
local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
- "$(cat "$version_file")")
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
local license_arg="$license_path=$railsdir/$(basename "$license_path")"
- local -a switches=(--iteration=$RAILS_PACKAGE_ITERATION
- --after-install "$scripts_dir/postinst"
+ local -a switches=(--after-install "$scripts_dir/postinst"
--before-remove "$scripts_dir/prerm"
--after-remove "$scripts_dir/postrm")
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ switches+=(--iteration $RAILS_PACKAGE_ITERATION)
+ fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
# .git and packages are for the SSO server, which is built from its
switches+=(-x "$exclude_root/$exclude")
done
fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
}
# Build packages for everything
if [[ "$VERSION" != "" ]]; then
COMMAND_ARR+=('-v' "$VERSION")
fi
- # We can always add an --iteration here. If another one is specified in $@,
- # that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ -n "$default_iteration_value" ]]; then
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$default_iteration_value")
+ fi
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
mkdir -p "$WORKSPACE/services/api/tmp/pids"
+ cert="$WORKSPACE/services/api/tmp/self-signed"
+ if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
+ (
+ dir="$WORKSPACE/services/api/tmp"
+ set -ex
+ openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
+ openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
+ ) || return 1
+ fi
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
#!/usr/bin/env perl
+# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
-# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
-
=head1 NAME
crunch-job: Execute job steps, save snapshots as requested, collate output.
. $slot[$proc{$pid}->{slot}]->{cpu});
my $jobstepidx = $proc{$pid}->{jobstepidx};
+ readfrompipes_after_exit ($jobstepidx);
+
$children_reaped++;
my $elapsed = time - $proc{$pid}->{time};
my $Jobstep = $jobstep[$jobstepidx];
$Jobstep->{finishtime} = time;
$Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
retry_op(sub { $Jobstep->{'arvados_task'}->save; }, "job_tasks.update API");
- process_stderr_final ($jobstepidx);
Log ($jobstepidx, sprintf("task output (%d bytes): %s",
length($Jobstep->{'arvados_task'}->{output}),
$Jobstep->{'arvados_task'}->{output}));
}
-sub process_stderr_final
+# Read whatever is still available on its stderr+stdout pipes after
+# the given child process has exited.
+sub readfrompipes_after_exit
{
my $jobstepidx = shift;
+
+ # The fact that the child has exited allows some convenient
+ # simplifications: (1) all data must have already been written, so
+ # there's no need to wait for more once sysread returns 0; (2) the
+ # total amount of data available is bounded by the pipe buffer size,
+ # so it's safe to read everything into one string.
+ my $buf;
+ while (0 < sysread ($reader{$jobstepidx}, $buf, 65536)) {
+ $jobstep[$jobstepidx]->{stderr_at} = time;
+ $jobstep[$jobstepidx]->{stderr} .= $buf;
+ }
+ if ($jobstep[$jobstepidx]->{stdout_r}) {
+ while (0 < sysread ($jobstep[$jobstepidx]->{stdout_r}, $buf, 65536)) {
+ $jobstep[$jobstepidx]->{stdout_captured} .= $buf;
+ }
+ }
preprocess_stderr ($jobstepidx);
map {
}
my $exited = $?;
- 1 while readfrompipes();
- process_stderr_final ($jobstepidx);
+ readfrompipes_after_exit ($jobstepidx);
Log (undef, "$label: exit ".exit_status_s($exited));
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pkg_resources
+
+__version__ = pkg_resources.require('arvados-cwl-runner')[0].version
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
-
-versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
-try:
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%H', gittaggers.choose_version_from()]).strip()
- with open(versionfile, "w") as f:
- f.write("__version__ = '%s'\n" % gitinfo)
-except Exception as e:
- # When installing from package, it won't be part of a git repository, and
- # check_output() will raise an exception. But the package should include the
- # version file, so we can proceed.
- if not os.path.exists(versionfile):
- raise
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "1.0"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-cwl-runner',
- version='1.0',
+ version=version,
description='Arvados Common Workflow Language runner',
long_description=open(README).read(),
author='Arvados',
README = os.path.join(SETUP_DIR, 'README.rst')
tagger = egg_info_cmd.egg_info
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except (ImportError, OSError):
- pass
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-pam',
- version='0.1',
+ version=version,
description='Arvados PAM module',
long_description=open(README).read(),
author='Arvados',
return body
-def _intercept_http_request(self, uri, method="GET", **kwargs):
+def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- if 'headers' not in kwargs:
- kwargs['headers'] = {}
-
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
- kwargs['headers']['X-External-Client'] = '1'
+ headers['X-External-Client'] = '1'
- kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ if not headers.get('X-Request-Id'):
+ headers['X-Request-Id'] = self._request_id()
retryable = method in [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
for _ in range(retry_count):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
http._retry_delay_initial = RETRY_DELAY_INITIAL
http._retry_delay_backoff = RETRY_DELAY_BACKOFF
http._retry_count = RETRY_COUNT
+ http._request_id = util.new_request_id
return http
# Monkey patch discovery._cast() so objects and arrays get serialized
return None
return cache.SafeHTTPCache(path, max_age=60*60*24*2)
-def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False,
+ request_id=None, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
:version:
:insecure:
If True, ignore SSL certificate validation errors.
+ :request_id:
+ Default X-Request-Id header value for outgoing requests that
+ don't already provide one. If None or omitted, generate a random
+ ID. When retrying failed requests, the same ID is used on all
+ attempts.
+
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
elif host and token:
pass
elif not host and not token:
- return api_from_config(version=version, cache=cache, **kwargs)
+ return api_from_config(
+ version=version, cache=cache, request_id=request_id, **kwargs)
else:
# Caller provided one but not the other
if not host:
svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
svc.api_token = token
svc.insecure = insecure
+ svc.request_id = request_id
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
+ kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
return svc
def api_from_config(version=None, apiconfig=None, **kwargs):
refuse to write *anything* if any of the output files already
exist. As a special case, -f is not needed to write to stdout.
""")
+group.add_argument('-v', action='count', default=0,
+ help="""
+Once for verbose mode, twice for debug mode.
+""")
group.add_argument('--skip-existing', action='store_true',
help="""
Skip files that already exist. The default behavior is to refuse to
stdout = stdout.buffer
args = parse_arguments(arguments, stdout, stderr)
+ logger.setLevel(logging.WARNING - 10 * args.v)
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
r = re.search(r'^(.*?)(/.*)?$', args.locator)
col_loc = r.group(1)
open_flags |= os.O_EXCL
try:
if args.destination == "-":
- write_block_or_manifest(dest=stdout, src=col_loc,
- api_client=api_client, args=args)
+ write_block_or_manifest(
+ dest=stdout, src=col_loc,
+ api_client=api_client, args=args)
else:
out_fd = os.open(args.destination, open_flags)
with os.fdopen(out_fd, 'wb') as out_file:
- write_block_or_manifest(dest=out_file,
- src=col_loc, api_client=api_client,
- args=args)
+ write_block_or_manifest(
+ dest=out_file, src=col_loc,
+ api_client=api_client, args=args)
except (IOError, OSError) as error:
logger.error("can't write to '{}': {}".format(args.destination, error))
return 1
return 0
try:
- reader = arvados.CollectionReader(col_loc, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ col_loc, api_client=api_client, num_retries=args.retries)
except Exception as error:
logger.error("failed to read collection: {}".format(error))
return 1
dest.write(kc.get(src, num_retries=args.retries))
else:
# collection UUID or portable data hash
- reader = arvados.CollectionReader(src, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ src, api_client=api_client, num_retries=args.retries)
dest.write(reader.manifest_text(strip=args.strip_manifest).encode())
total data size).
""")
-_group.add_argument('--silent', action='store_true',
- help="""
-Do not print any debug messages to console. (Any error messages will still be displayed.)
+run_opts.add_argument('--silent', action='store_true',
+ help="""
+Do not print any debug messages to console. (Any error messages will
+still be displayed.)
""")
_group = run_opts.add_mutually_exclusive_group()
}
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
- name=None, owner_uuid=None,
+ name=None, owner_uuid=None, api_client=None,
ensure_unique_name=False, num_retries=None,
put_threads=None, replication_desired=None,
filename=None, update_time=60.0, update_collection=None,
self.replication_desired = replication_desired
self.put_threads = put_threads
self.filename = filename
+ self._api_client = api_client
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
self._current_files = [] # Current run file list
if update_collection and re.match(arvados.util.collection_uuid_pattern,
update_collection):
try:
- self._remote_collection = arvados.collection.Collection(update_collection)
+ self._remote_collection = arvados.collection.Collection(
+ update_collection, api_client=self._api_client)
except arvados.errors.ApiError as error:
raise CollectionUpdateError("Cannot read collection {} ({})".format(update_collection, error))
else:
# No cache file, set empty state
self._state = copy.deepcopy(self.EMPTY_STATE)
# Load the previous manifest so we can check if files were modified remotely.
- self._local_collection = arvados.collection.Collection(self._state['manifest'], replication_desired=self.replication_desired, put_threads=self.put_threads)
+ self._local_collection = arvados.collection.Collection(
+ self._state['manifest'],
+ replication_desired=self.replication_desired,
+ put_threads=self.put_threads,
+ api_client=self._api_client)
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
else:
logger.setLevel(logging.INFO)
status = 0
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
# Determine the name to use
if args.name:
use_cache = args.use_cache,
filename = args.filename,
reporter = reporter,
+ api_client = api_client,
num_retries = args.retries,
replication_desired = args.replication,
put_threads = args.threads,
def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
- download_counter=None, **headers):
+ download_counter=None,
+ headers={}):
self.root = root
self._user_agent_pool = user_agent_pool
self._result = {'error': None}
_logger.debug("{}: {}".format(locator, sorted_roots))
return sorted_roots
- def map_new_services(self, roots_map, locator, force_rebuild, need_writable, **headers):
+ def map_new_services(self, roots_map, locator, force_rebuild, need_writable, headers):
# roots_map is a dictionary, mapping Keep service root strings
# to KeepService objects. Poll for Keep services, and add any
# new ones to roots_map. Return the current list of local
root, self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter,
- **headers)
+ headers=headers)
return local_roots
@staticmethod
return None
@retry.retry_method
- def head(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="HEAD", num_retries=num_retries)
+ def head(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="HEAD", **kwargs)
@retry.retry_method
- def get(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="GET", num_retries=num_retries)
+ def get(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="GET", **kwargs)
- def _get_or_head(self, loc_s, method="GET", num_retries=None):
+ def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None):
"""Get data from Keep.
This method fetches one or more blocks of data from Keep. It
self.misses_counter.add(1)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ }
+
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
roots_map = {
root: self.KeepService(root, self._user_agent_pool,
upload_counter=self.upload_counter,
- download_counter=self.download_counter)
+ download_counter=self.download_counter,
+ headers=headers)
for root in hint_roots
}
sorted_roots = self.map_new_services(
roots_map, locator,
force_rebuild=(tries_left < num_retries),
- need_writable=False)
+ need_writable=False,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
"failed to read {}".format(loc_s), service_errors, label="service")
@retry.retry_method
- def put(self, data, copies=2, num_retries=None):
+ def put(self, data, copies=2, num_retries=None, request_id=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
return loc_s
locator = KeepLocator(loc_s)
- headers = {}
- # Tell the proxy how many copies we want it to store
- headers['X-Keep-Desired-Replicas'] = str(copies)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ 'X-Keep-Desired-Replicas': str(copies),
+ }
roots_map = {}
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
try:
sorted_roots = self.map_new_services(
roots_map, locator,
- force_rebuild=(tries_left < num_retries), need_writable=True, **headers)
+ force_rebuild=(tries_left < num_retries),
+ need_writable=True,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import division
+from builtins import range
+
import fcntl
import hashlib
import httplib2
import os
+import random
import re
import subprocess
import errno
if os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
+
+def new_request_id():
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados-python-client',
- version='0.1',
+ version=version,
description='Arvados client library',
long_description=open(README).read(),
author='Arvados',
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
- if not os.path.exists('tmp/self-signed.pem'):
- # We assume here that either passenger reports its listening
- # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
- # then the certificate won't match the host and reset() will
- # fail certificate verification. If it reports "localhost",
- # clients (notably Python SDK's websocket client) might
- # resolve localhost as ::1 and then fail to connect.
- subprocess.check_call([
- 'openssl', 'req', '-new', '-x509', '-nodes',
- '-out', 'tmp/self-signed.pem',
- '-keyout', 'tmp/self-signed.key',
- '-days', '3650',
- '-subj', '/CN=0.0.0.0'],
- stdout=sys.stderr)
-
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
self.assertEqual(sleep.call_args_list,
[mock.call(RETRY_DELAY_INITIAL)])
+ @mock.patch('time.sleep')
+ def test_same_automatic_request_id_on_retry(self, sleep):
+ self.api._http.orig_http_request.side_effect = (
+ socket.error('mock error'),
+ self.request_success,
+ )
+ self.api.users().current().execute()
+ calls = self.api._http.orig_http_request.call_args_list
+ self.assertEqual(len(calls), 2)
+ self.assertEqual(
+ calls[0][1]['headers']['X-Request-Id'],
+ calls[1][1]['headers']['X-Request-Id'])
+ self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
+
+ @mock.patch('time.sleep')
+ def test_provided_request_id_on_retry(self, sleep):
+ self.api.request_id='fake-request-id'
+ self.api._http.orig_http_request.side_effect = (
+ socket.error('mock error'),
+ self.request_success,
+ )
+ self.api.users().current().execute()
+ calls = self.api._http.orig_http_request.call_args_list
+ self.assertEqual(len(calls), 2)
+ for call in calls:
+ self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
+
@mock.patch('time.sleep')
def test_socket_error_retry_delay(self, sleep):
self.api._http.orig_http_request.side_effect = socket.error('mock')
from __future__ import absolute_import
from future.utils import listitems
import io
+import logging
import mock
import os
import re
self.tempdir = tempfile.mkdtemp()
self.col_loc, self.col_pdh, self.col_manifest = self.write_test_collection()
+ self.stdout = tutil.BytesIO()
+ self.stderr = tutil.StringIO()
+ self.loggingHandler = logging.StreamHandler(self.stderr)
+ self.loggingHandler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ logging.getLogger().addHandler(self.loggingHandler)
+
def tearDown(self):
+ logging.getLogger().removeHandler(self.loggingHandler)
super(ArvadosGetTestCase, self).tearDown()
shutil.rmtree(self.tempdir)
c.manifest_text(strip=strip_manifest))
def run_get(self, args):
- self.stdout = tutil.BytesIO()
- self.stderr = tutil.StringIO()
+ self.stdout.seek(0, 0)
+ self.stdout.truncate(0)
+ self.stderr.seek(0, 0)
+ self.stderr.truncate(0)
return arv_get.main(args, self.stdout, self.stderr)
def test_version_argument(self):
self.assertEqual(0, r)
self.assertEqual(b'', stdout.getvalue())
self.assertFalse(stderr.write.called)
+
+ request_id_regex = r'INFO: X-Request-Id: req-[a-z0-9]{20}\n'
+
+ def test_request_id_logging_on(self):
+ r = self.run_get(["-v", "{}/".format(self.col_loc), self.tempdir])
+ self.assertEqual(0, r)
+ self.assertRegex(self.stderr.getvalue(), self.request_id_regex)
+
+ def test_request_id_logging_off(self):
+ r = self.run_get(["{}/".format(self.col_loc), self.tempdir])
+ self.assertEqual(0, r)
+ self.assertNotRegex(self.stderr.getvalue(), self.request_id_regex)
import datetime
import hashlib
import json
+import logging
import mock
import os
import pwd
Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
def call_main_with_args(self, args):
- self.main_stdout = tutil.StringIO()
- self.main_stderr = tutil.StringIO()
+ self.main_stdout.seek(0, 0)
+ self.main_stdout.truncate(0)
+ self.main_stderr.seek(0, 0)
+ self.main_stderr.truncate(0)
return arv_put.main(args, self.main_stdout, self.main_stderr)
def call_main_on_test_file(self, args=[]):
super(ArvadosPutTest, self).setUp()
run_test_server.authorize_with('active')
arv_put.api_client = None
+ self.main_stdout = tutil.StringIO()
+ self.main_stderr = tutil.StringIO()
+ self.loggingHandler = logging.StreamHandler(self.main_stderr)
+ self.loggingHandler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ logging.getLogger().addHandler(self.loggingHandler)
def tearDown(self):
+ logging.getLogger().removeHandler(self.loggingHandler)
for outbuf in ['main_stdout', 'main_stderr']:
if hasattr(self, outbuf):
getattr(self, outbuf).close()
self.assertLess(0, coll_save_mock.call_count)
self.assertEqual("", self.main_stdout.getvalue())
+ def test_request_id_logging(self):
+ matcher = r'INFO: X-Request-Id: req-[a-z0-9]{20}\n'
+
+ self.call_main_on_test_file()
+ self.assertRegex(self.main_stderr.getvalue(), matcher)
+
+ self.call_main_on_test_file(['--silent'])
+ self.assertNotRegex(self.main_stderr.getvalue(), matcher)
+
class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
ArvadosBaseTestCase):
"""Convert minutes-east-of-UTC to RFC3339- and ISO-compatible time zone designator"""
return '{:+03d}:{:02d}'.format(offset//60, offset%60)
- # Test websocket reconnection on (un)execpted close
+ # Test websocket reconnection on (un)expected close
def _test_websocket_reconnect(self, close_unexpected):
run_test_server.authorize_with('active')
events = queue.Queue(100)
import arvados
import arvados.retry
+import arvados.util
from . import arvados_testutil as tutil
from . import keepstub
from . import run_test_server
self.assertEqual(1, req_mock.call_count)
+@tutil.skip_sleep
+class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def setUp(self):
+ self.api_client = self.mock_keep_services(count=2)
+ self.keep_client = arvados.KeepClient(api_client=self.api_client)
+ self.data = b'xyzzy'
+ self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ self.test_id = arvados.util.new_request_id()
+ self.assertRegex(self.test_id, r'^req-[a-z0-9]{20}$')
+ # If we don't set request_id to None explicitly here, it will
+ # return <MagicMock name='api_client_mock.request_id'
+ # id='123456789'>:
+ self.api_client.request_id = None
+
+ def test_default_to_api_client_request_id(self):
+ self.api_client.request_id = self.test_id
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertProvidedRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ def test_explicit_request_id(self):
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data, request_id=self.test_id)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertProvidedRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator, request_id=self.test_id)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator, request_id=self.test_id)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ def test_automatic_request_id(self):
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertAutomaticRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator)
+ self.assertAutomaticRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator)
+ self.assertAutomaticRequestId(mock.responses[0])
+
+ def assertAutomaticRequestId(self, resp):
+ hdr = [x for x in resp.getopt(pycurl.HTTPHEADER)
+ if x.startswith('X-Request-Id: ')][0]
+ self.assertNotEqual(hdr, 'X-Request-Id: '+self.test_id)
+ self.assertRegex(hdr, r'^X-Request-Id: req-[a-z0-9]{20}$')
+
+ def assertProvidedRequestId(self, resp):
+ self.assertIn('X-Request-Id: '+self.test_id,
+ resp.getopt(pycurl.HTTPHEADER))
+
+
@tutil.skip_sleep
class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
gem 'arvados', '>= 0.1.20150615153458'
gem 'arvados-cli', '>= 0.1.20161017193526'
+gem 'httpclient'
gem 'sshkey'
gem 'safe_yaml'
hashie (3.5.5)
highline (1.7.8)
hike (1.2.3)
+ httpclient (2.8.3)
i18n (0.9.0)
concurrent-ruby (~> 1.0)
jquery-rails (4.2.2)
database_cleaner
factory_girl_rails
faye-websocket
+ httpclient
jquery-rails
lograge
logstash-event
.all
end
@read_auths.select! { |auth| auth.scopes_allow_request? request }
- @read_users = @read_auths.map { |auth| auth.user }.uniq
+ @read_users = @read_auths.map(&:user).uniq
end
def require_login
def find_objects_for_index
if params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name)
- @objects = Collection.readable_by(*@read_users, {include_trash: true, query_on: Collection.unscoped})
+ @objects = Collection.readable_by(*@read_users, {include_trash: true})
end
super
end
end
end.compact
- query_on = if klass == Collection and params[:include_trash]
- klass.unscoped
- else
- klass
- end
- @objects = query_on.readable_by(*@read_users, {:include_trash => params[:include_trash]}).
+ @objects = klass.readable_by(*@read_users, {:include_trash => params[:include_trash]}).
order(request_order).where(where_conds)
klass_limit = limit_all - all_objects.count
def index
expires_in 24.hours, public: true
- discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
+ send_json discovery_doc
+ end
+
+ protected
+
+ def discovery_doc
+ Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
discovery = {
kind: "discovery#restDescription",
crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
+ remoteHosts: Rails.configuration.remote_hosts,
+ remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
end
discovery
end
- send_json discovery
end
end
return redirect_to login_failure_url
end
- user = User.find_by_identity_url(omniauth['info']['identity_url'])
+ # Only local users can create sessions, hence uuid_like_pattern
+ # here.
+ user = User.where('identity_url = ? and uuid like ?',
+ omniauth['info']['identity_url'],
+ User.uuid_like_pattern).first
if not user
# Check for permission to log in to an existing User record with
# a different identity_url
end
def call env
- # First, clean up just in case we have a multithreaded server and thread
- # local variables are still set from a prior request. Also useful for
- # tests that call this code to set up the environment.
- Thread.current[:api_client_ip_address] = nil
- Thread.current[:api_client_authorization] = nil
- Thread.current[:api_client_uuid] = nil
- Thread.current[:api_client] = nil
- Thread.current[:user] = nil
-
request = Rack::Request.new(env)
params = request.params
remote_ip = env["action_dispatch.remote_ip"]
Thread.current[:request_starttime] = Time.now
- user = nil
- api_client = nil
- api_client_auth = nil
- if request.get? || params["_method"] == 'GET'
+
+ remote = false
+ reader_tokens = nil
+ if params["remote"] && request.get? && (
+ request.path.start_with?('/arvados/v1/groups') ||
+ request.path.start_with?('/arvados/v1/users/current'))
+ # Request from a remote API server, asking to validate a salted
+ # token.
+ remote = params["remote"]
+ elsif request.get? || params["_method"] == 'GET'
reader_tokens = params["reader_tokens"]
if reader_tokens.is_a? String
reader_tokens = SafeJSON.load(reader_tokens)
end
- else
- reader_tokens = nil
end
# Set current_user etc. based on the primary session token if a
# valid one is present. Otherwise, use the first valid token in
# reader_tokens.
+ auth = nil
[params["api_token"],
params["oauth_token"],
- env["HTTP_AUTHORIZATION"].andand.match(/OAuth2 ([a-zA-Z0-9]+)/).andand[1],
+ env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([-\/a-zA-Z0-9]+)/).andand[2],
*reader_tokens,
].each do |supplied|
next if !supplied
try_auth = ApiClientAuthorization.
- includes(:api_client, :user).
- where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', supplied).
- first
+ validate(token: supplied, remote: remote)
if try_auth.andand.user
- api_client_auth = try_auth
- user = api_client_auth.user
- api_client = api_client_auth.api_client
+ auth = try_auth
break
end
end
+
Thread.current[:api_client_ip_address] = remote_ip
- Thread.current[:api_client_authorization] = api_client_auth
- Thread.current[:api_client_uuid] = api_client.andand.uuid
- Thread.current[:api_client] = api_client
- Thread.current[:user] = user
+ Thread.current[:api_client_authorization] = auth
+ Thread.current[:api_client_uuid] = auth.andand.api_client.andand.uuid
+ Thread.current[:api_client] = auth.andand.api_client
+ Thread.current[:user] = auth.andand.user
@app.call env if @app
end
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ extend CurrentApiClient
belongs_to :api_client
belongs_to :user
t.add :owner_uuid
t.add :user_id
t.add :api_client_id
+ # NB the "api_token" db column is a misnomer in that it's only the
+ # "secret" part of a token: a v1 token is just the secret, but a
+ # v2 token is "v2/uuid/secret".
t.add :api_token
t.add :created_by_ip_address
t.add :default_owner_uuid
["#{table_name}.id desc"]
end
+ def self.remote_host(uuid_prefix:)
+ Rails.configuration.remote_hosts[uuid_prefix] ||
+ (Rails.configuration.remote_hosts_via_dns &&
+ uuid_prefix+".arvadosapi.com")
+ end
+
+ def self.validate(token:, remote:)
+ return nil if !token
+ remote ||= Rails.configuration.uuid_prefix
+
+ case token[0..2]
+ when 'v2/'
+ _, uuid, secret = token.split('/')
+ unless uuid.andand.length == 27 && secret.andand.length.andand > 0
+ return nil
+ end
+
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('uuid=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', uuid).
+ first
+ if auth && auth.user &&
+ (secret == auth.api_token ||
+ secret == OpenSSL::HMAC.hexdigest('sha1', auth.api_token, remote))
+ return auth
+ end
+
+ uuid_prefix = uuid[0..4]
+ if uuid_prefix == Rails.configuration.uuid_prefix
+ # If the token were valid, we would have validated it above
+ return nil
+ elsif uuid_prefix.length != 5
+ # malformed
+ return nil
+ end
+
+ host = remote_host(uuid_prefix: uuid_prefix)
+ if !host
+ Rails.logger.warn "remote authentication rejected: no host for #{uuid_prefix.inspect}"
+ return nil
+ end
+
+ # Token was issued by a different cluster. If it's expired or
+ # missing in our database, ask the originating cluster to
+ # [re]validate it.
+ begin
+ clnt = HTTPClient.new
+ if Rails.configuration.sso_insecure
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ end
+ remote_user = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/users/current',
+ {'remote' => Rails.configuration.uuid_prefix},
+ {'Authorization' => 'Bearer ' + token}))
+ rescue => e
+ Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}"
+ return nil
+ end
+ if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String) || remote_user['uuid'][0..4] != uuid[0..4]
+ Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}"
+ return nil
+ end
+ act_as_system_user do
+ # Add/update user and token in our database so we can
+ # validate subsequent requests faster.
+
+ user = User.find_or_create_by(uuid: remote_user['uuid']) do |user|
+ # (this block runs for the "create" case, not for "find")
+ user.is_admin = false
+ user.email = remote_user['email']
+ if remote_user['username'].andand.length.andand > 0
+ user.set_initial_username(requested: remote_user['username'])
+ end
+ end
+
+ if Rails.configuration.new_users_are_active
+ # Update is_active to whatever it is at the remote end
+ user.is_active = remote_user['is_active']
+ elsif !remote_user['is_active']
+ # Remote user is inactive; our mirror should be, too.
+ user.is_active = false
+ end
+
+ %w[first_name last_name email prefs].each do |attr|
+ user.send(attr+'=', remote_user[attr])
+ end
+
+ user.save!
+
+ auth = ApiClientAuthorization.find_or_create_by(uuid: uuid) do |auth|
+ auth.user = user
+ auth.api_token = secret
+ auth.api_client_id = 0
+ end
+
+ # Accept this token (and don't reload the user record) for
+ # 5 minutes. TODO: Request the actual api_client_auth
+ # record from the remote server in case it wants the token
+ # to expire sooner.
+ auth.update_attributes!(expires_at: Time.now + 5.minutes)
+ end
+ return auth
+ else
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token).
+ first
+ if auth && auth.user
+ return auth
+ end
+ end
+ return nil
+ end
+
protected
def permission_to_create
# Collect the UUIDs of the authorized users.
sql_table = kwargs.fetch(:table_name, table_name)
include_trash = kwargs.fetch(:include_trash, false)
- query_on = kwargs.fetch(:query_on, self)
sql_conds = []
user_uuids = users_list.map { |u| u.uuid }
+ exclude_trashed_records = if !include_trash and (sql_table == "groups" or sql_table == "collections") then
+ # Only include records that are not explicitly trashed
+ "AND #{sql_table}.is_trashed = false"
+ else
+ ""
+ end
+
if users_list.select { |u| u.is_admin }.any?
if !include_trash
- # exclude rows that are explicitly trashed.
if sql_table != "api_client_authorizations"
- sql_conds.push "NOT EXISTS(SELECT 1
- FROM #{PERMISSION_VIEW}
- WHERE trashed = 1 AND
- (#{sql_table}.uuid = target_uuid OR #{sql_table}.owner_uuid = target_uuid))"
+ # Exclude rows where the owner is trashed
+ sql_conds.push "NOT EXISTS(SELECT 1 "+
+ "FROM #{PERMISSION_VIEW} "+
+ "WHERE trashed = 1 AND "+
+ "(#{sql_table}.owner_uuid = target_uuid)) "+
+ exclude_trashed_records
end
end
else
- if include_trash
- trashed_check = ""
- else
- trashed_check = "AND trashed = 0"
- end
-
- if sql_table != "api_client_authorizations" and sql_table != "groups"
- owner_check = "OR (target_uuid = #{sql_table}.owner_uuid AND target_owner_uuid IS NOT NULL)"
- else
- owner_check = ""
- end
+ trashed_check = if !include_trash then
+ "AND trashed = 0"
+ else
+ ""
+ end
+
+ owner_check = if sql_table != "api_client_authorizations" and sql_table != "groups" then
+ "OR (target_uuid = #{sql_table}.owner_uuid AND target_owner_uuid IS NOT NULL)"
+ else
+ ""
+ end
sql_conds.push "EXISTS(SELECT 1 FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND (target_uuid = #{sql_table}.uuid #{owner_check}))"
+ "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND (target_uuid = #{sql_table}.uuid #{owner_check})) "+
+ exclude_trashed_records
if sql_table == "links"
# Match any permission link that gives one of the authorized
end
end
- query_on.where(sql_conds.join(' OR '),
+ self.where(sql_conds.join(' OR '),
user_uuids: user_uuids,
permission_link_classes: ['permission', 'resources'])
end
def self.full_text_searchable_columns
self.columns.select do |col|
- col.type == :string or col.type == :text
+ [:string, :text, :jsonb].include?(col.type)
end.map(&:name)
end
def self.full_text_tsvector
parts = full_text_searchable_columns.collect do |column|
- "coalesce(#{column},'')"
+ cast = serialized_attributes[column] ? '::text' : ''
+ "coalesce(#{column}#{cast},'')"
end
"to_tsvector('english', #{parts.join(" || ' ' || ")})"
end
if self == ArvadosModel
# If called directly as ArvadosModel.find_by_uuid rather than via subclass,
# delegate to the appropriate subclass based on the given uuid.
- self.resource_class_for_uuid(uuid).unscoped.find_by_uuid(uuid)
+ self.resource_class_for_uuid(uuid).find_by_uuid(uuid)
else
super
end
validate :ensure_pdh_matches_manifest_text
before_save :set_file_names
- # Query only untrashed collections by default.
- default_scope { where("is_trashed = false") }
-
api_accessible :user, extend: :common do |t|
t.add :name
t.add :description
# that a container cannot "claim" a collection that it doesn't otherwise
# have access to just by setting the output field to the collection PDH.
if output_changed?
- c = Collection.unscoped do
- Collection.
- readable_by(current_user).
+ c = Collection.
+ readable_by(current_user, {include_trash: true}).
where(portable_data_hash: self.output).
first
- end
if !c
errors.add :output, "collection must exist and be readable by current user."
end
trash_at = db_current_time + self.output_ttl
end
end
- manifest = Collection.unscoped do
- Collection.where(portable_data_hash: pdh).first.manifest_text
- end
+ manifest = Collection.where(portable_data_hash: pdh).first.manifest_text
coll = Collection.new(owner_uuid: owner_uuid,
manifest_text: manifest,
# original job reuse behavior, and is still the default).
reuse_job_if_outputs_differ: false
+ ###
+ ### Federation support.
+ ###
+
+ # You can enable use of this cluster by users who are authenticated
+ # by a remote Arvados site. Control which remote hosts are trusted
+ # to authenticate which user IDs by configuring remote_hosts,
+ # remote_hosts_via_dns, or both. The default configuration disables
+ # remote authentication.
+
+ # Map known prefixes to hosts. For example, if user IDs beginning
+ # with "zzzzz-" should be authenticated by the Arvados server at
+ # "zzzzz.example.com", use:
+ #
+ # remote_hosts:
+ # zzzzz: zzzzz.example.com
+ remote_hosts: {}
+
+ # Use {prefix}.arvadosapi.com for any prefix not given in
+ # remote_hosts above.
+ remote_hosts_via_dns: false
+
###
### Remaining assorted configuration options.
###
arvados_theme: default
- # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
- # On (sso) server. Should only be enabled during development when the SSO
- # server is using a self-signed cert.
+ # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the
+ # Single Sign On (sso) server and remote Arvados sites. Should only
+ # be enabled during development when the SSO server is using a
+ # self-signed cert.
sso_insecure: false
## Set Time.zone default to the specified zone and make Active
}
end
+ def replace_index(t)
+ i = fts_indexes[t]
+ t.classify.constantize.reset_column_information
+ execute "DROP INDEX IF EXISTS #{i}"
+ execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector})"
+ end
+
def up
- # remove existing fts indexes and create up to date ones with no leading space
- fts_indexes.each do |t, i|
- t.classify.constantize.reset_column_information
- ActiveRecord::Base.connection.indexes(t).each do |idx|
- if idx.name == i
- remove_index t.to_sym, :name => i
- break
- end
- end
- execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector});"
+ # remove existing fts indexes and create up to date ones with no
+ # leading space
+ fts_indexes.keys.each do |t|
+ replace_index(t)
end
end
--- /dev/null
+require 'migrate_yaml_to_json'
+
+class YamlToJson < ActiveRecord::Migration
+ def up
+ [
+ ['collections', 'properties'],
+ ['containers', 'environment'],
+ ['containers', 'mounts'],
+ ['containers', 'runtime_constraints'],
+ ['containers', 'command'],
+ ['containers', 'scheduling_parameters'],
+ ['container_requests', 'properties'],
+ ['container_requests', 'environment'],
+ ['container_requests', 'mounts'],
+ ['container_requests', 'runtime_constraints'],
+ ['container_requests', 'command'],
+ ['container_requests', 'scheduling_parameters'],
+ ['humans', 'properties'],
+ ['job_tasks', 'parameters'],
+ ['links', 'properties'],
+ ['nodes', 'info'],
+ ['nodes', 'properties'],
+ ['pipeline_instances', 'components'],
+ ['pipeline_instances', 'properties'],
+ ['pipeline_instances', 'components_summary'],
+ ['pipeline_templates', 'components'],
+ ['specimens', 'properties'],
+ ['traits', 'properties'],
+ ['users', 'prefs'],
+ ].each do |table, column|
+ MigrateYAMLToJSON.migrate(table, column)
+ end
+ end
+
+ def down
+ end
+end
--- /dev/null
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class JsonCollectionProperties < ActiveRecord::Migration
+ def up
+ # Drop the FT index before changing column type to avoid
+ # "PG::DatatypeMismatch: ERROR: COALESCE types jsonb and text
+ # cannot be matched".
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
+ FullTextSearchIndexes.new.replace_index('collections')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE text'
+ end
+end
--- /dev/null
+class FixTrashFlagFollow < ActiveRecord::Migration
+ def change
+ ActiveRecord::Base.connection.execute("DROP MATERIALIZED VIEW materialized_permission_view")
+ ActiveRecord::Base.connection.execute(
+"-- constructing perm_edges
+-- 1. get the list of all permission links,
+-- 2. any can_manage link or permission link to a group means permission should 'follow through'
+-- (as a special case, can_manage links to a user grant access to everything owned by the user,
+-- unlike can_read or can_write which only grant access to the user record)
+-- 3. add all owner->owned relationships between groups as can_manage edges
+--
+-- constructing permissions
+-- 1. base case: start with set of all users as the working set
+-- 2. recursive case:
+-- join with edges where the tail is in the working set and 'follow' is true
+-- produce a new working set with the head (target) of each edge
+-- set permission to the least permission encountered on the path
+-- propagate trashed flag down
+
+CREATE MATERIALIZED VIEW materialized_permission_view AS
+WITH RECURSIVE
+perm_value (name, val) AS (
+ VALUES
+ ('can_read', 1::smallint),
+ ('can_login', 1),
+ ('can_write', 2),
+ ('can_manage', 3)
+ ),
+perm_edges (tail_uuid, head_uuid, val, follow, trashed) AS (
+ SELECT links.tail_uuid,
+ links.head_uuid,
+ pv.val,
+ (pv.val = 3 OR groups.uuid IS NOT NULL) AS follow,
+ 0::smallint AS trashed,
+ 0::smallint AS followtrash
+ FROM links
+ LEFT JOIN perm_value pv ON pv.name = links.name
+ LEFT JOIN groups ON pv.val<3 AND groups.uuid = links.head_uuid
+ WHERE links.link_class = 'permission'
+ UNION ALL
+ SELECT owner_uuid, uuid, 3, true,
+ CASE WHEN trash_at IS NOT NULL and trash_at < clock_timestamp() THEN 1 ELSE 0 END,
+ 1
+ FROM groups
+ ),
+perm (val, follow, user_uuid, target_uuid, trashed) AS (
+ SELECT 3::smallint AS val,
+ true AS follow,
+ users.uuid::varchar(32) AS user_uuid,
+ users.uuid::varchar(32) AS target_uuid,
+ 0::smallint AS trashed
+ FROM users
+ UNION
+ SELECT LEAST(perm.val, edges.val)::smallint AS val,
+ edges.follow AS follow,
+ perm.user_uuid::varchar(32) AS user_uuid,
+ edges.head_uuid::varchar(32) AS target_uuid,
+ (GREATEST(perm.trashed, edges.trashed) * edges.followtrash)::smallint AS trashed
+ FROM perm
+ INNER JOIN perm_edges edges
+ ON perm.follow AND edges.tail_uuid = perm.target_uuid
+)
+SELECT user_uuid,
+ target_uuid,
+ MAX(val) AS perm_level,
+ CASE follow WHEN true THEN target_uuid ELSE NULL END AS target_owner_uuid,
+ MAX(trashed) AS trashed
+ FROM perm
+ GROUP BY user_uuid, target_uuid, target_owner_uuid;
+")
+ add_index :materialized_permission_view, [:trashed, :target_uuid], name: 'permission_target_trashed'
+ add_index :materialized_permission_view, [:user_uuid, :trashed, :perm_level], name: 'permission_target_user_trashed_level'
+ ActiveRecord::Base.connection.execute("REFRESH MATERIALIZED VIEW materialized_permission_view")
+ end
+end
manifest_text text,
name character varying(255),
description character varying(524288),
- properties text,
+ properties jsonb,
delete_at timestamp without time zone,
file_names character varying(8192),
trash_at timestamp without time zone,
links.head_uuid,
pv.val,
((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
- (0)::smallint AS trashed
+ (0)::smallint AS trashed,
+ (0)::smallint AS followtrash
FROM ((links
LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
LEFT JOIN groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
CASE
WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
ELSE 0
- END AS "case"
+ END AS "case",
+ 1
FROM groups
), perm(val, follow, user_uuid, target_uuid, trashed) AS (
SELECT (3)::smallint AS val,
edges.follow,
perm_1.user_uuid,
(edges.head_uuid)::character varying(32) AS target_uuid,
- (GREATEST((perm_1.trashed)::integer, edges.trashed))::smallint AS trashed
+ ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
FROM (perm perm_1
JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
)
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+
+INSERT INTO schema_migrations (version) VALUES ('20170706141334');
+
INSERT INTO schema_migrations (version) VALUES ('20170824202826');
INSERT INTO schema_migrations (version) VALUES ('20170906224040');
INSERT INTO schema_migrations (version) VALUES ('20171027183824');
+INSERT INTO schema_migrations (version) VALUES ('20171208203841');
end
def self.load(s)
- if s.nil?
+ if s.is_a?(object_class)
+ # Rails already deserialized for us
+ s
+ elsif s.nil?
object_class.new()
elsif s[0] == first_json_char
SafeJSON.load(s)
def self.sweep_now
act_as_system_user do
- Collection.unscoped.
+ Collection.
where('delete_at is not null and delete_at < statement_timestamp()').
destroy_all
- Collection.unscoped.
+ Collection.
where('is_trashed = false and trash_at < statement_timestamp()').
update_all('is_trashed = true')
end
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_equal c.delete_at, c.trash_at + Rails.configuration.blob_signature_ttl
end
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_operator c.delete_at, :<, db_current_time
end
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_operator c.delete_at, :>=, time_before_trashing + Rails.configuration.default_trash_lifetime
end
assert_response :success
assert_not_nil Group.readable_by(users(auth)).where(uuid: groups(:trashed_subproject).uuid).first
end
-
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'webrick'
+require 'webrick/https'
+require 'test_helper'
+require 'helpers/users_test_helper'
+
+class RemoteUsersTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
+
+ def salted_active_token(remote:)
+ salt_token(fixture: :active, remote: remote).sub('/zzzzz-', '/'+remote+'-')
+ end
+
+ def auth(remote:)
+ token = salted_active_token(remote: remote)
+ {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ end
+
+ # For remote authentication tests, we bring up a simple stub server
+ # (on a port chosen by webrick) and configure the SUT so the stub is
+ # responsible for clusters "zbbbb" (a well-behaved cluster) and
+ # "zbork" (a misbehaving cluster).
+ #
+ # Test cases can override the stub's default response to
+ # .../users/current by changing @stub_status and @stub_content.
+ setup do
+ clnt = HTTPClient.new
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ HTTPClient.stubs(:new).returns clnt
+
+ @controller = Arvados::V1::UsersController.new
+ ready = Thread::Queue.new
+ srv = WEBrick::HTTPServer.new(
+ Port: 0,
+ Logger: WEBrick::Log.new(
+ Rails.root.join("log", "webrick.log").to_s,
+ WEBrick::Log::INFO),
+ AccessLog: [[File.open(Rails.root.join(
+ "log", "webrick_access.log").to_s, 'a+'),
+ WEBrick::AccessLog::COMBINED_LOG_FORMAT]],
+ SSLEnable: true,
+ SSLVerifyClient: OpenSSL::SSL::VERIFY_NONE,
+ SSLPrivateKey: OpenSSL::PKey::RSA.new(
+ File.open(Rails.root.join("tmp", "self-signed.key")).read),
+ SSLCertificate: OpenSSL::X509::Certificate.new(
+ File.open(Rails.root.join("tmp", "self-signed.pem")).read),
+ SSLCertName: [["CN", WEBrick::Utils::getservername]],
+ StartCallback: lambda { ready.push(true) })
+ srv.mount_proc '/discovery/v1/apis/arvados/v1/rest' do |req, res|
+ Rails.cache.delete 'arvados_v1_rest_discovery'
+ res.body = Arvados::V1::SchemaController.new.send(:discovery_doc).to_json
+ end
+ srv.mount_proc '/arvados/v1/users/current' do |req, res|
+ res.status = @stub_status
+ res.body = @stub_content.is_a?(String) ? @stub_content : @stub_content.to_json
+ end
+ Thread.new do
+ srv.start
+ end
+ ready.pop
+ @remote_server = srv
+ @remote_host = "127.0.0.1:#{srv.config[:Port]}"
+ Rails.configuration.remote_hosts['zbbbb'] = @remote_host
+ Rails.configuration.remote_hosts['zbork'] = @remote_host
+ Arvados::V1::SchemaController.any_instance.stubs(:root_url).returns "https://#{@remote_host}"
+ @stub_status = 200
+ @stub_content = {
+ uuid: 'zbbbb-tpzed-000000000000000',
+ email: 'foo@example.com',
+ username: 'barney',
+ is_admin: true,
+ is_active: true,
+ }
+ end
+
+ teardown do
+ @remote_server.andand.stop
+ end
+
+ test 'authenticate with remote token' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
+ assert_equal false, json_response['is_admin']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+
+ # revoke original token
+ @stub_status = 401
+
+ # re-authorize before cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+
+ # simulate cache expiry
+ ApiClientAuthorization.where(
+ uuid: salted_active_token(remote: 'zbbbb').split('/')[1]).
+ update_all(expires_at: db_current_time - 1.minute)
+
+ # re-authorize after cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+
+ # revive original token and re-authorize
+ @stub_status = 200
+ @stub_content[:username] = 'blarney'
+ @stub_content[:email] = 'blarney@example.com'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'barney', json_response['username'], 'local username should not change once assigned'
+ assert_equal 'blarney@example.com', json_response['email']
+ end
+
+ test 'authenticate with remote token, remote username conflicts with local' do
+ @stub_content[:username] = 'active'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'active2', json_response['username']
+ end
+
+ test 'authenticate with remote token, remote username is nil' do
+ @stub_content.delete :username
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'foo', json_response['username']
+ end
+
+ test 'authenticate with remote token from misbhehaving remote cluster' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbork')
+ assert_response 401
+ end
+
+ test 'authenticate with remote token that fails validate' do
+ @stub_status = 401
+ @stub_content = {
+ error: 'not authorized',
+ }
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['v2',
+ 'v2/',
+ 'v2//',
+ 'v2///',
+ "v2/'; delete from users where 1=1; commit; select '/lol",
+ 'v2/foo/bar',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1/',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2//3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v8/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '"v2/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"',
+ '/',
+ '//',
+ '///',
+ ].each do |token|
+ test "authenticate with malformed remote token #{token}" do
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response 401
+ end
+ end
+
+ test "ignore extra fields in remote token" do
+ token = salted_active_token(remote: 'zbbbb') + '/foo/bar/baz/*'
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response :success
+ end
+
+ test 'remote api server is not an api server' do
+ @stub_status = 200
+ @stub_content = '<html>bad</html>'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['zbbbb', 'z0000'].each do |token_valid_for|
+ test "validate #{token_valid_for}-salted token for remote cluster zbbbb" do
+ salted_token = salt_token(fixture: :active, remote: token_valid_for)
+ get '/arvados/v1/users/current', {format: 'json', remote: 'zbbbb'}, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ if token_valid_for == 'zbbbb'
+ assert_response 200
+ assert_equal(users(:active).uuid, json_response['uuid'])
+ else
+ assert_response 401
+ end
+ end
+ end
+
+ test "list readable groups with salted token" do
+ salted_token = salt_token(fixture: :active, remote: 'zbbbb')
+ get '/arvados/v1/groups', {
+ format: 'json',
+ remote: 'zbbbb',
+ limit: 10000,
+ }, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ assert_response 200
+ group_uuids = json_response['items'].collect { |i| i['uuid'] }
+ assert_includes(group_uuids, 'zzzzz-j7d0g-fffffffffffffff')
+ refute_includes(group_uuids, 'zzzzz-j7d0g-000000000000000')
+ assert_includes(group_uuids, groups(:aproject).uuid)
+ refute_includes(group_uuids, groups(:trashed_project).uuid)
+ refute_includes(group_uuids, groups(:testusergroup_admins).uuid)
+ end
+end
"HTTP_AUTHORIZATION" => "OAuth2 #{t}")
end
+ def salt_token(fixture:, remote:)
+ auth = api_client_authorizations(fixture)
+ uuid = auth.uuid
+ token = auth.api_token
+ hmac = OpenSSL::HMAC.hexdigest('sha1', token, remote)
+ return "v2/#{uuid}/#{hmac}"
+ end
+
def self.skip_slow_tests?
!(ENV['RAILS_TEST_SHORT'] || '').empty?
end
assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
end
+
+ test 'jsonb column' do
+ set_user_from_auth :active
+
+ c = Collection.create!(properties: {})
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {'foo' => 'foo'})
+ c.reload
+ assert_equal({'foo' => 'foo'}, c.properties)
+
+ c.update_attributes(properties: nil)
+ c.reload
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {foo: 'bar'})
+ assert_equal({'foo' => 'bar'}, c.properties)
+ c.reload
+ assert_equal({'foo' => 'bar'}, c.properties)
+ end
end
assert c.valid?
uuid = c.uuid
+ c = Collection.readable_by(current_user).where(uuid: uuid)
+ assert_not_empty c, 'Should be able to find live collection'
+
# mark collection as expired
- c.update_attributes!(trash_at: Time.new.strftime("%Y-%m-%d"))
- c = Collection.where(uuid: uuid)
+ c.first.update_attributes!(trash_at: Time.new.strftime("%Y-%m-%d"))
+ c = Collection.readable_by(current_user).where(uuid: uuid)
assert_empty c, 'Should not be able to find expired collection'
# recreate collection with the same name
if fixture_name == :expired_collection
# Fixture-finder shorthand doesn't find trashed collections
# because they're not in the default scope.
- c = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3ih')
+ c = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3ih')
else
c = collections(fixture_name)
end
end
end
SweepTrashedCollections.sweep_now
- c = Collection.unscoped.where('uuid=? and is_trashed=true', c.uuid).first
+ c = Collection.where('uuid=? and is_trashed=true', c.uuid).first
assert c
act_as_user users(:active) do
assert Collection.create!(owner_uuid: c.owner_uuid,
test "delete in SweepTrashedCollections" do
uuid = 'zzzzz-4zz18-3u1p5umicfpqszp' # deleted_on_next_sweep
- assert_not_empty Collection.unscoped.where(uuid: uuid)
+ assert_not_empty Collection.where(uuid: uuid)
SweepTrashedCollections.sweep_now
- assert_empty Collection.unscoped.where(uuid: uuid)
+ assert_empty Collection.where(uuid: uuid)
end
test "delete referring links in SweepTrashedCollections" do
name: 'something')
end
past = db_current_time
- Collection.unscoped.where(uuid: uuid).
+ Collection.where(uuid: uuid).
update_all(is_trashed: true, trash_at: past, delete_at: past)
- assert_not_empty Collection.unscoped.where(uuid: uuid)
+ assert_not_empty Collection.where(uuid: uuid)
SweepTrashedCollections.sweep_now
- assert_empty Collection.unscoped.where(uuid: uuid)
+ assert_empty Collection.where(uuid: uuid)
end
end
c.lock
c.update_attributes! state: Container::Running
- output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
+ output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
assert output.is_trashed
assert c.update_attributes output: output.portable_data_hash
c.lock
c.update_attributes! state: Container::Running
- output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
+ output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
assert Group.readable_by(users(:active)).where(uuid: g_baz.uuid).any?
end
+ test "trashed does not propagate across permission links" do
+ set_user_from_auth :admin
+
+ g_foo = Group.create!(name: "foo")
+ u_bar = User.create!(first_name: "bar")
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+
+ g_foo.update! is_trashed: false
+ ln = Link.create!(tail_uuid: g_foo.uuid,
+ head_uuid: u_bar.uuid,
+ link_class: "permission",
+ name: "can_read")
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ end
+
end
from setuptools import setup, find_packages
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name="arvados-docker-cleaner",
- version="0.1",
+ version=version,
description="Arvados Docker cleaner",
author="Arvados",
author_email="info@arvados.org",
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados_fuse',
- version='0.1',
+ version=version,
description='Arvados FUSE driver',
long_description=open(README).read(),
author='Arvados',
SETUP_DIR = os.path.dirname(__file__) or "."
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-node-manager',
- version='0.1',
+ version=version,
description='Arvados compute node manager',
long_description=open(README).read(),
author='Arvados',
yarn install
if test "$1" != "--only-deps" ; then
- echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/arvados-configuration.yml
+ echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
exec ng serve --host 0.0.0.0 --port 4200 --env=webdev
fi
SETUP_DIR = os.path.dirname(__file__) or '.'
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='crunchstat_summary',
- version='0.1',
+ version=version,
description='read crunch log files and summarize resource usage',
author='Arvados',
author_email='info@arvados.org',
projects / arvados.git / commitdiff