Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>
sdk/java/log
tmp
sdk/cli/binstubs/
-sdk/cwl/arvados_cwl/_version.py
services/api/config/arvados-clients.yml
*#*
.DS_Store
txt += stamp + "Container "+eventData.object_uuid+" started\n";
break;
case "Complete":
- var outcome = eventData.properties.new_attributes.exit_code === 0 ? "success" : "failure";
- txt += stamp + "Container "+eventData.object_uuid+" finished with exit code " +
- eventData.properties.new_attributes.exit_code +
- " ("+outcome+")\n";
+ txt += stamp + "Container "+eventData.object_uuid+" finished\n";
break;
case "Cancelled":
txt += stamp + "Container "+eventData.object_uuid+" was cancelled\n";
end
%>
- <% if check_trash.andand.any? %>
- <h2>Trashed</h2>
-
- <% object = check_trash.first %>
+ <% untrash_object = nil %>
+ <% if check_trash.andand.any? %>
+ <% object = check_trash.first %>
+ <% if object.respond_to?(:is_trashed) && object.is_trashed %>
<% untrash_object = object %>
- <% while !untrash_object.is_trashed %>
- <% owner = Group.where(uuid: untrash_object.owner_uuid).include_trash(true).first %>
- <% if owner.nil? then %>
+ <% else %>
+ <% owner = object %>
+ <% while true %>
+ <% owner = Group.where(uuid: owner.owner_uuid).include_trash(true).first %>
+ <% if owner.nil? %>
<% break %>
- <% else %>
+ <% end %>
+ <% if owner.is_trashed %>
<% untrash_object = owner %>
+ <% break %>
<% end %>
<% end %>
+ <% end %>
+ <% end %>
+
+ <% if !untrash_object.nil? %>
+ <h2>Trashed</h2>
<% untrash_name = if !untrash_object.name.blank? then
"'#{untrash_object.name}'"
</li>
<% if current_user.is_active %>
<li role="menuitem"><a href="/projects/<%=current_user.uuid%>" role="menuitem"><i class="fa fa-lg fa-home fa-fw"></i> Home project </a></li>
-
+ <% if Rails.configuration.composer_url %>
+ <li role="menuitem">
+ <%= link_to Rails.configuration.composer_url, role: 'menu-item' do %>
+ <i class="fa fa-lg fa-share-alt fa-fw"></i> Workflow Composer
+ <% end %>
+ </li>
+ <% end %>
<li role="menuitem">
<%= link_to virtual_machines_user_path(current_user), role: 'menu-item' do %>
<i class="fa fa-lg fa-terminal fa-fw"></i> Virtual machines
<i class="fa fa-lg fa-key fa-fw"></i> SSH keys
<% end %>
</li>
-
<% if Rails.configuration.user_profile_form_fields %>
<li role="menuitem"><a href="/users/<%=current_user.uuid%>/profile" role="menuitem"><i class="fa fa-lg fa-user fa-fw"></i> Manage profile</a></li>
<% end %>
# Example:
# multi_site_search: https://workbench.qr1hi.arvadosapi.com/collections/multisite
multi_site_search: false
+
+ #
+ # Link to use for Arvados Workflow Composer app, or false if not available.
+ #
+ composer_url: false
\ No newline at end of file
# out of the popup now and should be back in the error page
assert_text 'fiddlesticks'
end
+
+ test "showing a trashed collection UUID gives untrash button" do
+ visit(page_with_token("active", "/collections/zzzzz-4zz18-trashedproj2col"))
+ assert(page.has_text?(/You must untrash the owner project to access this/i),
+ "missing untrash instructions")
+ end
+
+ test "showing a trashed container request gives untrash button" do
+ visit(page_with_token("active", "/container_requests/zzzzz-xvhdp-cr5trashedcontr"))
+ assert(page.has_text?(/You must untrash the owner project to access this/i),
+ "missing untrash instructions")
+ end
+
end
old_attributes: {state: 'Running'},
new_attributes: {state: 'Complete', exit_code: 1},
},
- }, "Container #{c['uuid']} finished with exit code 1 (failure)"],
+ }, "Container #{c['uuid']} finished"],
# It's unrealistic for state to change again once it's Complete,
# but the logging code doesn't care, so we do it to keep the test
# simple.
ubuntu1204,ubuntu1404|requests|2.4.3|2|python|all
ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
ubuntu1204,centos7|isodate|0.5.4|2|python|all
-centos7|daemon|2.1.1|2|python|all
+centos7|python-daemon|2.1.2|1|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
python_sdk_ts=$(cd sdk/python && timestamp_from_git)
cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)-2
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)-3
+python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
+cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)-3
- gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/python)
-else
- gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/cwl)
+ cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=$python_sdk_version \
- --build-arg cwl_runner_version=$cwl_runner_version \
- -t arvados/jobs:$gittag .
+ --build-arg python_sdk_version=${python_sdk_version}-2 \
+ --build-arg cwl_runner_version=${cwl_runner_version}-3 \
+ -t arvados/jobs:$cwl_runner_version .
ECODE=$?
FORCE=-f
fi
-docker tag $FORCE arvados/jobs:$gittag arvados/jobs:latest
+docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:latest
ECODE=$?
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs:$gittag
+ docker_push arvados/jobs:$cwl_runner_version
docker_push arvados/jobs:latest
title "upload arvados images finished (`timer`)"
else
Run package install tests
--debug
Output debug information (default: false)
+--build-version <string>
+ Version to build (default:
+ \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
+ 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--only-test)
ONLY_TEST="$1 $2"; shift
;;
+ --build-version)
+ ARVADOS_BUILDING_VERSION="$2"; shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" --build-version "$ARVADOS_BUILDING_VERSION" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
true
else
FINAL_EXITCODE=$?
Build only a specific package
--only-test <package>
Test only a specific package
+--build-version <string>
+ Version to build (default:
+ \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
+ 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test:,only-build: \
+ help,debug,test-packages,target:,command:,only-test:,only-build:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--test-packages)
test_packages=1
;;
+ --build-version)
+ if [[ -z "$2" ]]; then
+ :
+ elif ! [[ "$2" =~ (.*)-(.*) ]]; then
+ echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
+ exit 1
+ else
+ ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+ ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
+ fi
+ shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
set -e
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
+
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
+ set +e
+ /usr/bin/which createrepo >/dev/null
+ if [[ "$?" != "0" ]]; then
+ echo >&2
+ echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo"
+ echo >&2
+ exit 1
+ fi
+ set -e
createrepo $WORKSPACE/packages/$TARGET
fi
if test -z "$packages" ; then
packages="arvados-api-server
- arvados-docker-cleaner
arvados-git-httpd
arvados-node-manager
arvados-src
keep-rsync
keepstore
keep-web
- libarvados-perl"
+ libarvados-perl
+ python-arvados-fuse
+ python-arvados-python-client
+ python-arvados-cwl-runner"
- case "$TARGET" in
- *)
- packages="$packages python-arvados-fuse
- python-arvados-python-client python-arvados-cwl-runner"
- ;;
- esac
+ if [[ $TARGET != centos7 ]]; then
+ packages="$packages arvados-docker-cleaner"
+ fi
fi
FINAL_EXITCODE=0
# Build packages
if docker run --rm \
"${docker_volume_args[@]}" \
+ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
+ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
"$IMAGE" $COMMAND
# Go binaries
cd $WORKSPACE/packages/$TARGET
export GOPATH=$(mktemp -d)
-go get -v github.com/kardianos/govendor
+go get github.com/kardianos/govendor
package_go_binary sdk/go/crunchrunner crunchrunner \
"Crunchrunner executes a command inside a container and uploads the output"
package_go_binary services/arv-git-httpd arvados-git-httpd \
"Static web hosting service for user data stored in Arvados Keep"
package_go_binary services/ws arvados-ws \
"Arvados Websocket server"
+package_go_binary tools/sync-groups arvados-sync-groups \
+ "Synchronize remote groups into Arvados from an external source"
package_go_binary tools/keep-block-check keep-block-check \
"Verify that all data from one set of Keep servers to another was copied"
package_go_binary tools/keep-rsync keep-rsync \
# 2014-05-15
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/python/build"
-arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)
+arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)}
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
if [[ "$?" == "0" ]]; then
fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
# cwl-runner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
-arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
-arvados_cwl_runner_iteration=3
+arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)}
+declare -a iterargs=()
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ arvados_cwl_runner_iteration=3
+ iterargs+=(--iteration $arvados_cwl_runner_iteration)
+else
+ arvados_cwl_runner_iteration=
+fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 2
fi
# The PAM module
# not omit the python- prefix first.
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/fuse/build"
-arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)
+arvados_fuse_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)}
test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
if [[ "$?" == "0" ]]; then
fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
# The node manager
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/nodemanager/build"
-nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)
+nodemanager_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)}
test_package_presence arvados-node-manager "$nodemanager_version" python
if [[ "$?" == "0" ]]; then
fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
# The Docker image cleaner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
-dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-dockercleaner_iteration=3
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+dockercleaner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)}
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
-crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+crunchstat_summary_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)}
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
if [[ "$?" == "0" ]]; then
rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
- # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
LIBCLOUD_DIR=$(mktemp -d)
(
cd $LIBCLOUD_DIR
handle_python_package
DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
)
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
rm -rf $LIBCLOUD_DIR
fi
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+ bundle install --system >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
+--build-version <version>
+ Version to build (default:
+ \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
+ 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,target: \
+ help,upload,target:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
TARGET=debian8
UPLOAD=0
+declare -a build_args=()
+
eval set -- "$PARSEDOPTS"
while [ $# -gt 0 ]; do
case "$1" in
--upload)
UPLOAD=1
;;
+ --build-version)
+ build_args+=("$1" "$2")
+ shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
shift
done
+build_args+=(--target "$TARGET")
+
exit_cleanly() {
trap - INT
report_outcomes
title "Start build packages"
timer_reset
-$WORKSPACE/build/run-build-packages-one-target.sh --target $TARGET
+$WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}"
checkexit $? "build packages"
title "End of build packages (`timer`)"
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
- $WORKSPACE/build/run-build-packages-one-target.sh --target $TARGET --test-packages
+ $WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}" --test-packages
else
echo "Skipping package upload, there were errors building the packages"
fi
# older packages.
LICENSE_PACKAGE_TS=20151208015500
-RAILS_PACKAGE_ITERATION=8
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ RAILS_PACKAGE_ITERATION=8
+else
+ RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
+fi
debug_echo () {
echo "$@" >"$STDOUT_IF_DEBUG"
}
version_from_git() {
- # Generates a version number from the git log for the current working
- # directory, and writes it to stdout.
- local git_ts git_hash prefix
- if [[ -n "$1" ]] ; then
- prefix="$1"
- else
- prefix="0.1"
- fi
+ # Output the version being built, or if we're building a
+ # dev/prerelease, output a version number based on the git log for
+ # the current working directory.
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
+
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
- declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
version_from_git $1 | cut -d. -f1-3
}
go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
- declare -a switches=()
+ local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
}
default_iteration() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_ITERATION"
+ return
+ fi
local package_name="$1"; shift
local package_version="$1"; shift
local package_type="$1"; shift
cd $tmppwd
- test_package_presence $pkgname $version rails $RAILS_PACKAGE_ITERATION
+ test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
test_package_presence() {
fi
if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
fi
if [[ "$arch" == "" ]]; then
fi
if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname=$pkgname"_"$version"-"$iteration"_"$deb_architecture".deb"
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
else
- local complete_pkgname="$pkgname-$version-$iteration.$rpm_architecture.rpm"
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
fi
# See if we can skip building the package, only if it already exists in the
return 0
fi
local srcdir="$1"; shift
+ cd "$srcdir"
local license_path="$1"; shift
+ local version="$(version_from_git)"
local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
- local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ (
set -e
_build_rails_package_scripts "$pkgname" "$scripts_dir"
cd "$srcdir"
mkdir -p tmp
- version_from_git >"$version_file"
git rev-parse HEAD >git-commit.version
bundle package --all
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
EXITCODE=1
return 1
fi
local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
- "$(cat "$version_file")")
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
local license_arg="$license_path=$railsdir/$(basename "$license_path")"
- local -a switches=(--iteration=$RAILS_PACKAGE_ITERATION
- --after-install "$scripts_dir/postinst"
+ local -a switches=(--after-install "$scripts_dir/postinst"
--before-remove "$scripts_dir/prerm"
--after-remove "$scripts_dir/postrm")
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ switches+=(--iteration $RAILS_PACKAGE_ITERATION)
+ fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
# .git and packages are for the SSO server, which is built from its
switches+=(-x "$exclude_root/$exclude")
done
fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
}
# Build packages for everything
if [[ "$VERSION" != "" ]]; then
COMMAND_ARR+=('-v' "$VERSION")
fi
- # We can always add an --iteration here. If another one is specified in $@,
- # that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ -n "$default_iteration_value" ]]; then
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$default_iteration_value")
+ fi
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
sdk/go/stats
sdk/go/crunchrunner
sdk/cwl
-tools/arv-sync-groups
+tools/sync-groups
tools/crunchstat-summary
tools/keep-exercise
tools/keep-rsync
esac
done
-start_api() {
- echo 'Starting API server...'
+start_services() {
+ echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
- && python sdk/python/tests/run_test_server.py start_ws \
- && python sdk/python/tests/run_test_server.py start_nginx \
- && (env | egrep ^ARVADOS)
-}
-
-start_nginx_proxy_services() {
- echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
- cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
&& python sdk/python/tests/run_test_server.py start_nginx \
- && export ARVADOS_TEST_PROXY_SERVICES=1
+ && (env | egrep ^ARVADOS)
}
stop_services() {
- if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
- unset ARVADOS_TEST_PROXY_SERVICES
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop_keep-web \
- && python sdk/python/tests/run_test_server.py stop_keep_proxy
- fi
- if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
- unset ARVADOS_TEST_API_HOST
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop
+ if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then
+ return
fi
+ unset ARVADOS_TEST_API_HOST
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_ws \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+ && python sdk/python/tests/run_test_server.py stop
}
interrupt() {
mkdir -p "$WORKSPACE/services/api/tmp/pids"
+ cert="$WORKSPACE/services/api/tmp/self-signed"
+ if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
+ (
+ dir="$WORKSPACE/services/api/tmp"
+ set -ex
+ openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
+ openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
+ ) || return 1
+ fi
+
+ cd "$WORKSPACE/services/api" \
+ && rm -rf tmp/git \
+ && mkdir -p tmp/git \
+ && cd tmp/git \
+ && tar xf ../../test/test.git.tar \
+ && mkdir -p internal.git \
+ && git --git-dir internal.git init \
+ || return 1
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
tools/keep-block-check
tools/keep-exercise
tools/keep-rsync
- tools/arv-sync-groups
+ tools/sync-groups
)
for g in "${gostuff[@]}"
do
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_services || { stop_services; fatal "start_services"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
done
test_workbench_units() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_units workbench_units
test_workbench_functionals() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_functionals workbench_functionals
test_workbench_integration() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_integration workbench_integration
test_workbench_benchmark() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
}
do_test apps/workbench_benchmark workbench_benchmark
test_workbench_profile() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
do_test apps/workbench_profile workbench_profile
- user/topics/run-command.html.textile.liquid
- user/reference/job-pipeline-ref.html.textile.liquid
- user/examples/crunch-examples.html.textile.liquid
- - user/topics/arv-sync-groups.html.textile.liquid
+ - user/topics/arvados-sync-groups.html.textile.liquid
- Query the metadata database:
- user/topics/tutorial-trait-search.html.textile.liquid
- Arvados License:
"kind":"collection",
"uuid":"..."
}</code></pre>|
-|Git tree|@git_tree@|One of { @"git-url"@, @"repository_name"@, @"uuid"@ } must be provided.
-One of { @"commit"@, @"revisions"@ } must be provided.
-"path" may be provided. The default path is "/".
-At container startup, the target path will have the source tree indicated by the given revision. The @.git@ metadata directory _will not_ be available: typically the system will use @git-archive@ rather than @git-checkout@ to prepare the target directory.
-- If a value is given for @"revisions"@, it will be resolved to a set of commits (as desribed in the "ranges" section of git-revisions(1)) and the container request will be satisfiable by any commit in that set.
-- If a value is given for @"commit"@, it will be resolved to a single commit, and the tree resulting from that commit will be used.
-- @"path"@ can be used to select a subdirectory or a single file from the tree indicated by the selected commit.
-- Multiple commits can resolve to the same tree: for example, the file/directory given in @"path"@ might not have changed between commits A and B.
-- The resolved mount (found in the Container record) will have only the "kind" key and a "blob" or "tree" key indicating the 40-character hash of the git tree/blob used.|<pre><code>{
+|Git tree|@git_tree@|@"uuid"@ must be the UUID of an Arvados-hosted git repository.
+@"commit"@ must be a full 40-character commit hash.
+@"path"@, if provided, must be "/".
+At container startup, the target path will have the source tree indicated by the given commit. The @.git@ metadata directory _will not_ be available.|<pre><code>{
"kind":"git_tree",
"uuid":"zzzzz-s0uqq-xxxxxxxxxxxxxxx",
- "commit":"master"
+ "commit":"f315c59f90934cccae6381e72bba59d27ba42099"
}
-{
- "kind":"git_tree",
- "uuid":"zzzzz-s0uqq-xxxxxxxxxxxxxxx",
- "commit_range":"bugfix^..master",
- "path":"/crunch_scripts/grep"
-}</code></pre>|
+</code></pre>|
|Temporary directory|@tmp@|@"capacity"@: capacity (in bytes) of the storage device.
@"device_type"@ (optional, default "network"): one of @{"ram", "ssd", "disk", "network"}@ indicating the acceptable level of performance.
At container startup, the target path will be empty. When the container finishes, the content will be discarded. This will be backed by a storage mechanism no slower than the specified type.|<pre><code>{
The following operators are available.
table(table table-bordered table-condensed).
-|_. Operator|_. Operand type|_. Example|
-|@<@, @<=@, @>=@, @>@, @like@, @ilike@|string|@["script_version","like","d00220fb%"]@|
-|@=@, @!=@|string or null|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
-@["tail_uuid","!=",null]@|
-|@in@, @not in@|array of strings|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
-|@is_a@|string|@["head_uuid","is_a","arvados#pipelineInstance"]@|
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
+|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
+|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
+|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+
+h4. Filtering on subproperties
+
+Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property@. You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@. Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
+
+table(table table-bordered table-condensed).
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number or boolean|Equality comparison|@["properties.my_subproperty", "=", "fizzy whizy sparkle pop"]@|
+|@<@, @<=@, @>=@, @>@|string or number|Ordering comparison|@["properties.my_subproperty", "<", 3]@|
+|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
+|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
+
+Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all. To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
h3. Results of list method
</code></pre>
</notextile>
-The @-max-buffers@ argument limits keepstore's memory usage. It should be set such that @max-buffers * 64MiB + 10%@ fits comfortably in memory. For example, @-max-buffers=100@ is suitable for a host with 8 GiB RAM.
+p(#max-buffers). The @-max-buffers@ argument limits keepstore's memory usage. It should be set such that @max-buffers * 64MiB + 10%@ fits comfortably in memory. On a host dedicated to running keepstore, divide total memory by 88MiB to suggest a suitable value. For example, if @grep MemTotal /proc/meminfo@ reports @MemTotal: 7125440 kB@, compute 7125440÷(88×1024)=79 and configure @-max-buffers=79@.
If you want access control on your Keepstore server(s), you must specify the @-enforce-permissions@ flag and provide a signing key. The @-blob-signing-key-file@ argument should be a file containing a long random alphanumeric string with no internal line breaks (it is also possible to use a socket or FIFO: keepstore reads it only once, at startup). This key must be the same as the @blob_signing_key@ configured in the "API server's":install-api-server.html configuration file, @/etc/arvados/api/application.yml@.
---
layout: default
navsection: userguide
-title: "Using arv-sync-groups"
+title: "Using arvados-sync-groups"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The @arv-sync-groups@ tool allows to synchronize remote groups into Arvados from an external source.
+The @arvados-sync-groups@ tool allows to synchronize remote groups into Arvados from an external source.
-h1. Using arv-sync-groups
+h1. Using arvados-sync-groups
This tool reads a CSV (comma-separated values) file having information about external groups and their members. When running it for the first time, it'll create a special group named 'Externally synchronized groups' meant to be the parent of all the remote groups.
This tool is designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
-bq. NOTE: @arv-sync-groups@ needs to perform several administrative tasks on Arvados, so must be run using a superuser token
+bq. NOTE: @arvados-sync-groups@ needs to perform several administrative tasks on Arvados, so must be run using a superuser token
h2. Options
To sync groups using the username to identify every account, reading from some @external_groups.csv@ file, the command should be called as follows:
<notextile>
-<pre><code>~$ <span class="userinput">arv-sync-groups --user-id username /path/to/external_groups.csv </span>
+<pre><code>~$ <span class="userinput">arvados-sync-groups --user-id username /path/to/external_groups.csv </span>
</code></pre>
</notextile>
If you want to use a specific preexisting group as the parent of all the remote groups, you can do it this way:
<notextile>
-<pre><code>~$ <span class="userinput">arv-sync-groups --parent-group-uuid <preexisting group UUID> --user-id username /path/to/external_groups.csv </span>
+<pre><code>~$ <span class="userinput">arvados-sync-groups --parent-group-uuid <preexisting group UUID> --user-id username /path/to/external_groups.csv </span>
</code></pre>
</notextile>
#!/usr/bin/env perl
+# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
-# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
-
=head1 NAME
crunch-job: Execute job steps, save snapshots as requested, collate output.
. $slot[$proc{$pid}->{slot}]->{cpu});
my $jobstepidx = $proc{$pid}->{jobstepidx};
+ readfrompipes_after_exit ($jobstepidx);
+
$children_reaped++;
my $elapsed = time - $proc{$pid}->{time};
my $Jobstep = $jobstep[$jobstepidx];
$Jobstep->{finishtime} = time;
$Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
retry_op(sub { $Jobstep->{'arvados_task'}->save; }, "job_tasks.update API");
- process_stderr_final ($jobstepidx);
Log ($jobstepidx, sprintf("task output (%d bytes): %s",
length($Jobstep->{'arvados_task'}->{output}),
$Jobstep->{'arvados_task'}->{output}));
}
-sub process_stderr_final
+# Read whatever is still available on its stderr+stdout pipes after
+# the given child process has exited.
+sub readfrompipes_after_exit
{
my $jobstepidx = shift;
+
+ # The fact that the child has exited allows some convenient
+ # simplifications: (1) all data must have already been written, so
+ # there's no need to wait for more once sysread returns 0; (2) the
+ # total amount of data available is bounded by the pipe buffer size,
+ # so it's safe to read everything into one string.
+ my $buf;
+ while (0 < sysread ($reader{$jobstepidx}, $buf, 65536)) {
+ $jobstep[$jobstepidx]->{stderr_at} = time;
+ $jobstep[$jobstepidx]->{stderr} .= $buf;
+ }
+ if ($jobstep[$jobstepidx]->{stdout_r}) {
+ while (0 < sysread ($jobstep[$jobstepidx]->{stdout_r}, $buf, 65536)) {
+ $jobstep[$jobstepidx]->{stdout_captured} .= $buf;
+ }
+ }
preprocess_stderr ($jobstepidx);
map {
}
my $exited = $?;
- 1 while readfrompipes();
- process_stderr_final ($jobstepidx);
+ readfrompipes_after_exit ($jobstepidx);
Log (undef, "$label: exit ".exit_status_s($exited));
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pkg_resources
+
+__version__ = pkg_resources.require('arvados-cwl-runner')[0].version
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
-
-versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
-try:
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%H', gittaggers.choose_version_from()]).strip()
- with open(versionfile, "w") as f:
- f.write("__version__ = '%s'\n" % gitinfo)
-except Exception as e:
- # When installing from package, it won't be part of a git repository, and
- # check_output() will raise an exception. But the package should include the
- # version file, so we can proceed.
- if not os.path.exists(versionfile):
- raise
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "1.0"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-cwl-runner',
- version='1.0',
+ version=version,
description='Arvados Common Workflow Language runner',
long_description=open(README).read(),
author='Arvados',
BasePath string `json:"basePath"`
DefaultCollectionReplication int `json:"defaultCollectionReplication"`
BlobSignatureTTL int64 `json:"blobSignatureTtl"`
+ GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
}
return fmt.Errorf("line %d: bad locator %q", lineno, token)
}
- toks := strings.Split(token, ":")
+ toks := strings.SplitN(token, ":", 3)
if len(toks) != 3 {
return fmt.Errorf("line %d: bad file segment %q", lineno, token)
}
c.Check(ok, check.Equals, true)
}
+func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
+ fs, err := (&Collection{
+ ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
+ }).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.Open("/foo:foo")
+ c.Assert(err, check.IsNil)
+
+ fis, err := f.Readdir(0)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 1)
+ c.Check(fis[0].Name(), check.Equals, "bar:bar")
+}
+
func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
f, err := s.fs.Open("/dir1")
c.Assert(err, check.IsNil)
". 0:0:foo\n",
". 0:0:foo 0:0:bar\n",
". d41d8cd98f00b204e9800998ecf8427e 0:0:foo\n",
- ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo:bar\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 :0:0:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
+ Commit string `json:"commit"` // only if kind=="git_tree"
+ RepositoryName string `json:"repository_name"` // only if kind=="git_tree"
+ GitURL string `json:"git_url"` // only if kind=="git_tree"
}
// RuntimeConstraints specify a container's compute resources (RAM,
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+ ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+ ArvadosRepoName = "arvados"
+ FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
+ FooRepoName = "active/foo"
+ Repository2UUID = "zzzzz-s0uqq-382brsig8rp3667"
+ Repository2Name = "active/foo2"
+
+ FooCollectionSharingTokenUUID = "zzzzz-gj3su-gf02tdm4g1z3e3u"
+ FooCollectionSharingToken = "iknqgmunrhgsyfok8uzjlwun9iscwm3xacmzmg65fa1j1lpdss"
)
// PathologicalManifest : A valid manifest designed to test
README = os.path.join(SETUP_DIR, 'README.rst')
tagger = egg_info_cmd.egg_info
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except (ImportError, OSError):
- pass
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-pam',
- version='0.1',
+ version=version,
description='Arvados PAM module',
long_description=open(README).read(),
author='Arvados',
return body
-def _intercept_http_request(self, uri, method="GET", **kwargs):
+def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- if 'headers' not in kwargs:
- kwargs['headers'] = {}
-
if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
- kwargs['headers']['X-External-Client'] = '1'
+ headers['X-External-Client'] = '1'
- kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+ if not headers.get('X-Request-Id'):
+ headers['X-Request-Id'] = self._request_id()
retryable = method in [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
for _ in range(retry_count):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
- return self.orig_http_request(uri, method, **kwargs)
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
http._retry_delay_initial = RETRY_DELAY_INITIAL
http._retry_delay_backoff = RETRY_DELAY_BACKOFF
http._retry_count = RETRY_COUNT
+ http._request_id = util.new_request_id
return http
# Monkey patch discovery._cast() so objects and arrays get serialized
return None
return cache.SafeHTTPCache(path, max_age=60*60*24*2)
-def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False,
+ request_id=None, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
:version:
:insecure:
If True, ignore SSL certificate validation errors.
+ :request_id:
+ Default X-Request-Id header value for outgoing requests that
+ don't already provide one. If None or omitted, generate a random
+ ID. When retrying failed requests, the same ID is used on all
+ attempts.
+
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
elif host and token:
pass
elif not host and not token:
- return api_from_config(version=version, cache=cache, **kwargs)
+ return api_from_config(
+ version=version, cache=cache, request_id=request_id, **kwargs)
else:
# Caller provided one but not the other
if not host:
svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs)
svc.api_token = token
svc.insecure = insecure
+ svc.request_id = request_id
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
+ kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
return svc
def api_from_config(version=None, apiconfig=None, **kwargs):
refuse to write *anything* if any of the output files already
exist. As a special case, -f is not needed to write to stdout.
""")
+group.add_argument('-v', action='count', default=0,
+ help="""
+Once for verbose mode, twice for debug mode.
+""")
group.add_argument('--skip-existing', action='store_true',
help="""
Skip files that already exist. The default behavior is to refuse to
stdout = stdout.buffer
args = parse_arguments(arguments, stdout, stderr)
+ logger.setLevel(logging.WARNING - 10 * args.v)
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
r = re.search(r'^(.*?)(/.*)?$', args.locator)
col_loc = r.group(1)
open_flags |= os.O_EXCL
try:
if args.destination == "-":
- write_block_or_manifest(dest=stdout, src=col_loc,
- api_client=api_client, args=args)
+ write_block_or_manifest(
+ dest=stdout, src=col_loc,
+ api_client=api_client, args=args)
else:
out_fd = os.open(args.destination, open_flags)
with os.fdopen(out_fd, 'wb') as out_file:
- write_block_or_manifest(dest=out_file,
- src=col_loc, api_client=api_client,
- args=args)
+ write_block_or_manifest(
+ dest=out_file, src=col_loc,
+ api_client=api_client, args=args)
except (IOError, OSError) as error:
logger.error("can't write to '{}': {}".format(args.destination, error))
return 1
return 0
try:
- reader = arvados.CollectionReader(col_loc, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ col_loc, api_client=api_client, num_retries=args.retries)
except Exception as error:
logger.error("failed to read collection: {}".format(error))
return 1
dest.write(kc.get(src, num_retries=args.retries))
else:
# collection UUID or portable data hash
- reader = arvados.CollectionReader(src, num_retries=args.retries)
+ reader = arvados.CollectionReader(
+ src, api_client=api_client, num_retries=args.retries)
dest.write(reader.manifest_text(strip=args.strip_manifest).encode())
total data size).
""")
-_group.add_argument('--silent', action='store_true',
- help="""
-Do not print any debug messages to console. (Any error messages will still be displayed.)
+run_opts.add_argument('--silent', action='store_true',
+ help="""
+Do not print any debug messages to console. (Any error messages will
+still be displayed.)
""")
_group = run_opts.add_mutually_exclusive_group()
}
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
- name=None, owner_uuid=None,
+ name=None, owner_uuid=None, api_client=None,
ensure_unique_name=False, num_retries=None,
put_threads=None, replication_desired=None,
filename=None, update_time=60.0, update_collection=None,
self.replication_desired = replication_desired
self.put_threads = put_threads
self.filename = filename
+ self._api_client = api_client
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
self._current_files = [] # Current run file list
if update_collection and re.match(arvados.util.collection_uuid_pattern,
update_collection):
try:
- self._remote_collection = arvados.collection.Collection(update_collection)
+ self._remote_collection = arvados.collection.Collection(
+ update_collection, api_client=self._api_client)
except arvados.errors.ApiError as error:
raise CollectionUpdateError("Cannot read collection {} ({})".format(update_collection, error))
else:
# No cache file, set empty state
self._state = copy.deepcopy(self.EMPTY_STATE)
# Load the previous manifest so we can check if files were modified remotely.
- self._local_collection = arvados.collection.Collection(self._state['manifest'], replication_desired=self.replication_desired, put_threads=self.put_threads)
+ self._local_collection = arvados.collection.Collection(
+ self._state['manifest'],
+ replication_desired=self.replication_desired,
+ put_threads=self.put_threads,
+ api_client=self._api_client)
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
else:
logger.setLevel(logging.INFO)
status = 0
+
+ request_id = arvados.util.new_request_id()
+ logger.info('X-Request-Id: '+request_id)
+
if api_client is None:
- api_client = arvados.api('v1')
+ api_client = arvados.api('v1', request_id=request_id)
# Determine the name to use
if args.name:
use_cache = args.use_cache,
filename = args.filename,
reporter = reporter,
+ api_client = api_client,
num_retries = args.retries,
replication_desired = args.replication,
put_threads = args.threads,
def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
- download_counter=None, **headers):
+ download_counter=None,
+ headers={}):
self.root = root
self._user_agent_pool = user_agent_pool
self._result = {'error': None}
_logger.debug("{}: {}".format(locator, sorted_roots))
return sorted_roots
- def map_new_services(self, roots_map, locator, force_rebuild, need_writable, **headers):
+ def map_new_services(self, roots_map, locator, force_rebuild, need_writable, headers):
# roots_map is a dictionary, mapping Keep service root strings
# to KeepService objects. Poll for Keep services, and add any
# new ones to roots_map. Return the current list of local
root, self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter,
- **headers)
+ headers=headers)
return local_roots
@staticmethod
return None
@retry.retry_method
- def head(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="HEAD", num_retries=num_retries)
+ def head(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="HEAD", **kwargs)
@retry.retry_method
- def get(self, loc_s, num_retries=None):
- return self._get_or_head(loc_s, method="GET", num_retries=num_retries)
+ def get(self, loc_s, **kwargs):
+ return self._get_or_head(loc_s, method="GET", **kwargs)
- def _get_or_head(self, loc_s, method="GET", num_retries=None):
+ def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None):
"""Get data from Keep.
This method fetches one or more blocks of data from Keep. It
self.misses_counter.add(1)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ }
+
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
roots_map = {
root: self.KeepService(root, self._user_agent_pool,
upload_counter=self.upload_counter,
- download_counter=self.download_counter)
+ download_counter=self.download_counter,
+ headers=headers)
for root in hint_roots
}
sorted_roots = self.map_new_services(
roots_map, locator,
force_rebuild=(tries_left < num_retries),
- need_writable=False)
+ need_writable=False,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
"failed to read {}".format(loc_s), service_errors, label="service")
@retry.retry_method
- def put(self, data, copies=2, num_retries=None):
+ def put(self, data, copies=2, num_retries=None, request_id=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
return loc_s
locator = KeepLocator(loc_s)
- headers = {}
- # Tell the proxy how many copies we want it to store
- headers['X-Keep-Desired-Replicas'] = str(copies)
+ headers = {
+ 'X-Request-Id': (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id()),
+ 'X-Keep-Desired-Replicas': str(copies),
+ }
roots_map = {}
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
try:
sorted_roots = self.map_new_services(
roots_map, locator,
- force_rebuild=(tries_left < num_retries), need_writable=True, **headers)
+ force_rebuild=(tries_left < num_retries),
+ need_writable=True,
+ headers=headers)
except Exception as error:
loop.save_result(error)
continue
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import division
+from builtins import range
+
import fcntl
import hashlib
import httplib2
import os
+import random
import re
import subprocess
import errno
if os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
+
+def new_request_id():
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados-python-client',
- version='0.1',
+ version=version,
description='Arvados client library',
long_description=open(README).read(),
author='Arvados',
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
- if not os.path.exists('tmp/self-signed.pem'):
- # We assume here that either passenger reports its listening
- # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
- # then the certificate won't match the host and reset() will
- # fail certificate verification. If it reports "localhost",
- # clients (notably Python SDK's websocket client) might
- # resolve localhost as ::1 and then fail to connect.
- subprocess.check_call([
- 'openssl', 'req', '-new', '-x509', '-nodes',
- '-out', 'tmp/self-signed.pem',
- '-keyout', 'tmp/self-signed.key',
- '-days', '3650',
- '-subj', '/CN=0.0.0.0'],
- stdout=sys.stderr)
-
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
self.assertEqual(sleep.call_args_list,
[mock.call(RETRY_DELAY_INITIAL)])
+ @mock.patch('time.sleep')
+ def test_same_automatic_request_id_on_retry(self, sleep):
+ self.api._http.orig_http_request.side_effect = (
+ socket.error('mock error'),
+ self.request_success,
+ )
+ self.api.users().current().execute()
+ calls = self.api._http.orig_http_request.call_args_list
+ self.assertEqual(len(calls), 2)
+ self.assertEqual(
+ calls[0][1]['headers']['X-Request-Id'],
+ calls[1][1]['headers']['X-Request-Id'])
+ self.assertRegex(calls[0][1]['headers']['X-Request-Id'], r'^req-[a-z0-9]{20}$')
+
+ @mock.patch('time.sleep')
+ def test_provided_request_id_on_retry(self, sleep):
+ self.api.request_id='fake-request-id'
+ self.api._http.orig_http_request.side_effect = (
+ socket.error('mock error'),
+ self.request_success,
+ )
+ self.api.users().current().execute()
+ calls = self.api._http.orig_http_request.call_args_list
+ self.assertEqual(len(calls), 2)
+ for call in calls:
+ self.assertEqual(call[1]['headers']['X-Request-Id'], 'fake-request-id')
+
@mock.patch('time.sleep')
def test_socket_error_retry_delay(self, sleep):
self.api._http.orig_http_request.side_effect = socket.error('mock')
from __future__ import absolute_import
from future.utils import listitems
import io
+import logging
import mock
import os
import re
self.tempdir = tempfile.mkdtemp()
self.col_loc, self.col_pdh, self.col_manifest = self.write_test_collection()
+ self.stdout = tutil.BytesIO()
+ self.stderr = tutil.StringIO()
+ self.loggingHandler = logging.StreamHandler(self.stderr)
+ self.loggingHandler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ logging.getLogger().addHandler(self.loggingHandler)
+
def tearDown(self):
+ logging.getLogger().removeHandler(self.loggingHandler)
super(ArvadosGetTestCase, self).tearDown()
shutil.rmtree(self.tempdir)
c.manifest_text(strip=strip_manifest))
def run_get(self, args):
- self.stdout = tutil.BytesIO()
- self.stderr = tutil.StringIO()
+ self.stdout.seek(0, 0)
+ self.stdout.truncate(0)
+ self.stderr.seek(0, 0)
+ self.stderr.truncate(0)
return arv_get.main(args, self.stdout, self.stderr)
def test_version_argument(self):
self.assertEqual(0, r)
self.assertEqual(b'', stdout.getvalue())
self.assertFalse(stderr.write.called)
+
+ request_id_regex = r'INFO: X-Request-Id: req-[a-z0-9]{20}\n'
+
+ def test_request_id_logging_on(self):
+ r = self.run_get(["-v", "{}/".format(self.col_loc), self.tempdir])
+ self.assertEqual(0, r)
+ self.assertRegex(self.stderr.getvalue(), self.request_id_regex)
+
+ def test_request_id_logging_off(self):
+ r = self.run_get(["{}/".format(self.col_loc), self.tempdir])
+ self.assertEqual(0, r)
+ self.assertNotRegex(self.stderr.getvalue(), self.request_id_regex)
import datetime
import hashlib
import json
+import logging
import mock
import os
import pwd
Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz'
def call_main_with_args(self, args):
- self.main_stdout = tutil.StringIO()
- self.main_stderr = tutil.StringIO()
+ self.main_stdout.seek(0, 0)
+ self.main_stdout.truncate(0)
+ self.main_stderr.seek(0, 0)
+ self.main_stderr.truncate(0)
return arv_put.main(args, self.main_stdout, self.main_stderr)
def call_main_on_test_file(self, args=[]):
super(ArvadosPutTest, self).setUp()
run_test_server.authorize_with('active')
arv_put.api_client = None
+ self.main_stdout = tutil.StringIO()
+ self.main_stderr = tutil.StringIO()
+ self.loggingHandler = logging.StreamHandler(self.main_stderr)
+ self.loggingHandler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ logging.getLogger().addHandler(self.loggingHandler)
def tearDown(self):
+ logging.getLogger().removeHandler(self.loggingHandler)
for outbuf in ['main_stdout', 'main_stderr']:
if hasattr(self, outbuf):
getattr(self, outbuf).close()
self.assertLess(0, coll_save_mock.call_count)
self.assertEqual("", self.main_stdout.getvalue())
+ def test_request_id_logging(self):
+ matcher = r'INFO: X-Request-Id: req-[a-z0-9]{20}\n'
+
+ self.call_main_on_test_file()
+ self.assertRegex(self.main_stderr.getvalue(), matcher)
+
+ self.call_main_on_test_file(['--silent'])
+ self.assertNotRegex(self.main_stderr.getvalue(), matcher)
+
class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
ArvadosBaseTestCase):
"""Convert minutes-east-of-UTC to RFC3339- and ISO-compatible time zone designator"""
return '{:+03d}:{:02d}'.format(offset//60, offset%60)
- # Test websocket reconnection on (un)execpted close
+ # Test websocket reconnection on (un)expected close
def _test_websocket_reconnect(self, close_unexpected):
run_test_server.authorize_with('active')
events = queue.Queue(100)
import arvados
import arvados.retry
+import arvados.util
from . import arvados_testutil as tutil
from . import keepstub
from . import run_test_server
self.assertEqual(1, req_mock.call_count)
+@tutil.skip_sleep
+class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def setUp(self):
+ self.api_client = self.mock_keep_services(count=2)
+ self.keep_client = arvados.KeepClient(api_client=self.api_client)
+ self.data = b'xyzzy'
+ self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ self.test_id = arvados.util.new_request_id()
+ self.assertRegex(self.test_id, r'^req-[a-z0-9]{20}$')
+ # If we don't set request_id to None explicitly here, it will
+ # return <MagicMock name='api_client_mock.request_id'
+ # id='123456789'>:
+ self.api_client.request_id = None
+
+ def test_default_to_api_client_request_id(self):
+ self.api_client.request_id = self.test_id
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertProvidedRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ def test_explicit_request_id(self):
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data, request_id=self.test_id)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertProvidedRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator, request_id=self.test_id)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator, request_id=self.test_id)
+ self.assertProvidedRequestId(mock.responses[0])
+
+ def test_automatic_request_id(self):
+ with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
+ self.keep_client.put(self.data)
+ self.assertEqual(2, len(mock.responses))
+ for resp in mock.responses:
+ self.assertAutomaticRequestId(resp)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.keep_client.get(self.locator)
+ self.assertAutomaticRequestId(mock.responses[0])
+
+ with tutil.mock_keep_responses(b'', 200) as mock:
+ self.keep_client.head(self.locator)
+ self.assertAutomaticRequestId(mock.responses[0])
+
+ def assertAutomaticRequestId(self, resp):
+ hdr = [x for x in resp.getopt(pycurl.HTTPHEADER)
+ if x.startswith('X-Request-Id: ')][0]
+ self.assertNotEqual(hdr, 'X-Request-Id: '+self.test_id)
+ self.assertRegex(hdr, r'^X-Request-Id: req-[a-z0-9]{20}$')
+
+ def assertProvidedRequestId(self, resp):
+ self.assertIn('X-Request-Id: '+self.test_id,
+ resp.getopt(pycurl.HTTPHEADER))
+
+
@tutil.skip_sleep
class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
gem 'arvados', '>= 0.1.20150615153458'
gem 'arvados-cli', '>= 0.1.20161017193526'
+gem 'httpclient'
gem 'sshkey'
gem 'safe_yaml'
hashie (3.5.5)
highline (1.7.8)
hike (1.2.3)
+ httpclient (2.8.3)
i18n (0.9.0)
concurrent-ruby (~> 1.0)
jquery-rails (4.2.2)
database_cleaner
factory_girl_rails
faye-websocket
+ httpclient
jquery-rails
lograge
logstash-event
.all
end
@read_auths.select! { |auth| auth.scopes_allow_request? request }
- @read_users = @read_auths.map { |auth| auth.user }.uniq
+ @read_users = @read_auths.map(&:user).uniq
end
def require_login
def set_cors_headers
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, HEAD, PUT, POST, DELETE'
- response.headers['Access-Control-Allow-Headers'] = 'Authorization'
+ response.headers['Access-Control-Allow-Headers'] = 'Authorization, Content-Type'
response.headers['Access-Control-Max-Age'] = '86486400'
end
def find_objects_for_index
if params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name)
- @objects = Collection.readable_by(*@read_users, {include_trash: true, query_on: Collection.unscoped})
+ @objects = Collection.readable_by(*@read_users, {include_trash: true})
end
super
end
end
end.compact
- query_on = if klass == Collection and params[:include_trash]
- klass.unscoped
- else
- klass
- end
- @objects = query_on.readable_by(*@read_users, {:include_trash => params[:include_trash]}).
+ @objects = klass.readable_by(*@read_users, {:include_trash => params[:include_trash]}).
order(request_order).where(where_conds)
klass_limit = limit_all - all_objects.count
def index
expires_in 24.hours, public: true
- discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
+ send_json discovery_doc
+ end
+
+ protected
+
+ def discovery_doc
+ Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
discovery = {
kind: "discovery#restDescription",
crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
+ remoteHosts: Rails.configuration.remote_hosts,
+ remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
+ keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+ gitUrl: case Rails.application.config.git_repo_https_base
+ when false
+ ''
+ when true
+ 'https://git.%s.arvadosapi.com/' % Rails.configuration.uuid_prefix
+ else
+ Rails.application.config.git_repo_https_base
+ end,
parameters: {
alt: {
type: "string",
end
discovery
end
- send_json discovery
end
end
return redirect_to login_failure_url
end
- user = User.find_by_identity_url(omniauth['info']['identity_url'])
+ # Only local users can create sessions, hence uuid_like_pattern
+ # here.
+ user = User.where('identity_url = ? and uuid like ?',
+ omniauth['info']['identity_url'],
+ User.uuid_like_pattern).first
if not user
# Check for permission to log in to an existing User record with
# a different identity_url
end
def call env
- # First, clean up just in case we have a multithreaded server and thread
- # local variables are still set from a prior request. Also useful for
- # tests that call this code to set up the environment.
- Thread.current[:api_client_ip_address] = nil
- Thread.current[:api_client_authorization] = nil
- Thread.current[:api_client_uuid] = nil
- Thread.current[:api_client] = nil
- Thread.current[:user] = nil
-
request = Rack::Request.new(env)
params = request.params
remote_ip = env["action_dispatch.remote_ip"]
Thread.current[:request_starttime] = Time.now
- user = nil
- api_client = nil
- api_client_auth = nil
- if request.get? || params["_method"] == 'GET'
+
+ remote = false
+ reader_tokens = nil
+ if params["remote"] && request.get? && (
+ request.path.start_with?('/arvados/v1/groups') ||
+ request.path.start_with?('/arvados/v1/users/current'))
+ # Request from a remote API server, asking to validate a salted
+ # token.
+ remote = params["remote"]
+ elsif request.get? || params["_method"] == 'GET'
reader_tokens = params["reader_tokens"]
if reader_tokens.is_a? String
reader_tokens = SafeJSON.load(reader_tokens)
end
- else
- reader_tokens = nil
end
# Set current_user etc. based on the primary session token if a
# valid one is present. Otherwise, use the first valid token in
# reader_tokens.
+ auth = nil
[params["api_token"],
params["oauth_token"],
- env["HTTP_AUTHORIZATION"].andand.match(/OAuth2 ([a-zA-Z0-9]+)/).andand[1],
+ env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([-\/a-zA-Z0-9]+)/).andand[2],
*reader_tokens,
].each do |supplied|
next if !supplied
try_auth = ApiClientAuthorization.
- includes(:api_client, :user).
- where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', supplied).
- first
+ validate(token: supplied, remote: remote)
if try_auth.andand.user
- api_client_auth = try_auth
- user = api_client_auth.user
- api_client = api_client_auth.api_client
+ auth = try_auth
break
end
end
+
Thread.current[:api_client_ip_address] = remote_ip
- Thread.current[:api_client_authorization] = api_client_auth
- Thread.current[:api_client_uuid] = api_client.andand.uuid
- Thread.current[:api_client] = api_client
- Thread.current[:user] = user
+ Thread.current[:api_client_authorization] = auth
+ Thread.current[:api_client_uuid] = auth.andand.api_client.andand.uuid
+ Thread.current[:api_client] = auth.andand.api_client
+ Thread.current[:user] = auth.andand.user
@app.call env if @app
end
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ extend CurrentApiClient
belongs_to :api_client
belongs_to :user
t.add :owner_uuid
t.add :user_id
t.add :api_client_id
+ # NB the "api_token" db column is a misnomer in that it's only the
+ # "secret" part of a token: a v1 token is just the secret, but a
+ # v2 token is "v2/uuid/secret".
t.add :api_token
t.add :created_by_ip_address
t.add :default_owner_uuid
["#{table_name}.id desc"]
end
+ def self.remote_host(uuid_prefix:)
+ Rails.configuration.remote_hosts[uuid_prefix] ||
+ (Rails.configuration.remote_hosts_via_dns &&
+ uuid_prefix+".arvadosapi.com")
+ end
+
+ def self.validate(token:, remote:)
+ return nil if !token
+ remote ||= Rails.configuration.uuid_prefix
+
+ case token[0..2]
+ when 'v2/'
+ _, uuid, secret = token.split('/')
+ unless uuid.andand.length == 27 && secret.andand.length.andand > 0
+ return nil
+ end
+
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('uuid=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', uuid).
+ first
+ if auth && auth.user &&
+ (secret == auth.api_token ||
+ secret == OpenSSL::HMAC.hexdigest('sha1', auth.api_token, remote))
+ return auth
+ end
+
+ uuid_prefix = uuid[0..4]
+ if uuid_prefix == Rails.configuration.uuid_prefix
+ # If the token were valid, we would have validated it above
+ return nil
+ elsif uuid_prefix.length != 5
+ # malformed
+ return nil
+ end
+
+ host = remote_host(uuid_prefix: uuid_prefix)
+ if !host
+ Rails.logger.warn "remote authentication rejected: no host for #{uuid_prefix.inspect}"
+ return nil
+ end
+
+ # Token was issued by a different cluster. If it's expired or
+ # missing in our database, ask the originating cluster to
+ # [re]validate it.
+ begin
+ clnt = HTTPClient.new
+ if Rails.configuration.sso_insecure
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ end
+ remote_user = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/users/current',
+ {'remote' => Rails.configuration.uuid_prefix},
+ {'Authorization' => 'Bearer ' + token}))
+ rescue => e
+ Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}"
+ return nil
+ end
+ if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String) || remote_user['uuid'][0..4] != uuid[0..4]
+ Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}"
+ return nil
+ end
+ act_as_system_user do
+ # Add/update user and token in our database so we can
+ # validate subsequent requests faster.
+
+ user = User.find_or_create_by(uuid: remote_user['uuid']) do |user|
+ # (this block runs for the "create" case, not for "find")
+ user.is_admin = false
+ user.email = remote_user['email']
+ if remote_user['username'].andand.length.andand > 0
+ user.set_initial_username(requested: remote_user['username'])
+ end
+ end
+
+ if Rails.configuration.new_users_are_active
+ # Update is_active to whatever it is at the remote end
+ user.is_active = remote_user['is_active']
+ elsif !remote_user['is_active']
+ # Remote user is inactive; our mirror should be, too.
+ user.is_active = false
+ end
+
+ %w[first_name last_name email prefs].each do |attr|
+ user.send(attr+'=', remote_user[attr])
+ end
+
+ user.save!
+
+ auth = ApiClientAuthorization.find_or_create_by(uuid: uuid) do |auth|
+ auth.user = user
+ auth.api_token = secret
+ auth.api_client_id = 0
+ end
+
+ # Accept this token (and don't reload the user record) for
+ # 5 minutes. TODO: Request the actual api_client_auth
+ # record from the remote server in case it wants the token
+ # to expire sooner.
+ auth.update_attributes!(expires_at: Time.now + 5.minutes)
+ end
+ return auth
+ else
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token).
+ first
+ if auth && auth.user
+ return auth
+ end
+ end
+ return nil
+ end
+
protected
def permission_to_create
# Collect the UUIDs of the authorized users.
sql_table = kwargs.fetch(:table_name, table_name)
include_trash = kwargs.fetch(:include_trash, false)
- query_on = kwargs.fetch(:query_on, self)
sql_conds = []
user_uuids = users_list.map { |u| u.uuid }
+ exclude_trashed_records = if !include_trash and (sql_table == "groups" or sql_table == "collections") then
+ # Only include records that are not explicitly trashed
+ "AND #{sql_table}.is_trashed = false"
+ else
+ ""
+ end
+
if users_list.select { |u| u.is_admin }.any?
if !include_trash
- # exclude rows that are explicitly trashed.
if sql_table != "api_client_authorizations"
- sql_conds.push "NOT EXISTS(SELECT 1
- FROM #{PERMISSION_VIEW}
- WHERE trashed = 1 AND
- (#{sql_table}.uuid = target_uuid OR #{sql_table}.owner_uuid = target_uuid))"
+ # Exclude rows where the owner is trashed
+ sql_conds.push "NOT EXISTS(SELECT 1 "+
+ "FROM #{PERMISSION_VIEW} "+
+ "WHERE trashed = 1 AND "+
+ "(#{sql_table}.owner_uuid = target_uuid)) "+
+ exclude_trashed_records
end
end
else
- if include_trash
- trashed_check = ""
- else
- trashed_check = "AND trashed = 0"
- end
-
- if sql_table != "api_client_authorizations" and sql_table != "groups"
- owner_check = "OR (target_uuid = #{sql_table}.owner_uuid AND target_owner_uuid IS NOT NULL)"
- else
- owner_check = ""
- end
+ trashed_check = if !include_trash then
+ "AND trashed = 0"
+ else
+ ""
+ end
+
+ owner_check = if sql_table != "api_client_authorizations" and sql_table != "groups" then
+ "OR (target_uuid = #{sql_table}.owner_uuid AND target_owner_uuid IS NOT NULL)"
+ else
+ ""
+ end
sql_conds.push "EXISTS(SELECT 1 FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND (target_uuid = #{sql_table}.uuid #{owner_check}))"
+ "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND (target_uuid = #{sql_table}.uuid #{owner_check})) "+
+ exclude_trashed_records
if sql_table == "links"
# Match any permission link that gives one of the authorized
end
end
- query_on.where(sql_conds.join(' OR '),
+ self.where(sql_conds.join(' OR '),
user_uuids: user_uuids,
permission_link_classes: ['permission', 'resources'])
end
def self.full_text_searchable_columns
self.columns.select do |col|
- col.type == :string or col.type == :text
+ [:string, :text, :jsonb].include?(col.type)
end.map(&:name)
end
def self.full_text_tsvector
parts = full_text_searchable_columns.collect do |column|
- "coalesce(#{column},'')"
+ cast = serialized_attributes[column] ? '::text' : ''
+ "coalesce(#{column}#{cast},'')"
end
"to_tsvector('english', #{parts.join(" || ' ' || ")})"
end
if self == ArvadosModel
# If called directly as ArvadosModel.find_by_uuid rather than via subclass,
# delegate to the appropriate subclass based on the given uuid.
- self.resource_class_for_uuid(uuid).unscoped.find_by_uuid(uuid)
+ self.resource_class_for_uuid(uuid).find_by_uuid(uuid)
else
super
end
validate :ensure_pdh_matches_manifest_text
before_save :set_file_names
- # Query only untrashed collections by default.
- default_scope { where("is_trashed = false") }
-
api_accessible :user, extend: :common do |t|
t.add :name
t.add :description
end
def self.cache_dir_base
- Rails.root.join 'tmp', 'git'
+ Rails.root.join 'tmp', 'git-cache'
end
def self.fetch_remote_repository gitdir, git_url
# that a container cannot "claim" a collection that it doesn't otherwise
# have access to just by setting the output field to the collection PDH.
if output_changed?
- c = Collection.unscoped do
- Collection.
- readable_by(current_user).
+ c = Collection.
+ readable_by(current_user, {include_trash: true}).
where(portable_data_hash: self.output).
first
- end
if !c
errors.add :output, "collection must exist and be readable by current user."
end
trash_at = db_current_time + self.output_ttl
end
end
- manifest = Collection.unscoped do
- Collection.where(portable_data_hash: pdh).first.manifest_text
- end
+ manifest = Collection.where(portable_data_hash: pdh).first.manifest_text
coll = Collection.new(owner_uuid: owner_uuid,
manifest_text: manifest,
# original job reuse behavior, and is still the default).
reuse_job_if_outputs_differ: false
+ ###
+ ### Federation support.
+ ###
+
+ # You can enable use of this cluster by users who are authenticated
+ # by a remote Arvados site. Control which remote hosts are trusted
+ # to authenticate which user IDs by configuring remote_hosts,
+ # remote_hosts_via_dns, or both. The default configuration disables
+ # remote authentication.
+
+ # Map known prefixes to hosts. For example, if user IDs beginning
+ # with "zzzzz-" should be authenticated by the Arvados server at
+ # "zzzzz.example.com", use:
+ #
+ # remote_hosts:
+ # zzzzz: zzzzz.example.com
+ remote_hosts: {}
+
+ # Use {prefix}.arvadosapi.com for any prefix not given in
+ # remote_hosts above.
+ remote_hosts_via_dns: false
+
###
### Remaining assorted configuration options.
###
arvados_theme: default
- # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the Single Sign
- # On (sso) server. Should only be enabled during development when the SSO
- # server is using a self-signed cert.
+ # Permit insecure (OpenSSL::SSL::VERIFY_NONE) connections to the
+ # Single Sign On (sso) server and remote Arvados sites. Should only
+ # be enabled during development when the SSO server is using a
+ # self-signed cert.
sso_insecure: false
## Set Time.zone default to the specified zone and make Active
# Server expects request header of the format "Authorization: Bearer xxx"
ManagementToken: false
+ # URL of keep-web service. Provides read/write access to collections via
+ # HTTP and WebDAV protocols.
+ #
+ # Example:
+ # keep_web_service_url: https://download.uuid_prefix.arvadosapi.com/
+ keep_web_service_url: false
+
development:
force_ssl: false
cache_classes: false
}
end
+ def replace_index(t)
+ i = fts_indexes[t]
+ t.classify.constantize.reset_column_information
+ execute "DROP INDEX IF EXISTS #{i}"
+ execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector})"
+ end
+
def up
- # remove existing fts indexes and create up to date ones with no leading space
- fts_indexes.each do |t, i|
- t.classify.constantize.reset_column_information
- ActiveRecord::Base.connection.indexes(t).each do |idx|
- if idx.name == i
- remove_index t.to_sym, :name => i
- break
- end
- end
- execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector});"
+ # remove existing fts indexes and create up to date ones with no
+ # leading space
+ fts_indexes.keys.each do |t|
+ replace_index(t)
end
end
--- /dev/null
+require 'migrate_yaml_to_json'
+
+class YamlToJson < ActiveRecord::Migration
+ def up
+ [
+ ['collections', 'properties'],
+ ['containers', 'environment'],
+ ['containers', 'mounts'],
+ ['containers', 'runtime_constraints'],
+ ['containers', 'command'],
+ ['containers', 'scheduling_parameters'],
+ ['container_requests', 'properties'],
+ ['container_requests', 'environment'],
+ ['container_requests', 'mounts'],
+ ['container_requests', 'runtime_constraints'],
+ ['container_requests', 'command'],
+ ['container_requests', 'scheduling_parameters'],
+ ['humans', 'properties'],
+ ['job_tasks', 'parameters'],
+ ['links', 'properties'],
+ ['nodes', 'info'],
+ ['nodes', 'properties'],
+ ['pipeline_instances', 'components'],
+ ['pipeline_instances', 'properties'],
+ ['pipeline_instances', 'components_summary'],
+ ['pipeline_templates', 'components'],
+ ['specimens', 'properties'],
+ ['traits', 'properties'],
+ ['users', 'prefs'],
+ ].each do |table, column|
+ MigrateYAMLToJSON.migrate(table, column)
+ end
+ end
+
+ def down
+ end
+end
--- /dev/null
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class JsonCollectionProperties < ActiveRecord::Migration
+ def up
+ # Drop the FT index before changing column type to avoid
+ # "PG::DatatypeMismatch: ERROR: COALESCE types jsonb and text
+ # cannot be matched".
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
+ FullTextSearchIndexes.new.replace_index('collections')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE text'
+ end
+end
--- /dev/null
+class FixTrashFlagFollow < ActiveRecord::Migration
+ def change
+ ActiveRecord::Base.connection.execute("DROP MATERIALIZED VIEW materialized_permission_view")
+ ActiveRecord::Base.connection.execute(
+"-- constructing perm_edges
+-- 1. get the list of all permission links,
+-- 2. any can_manage link or permission link to a group means permission should 'follow through'
+-- (as a special case, can_manage links to a user grant access to everything owned by the user,
+-- unlike can_read or can_write which only grant access to the user record)
+-- 3. add all owner->owned relationships between groups as can_manage edges
+--
+-- constructing permissions
+-- 1. base case: start with set of all users as the working set
+-- 2. recursive case:
+-- join with edges where the tail is in the working set and 'follow' is true
+-- produce a new working set with the head (target) of each edge
+-- set permission to the least permission encountered on the path
+-- propagate trashed flag down
+
+CREATE MATERIALIZED VIEW materialized_permission_view AS
+WITH RECURSIVE
+perm_value (name, val) AS (
+ VALUES
+ ('can_read', 1::smallint),
+ ('can_login', 1),
+ ('can_write', 2),
+ ('can_manage', 3)
+ ),
+perm_edges (tail_uuid, head_uuid, val, follow, trashed) AS (
+ SELECT links.tail_uuid,
+ links.head_uuid,
+ pv.val,
+ (pv.val = 3 OR groups.uuid IS NOT NULL) AS follow,
+ 0::smallint AS trashed,
+ 0::smallint AS followtrash
+ FROM links
+ LEFT JOIN perm_value pv ON pv.name = links.name
+ LEFT JOIN groups ON pv.val<3 AND groups.uuid = links.head_uuid
+ WHERE links.link_class = 'permission'
+ UNION ALL
+ SELECT owner_uuid, uuid, 3, true,
+ CASE WHEN trash_at IS NOT NULL and trash_at < clock_timestamp() THEN 1 ELSE 0 END,
+ 1
+ FROM groups
+ ),
+perm (val, follow, user_uuid, target_uuid, trashed) AS (
+ SELECT 3::smallint AS val,
+ true AS follow,
+ users.uuid::varchar(32) AS user_uuid,
+ users.uuid::varchar(32) AS target_uuid,
+ 0::smallint AS trashed
+ FROM users
+ UNION
+ SELECT LEAST(perm.val, edges.val)::smallint AS val,
+ edges.follow AS follow,
+ perm.user_uuid::varchar(32) AS user_uuid,
+ edges.head_uuid::varchar(32) AS target_uuid,
+ (GREATEST(perm.trashed, edges.trashed) * edges.followtrash)::smallint AS trashed
+ FROM perm
+ INNER JOIN perm_edges edges
+ ON perm.follow AND edges.tail_uuid = perm.target_uuid
+)
+SELECT user_uuid,
+ target_uuid,
+ MAX(val) AS perm_level,
+ CASE follow WHEN true THEN target_uuid ELSE NULL END AS target_owner_uuid,
+ MAX(trashed) AS trashed
+ FROM perm
+ GROUP BY user_uuid, target_uuid, target_owner_uuid;
+")
+ add_index :materialized_permission_view, [:trashed, :target_uuid], name: 'permission_target_trashed'
+ add_index :materialized_permission_view, [:user_uuid, :trashed, :perm_level], name: 'permission_target_user_trashed_level'
+ ActiveRecord::Base.connection.execute("REFRESH MATERIALIZED VIEW materialized_permission_view")
+ end
+end
--- /dev/null
+class AddGinIndexToCollectionProperties < ActiveRecord::Migration
+ def up
+ ActiveRecord::Base.connection.execute("CREATE INDEX collection_index_on_properties ON collections USING gin (properties);")
+ end
+ def down
+ ActiveRecord::Base.connection.execute("DROP INDEX collection_index_on_properties")
+ end
+end
manifest_text text,
name character varying(255),
description character varying(524288),
- properties text,
+ properties jsonb,
delete_at timestamp without time zone,
file_names character varying(8192),
trash_at timestamp without time zone,
links.head_uuid,
pv.val,
((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
- (0)::smallint AS trashed
+ (0)::smallint AS trashed,
+ (0)::smallint AS followtrash
FROM ((links
LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
LEFT JOIN groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
CASE
WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
ELSE 0
- END AS "case"
+ END AS "case",
+ 1
FROM groups
), perm(val, follow, user_uuid, target_uuid, trashed) AS (
SELECT (3)::smallint AS val,
edges.follow,
perm_1.user_uuid,
(edges.head_uuid)::character varying(32) AS target_uuid,
- (GREATEST((perm_1.trashed)::integer, edges.trashed))::smallint AS trashed
+ ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
FROM (perm perm_1
JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
)
CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, name, key_type, authorized_user_uuid);
+--
+-- Name: collection_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX collection_index_on_properties ON collections USING gin (properties);
+
+
--
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+
+INSERT INTO schema_migrations (version) VALUES ('20170706141334');
+
INSERT INTO schema_migrations (version) VALUES ('20170824202826');
INSERT INTO schema_migrations (version) VALUES ('20170906224040');
INSERT INTO schema_migrations (version) VALUES ('20171027183824');
+INSERT INTO schema_migrations (version) VALUES ('20171208203841');
+
+INSERT INTO schema_migrations (version) VALUES ('20171212153352');
+
# model_class
# Operates on:
# @objects
+
+require 'safe_json'
+
module RecordFilters
# Input:
param_out << operand.split.join(' & ')
end
attrs.each do |attr|
- if !model_class.searchable_columns(operator).index attr.to_s
- raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
- end
- case operator.downcase
- when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
- attr_type = model_class.attribute_column(attr).type
- operator = '<>' if operator == '!='
- if operand.is_a? String
- if attr_type == :boolean
- if not ['=', '<>'].include?(operator)
- raise ArgumentError.new("Invalid operator '#{operator}' for " \
- "boolean attribute '#{attr}'")
- end
- case operand.downcase
- when '1', 't', 'true', 'y', 'yes'
- operand = true
- when '0', 'f', 'false', 'n', 'no'
- operand = false
- else
- raise ArgumentError("Invalid operand '#{operand}' for " \
- "boolean attribute '#{attr}'")
+ subproperty = attr.split(".", 2)
+
+ col = model_class.columns.select { |c| c.name == subproperty[0] }.first
+
+ if subproperty.length == 2
+ if col.nil? or col.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for subproperty filter")
+ end
+
+ if subproperty[1][0] == "<" and subproperty[1][-1] == ">"
+ subproperty[1] = subproperty[1][1..-2]
+ end
+
+ # jsonb search
+ case operator.downcase
+ when '=', '!='
+ not_in = if operator.downcase == "!=" then "NOT " else "" end
+ cond_out << "#{not_in}(#{ar_table_name}.#{subproperty[0]} @> ?::jsonb)"
+ param_out << SafeJSON.dump({subproperty[1] => operand})
+ when 'in'
+ if operand.is_a? Array
+ operand.each do |opr|
+ cond_out << "#{ar_table_name}.#{subproperty[0]} @> ?::jsonb"
+ param_out << SafeJSON.dump({subproperty[1] => opr})
end
- end
- if operator == '<>'
- # explicitly allow NULL
- cond_out << "#{ar_table_name}.#{attr} #{operator} ? OR #{ar_table_name}.#{attr} IS NULL"
else
- cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
- end
- if (# any operator that operates on value rather than
- # representation:
- operator.match(/[<=>]/) and (attr_type == :datetime))
- operand = Time.parse operand
+ raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+ "for '#{operator}' operator in filters")
end
+ when '<', '<=', '>', '>='
+ cond_out << "#{ar_table_name}.#{subproperty[0]}->? #{operator} ?::jsonb"
+ param_out << subproperty[1]
+ param_out << SafeJSON.dump(operand)
+ when 'like', 'ilike'
+ cond_out << "#{ar_table_name}.#{subproperty[0]}->>? #{operator} ?"
+ param_out << subproperty[1]
param_out << operand
- elsif operand.nil? and operator == '='
- cond_out << "#{ar_table_name}.#{attr} is null"
- elsif operand.nil? and operator == '<>'
- cond_out << "#{ar_table_name}.#{attr} is not null"
- elsif (attr_type == :boolean) and ['=', '<>'].include?(operator) and
- [true, false].include?(operand)
- cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
- param_out << operand
+ when 'not in'
+ if operand.is_a? Array
+ cond_out << "#{ar_table_name}.#{subproperty[0]}->>? NOT IN (?) OR #{ar_table_name}.#{subproperty[0]}->>? IS NULL"
+ param_out << subproperty[1]
+ param_out << operand
+ param_out << subproperty[1]
+ else
+ raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+ "for '#{operator}' operator in filters")
+ end
+ when 'exists'
+ if operand == true
+ cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+ elsif operand == false
+ cond_out << "(NOT jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)) OR #{ar_table_name}.#{subproperty[0]} is NULL"
else
- raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
- "for '#{operator}' operator in filters")
+ raise ArgumentError.new("Invalid operand '#{operand}' for '#{operator}' must be true or false")
end
- when 'in', 'not in'
- if operand.is_a? Array
- cond_out << "#{ar_table_name}.#{attr} #{operator} (?)"
- param_out << operand
- if operator == 'not in' and not operand.include?(nil)
- # explicitly allow NULL
- cond_out[-1] = "(#{cond_out[-1]} OR #{ar_table_name}.#{attr} IS NULL)"
- end
+ param_out << subproperty[1]
else
- raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
- "for '#{operator}' operator in filters")
+ raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
end
- when 'is_a'
- operand = [operand] unless operand.is_a? Array
- cond = []
- operand.each do |op|
- cl = ArvadosModel::kind_class op
- if cl
- cond << "#{ar_table_name}.#{attr} like ?"
- param_out << cl.uuid_like_pattern
+ elsif operator.downcase == "exists"
+ if col.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for operator '#{operator}' in filter")
+ end
+
+ cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+ param_out << operand
+ else
+ if !model_class.searchable_columns(operator).index subproperty[0]
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' in filter")
+ end
+
+ case operator.downcase
+ when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
+ attr_type = model_class.attribute_column(attr).type
+ operator = '<>' if operator == '!='
+ if operand.is_a? String
+ if attr_type == :boolean
+ if not ['=', '<>'].include?(operator)
+ raise ArgumentError.new("Invalid operator '#{operator}' for " \
+ "boolean attribute '#{attr}'")
+ end
+ case operand.downcase
+ when '1', 't', 'true', 'y', 'yes'
+ operand = true
+ when '0', 'f', 'false', 'n', 'no'
+ operand = false
+ else
+ raise ArgumentError("Invalid operand '#{operand}' for " \
+ "boolean attribute '#{attr}'")
+ end
+ end
+ if operator == '<>'
+ # explicitly allow NULL
+ cond_out << "#{ar_table_name}.#{attr} #{operator} ? OR #{ar_table_name}.#{attr} IS NULL"
+ else
+ cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+ end
+ if (# any operator that operates on value rather than
+ # representation:
+ operator.match(/[<=>]/) and (attr_type == :datetime))
+ operand = Time.parse operand
+ end
+ param_out << operand
+ elsif operand.nil? and operator == '='
+ cond_out << "#{ar_table_name}.#{attr} is null"
+ elsif operand.nil? and operator == '<>'
+ cond_out << "#{ar_table_name}.#{attr} is not null"
+ elsif (attr_type == :boolean) and ['=', '<>'].include?(operator) and
+ [true, false].include?(operand)
+ cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+ param_out << operand
+ elsif (attr_type == :integer)
+ cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
+ param_out << operand
+ else
+ raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+ "for '#{operator}' operator in filters")
+ end
+ when 'in', 'not in'
+ if operand.is_a? Array
+ cond_out << "#{ar_table_name}.#{attr} #{operator} (?)"
+ param_out << operand
+ if operator == 'not in' and not operand.include?(nil)
+ # explicitly allow NULL
+ cond_out[-1] = "(#{cond_out[-1]} OR #{ar_table_name}.#{attr} IS NULL)"
+ end
else
- cond << "1=0"
+ raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
+ "for '#{operator}' operator in filters")
end
+ when 'is_a'
+ operand = [operand] unless operand.is_a? Array
+ cond = []
+ operand.each do |op|
+ cl = ArvadosModel::kind_class op
+ if cl
+ cond << "#{ar_table_name}.#{attr} like ?"
+ param_out << cl.uuid_like_pattern
+ else
+ cond << "1=0"
+ end
+ end
+ cond_out << cond.join(' OR ')
+ else
+ raise ArgumentError.new("Invalid operator '#{operator}'")
end
- cond_out << cond.join(' OR ')
- else
- raise ArgumentError.new("Invalid operator '#{operator}'")
end
end
conds_out << cond_out.join(' OR ') if cond_out.any?
end
def self.load(s)
- if s.nil?
+ if s.is_a?(object_class)
+ # Rails already deserialized for us
+ s
+ elsif s.nil?
object_class.new()
elsif s[0] == first_json_char
SafeJSON.load(s)
def self.sweep_now
act_as_system_user do
- Collection.unscoped.
+ Collection.
where('delete_at is not null and delete_at < statement_timestamp()').
destroy_all
- Collection.unscoped.
+ Collection.
where('is_trashed = false and trash_at < statement_timestamp()').
update_all('is_trashed = true')
end
user: permission_perftest
api_token: 3kg6k6lzmp9kjabonentustoecn5bahbt2fod9zru30k1jqdmi
expires_at: 2038-01-01 00:00:00
+
+foo_collection_sharing_token:
+ uuid: zzzzz-gj3su-gf02tdm4g1z3e3u
+ api_client: untrusted
+ user: active
+ api_token: iknqgmunrhgsyfok8uzjlwun9iscwm3xacmzmg65fa1j1lpdss
+ expires_at: 2038-01-01 00:00:00
+ scopes:
+ - GET /arvados/v1/collections/zzzzz-4zz18-znfnqtbbv4spc3w
+ - GET /arvados/v1/collections/zzzzz-4zz18-znfnqtbbv4spc3w/
+ - GET /arvados/v1/keep_services/accessible
manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n"
name: collection in trashed subproject
+collection_with_prop1_value1:
+ uuid: zzzzz-4zz18-withprop1value1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 value1
+ properties:
+ prop1: value1
+
+collection_with_prop1_value2:
+ uuid: zzzzz-4zz18-withprop1value2
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 value2
+ properties:
+ prop1: value2
+
+collection_with_prop1_value3:
+ uuid: zzzzz-4zz18-withprop1value3
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 value3
+ properties:
+ prop1: value3
+
+collection_with_prop1_other1:
+ uuid: zzzzz-4zz18-withprop1other1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 other1
+ properties:
+ prop1: other1
+
+collection_with_prop2_1:
+ uuid: zzzzz-4zz18-withprop2value1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 1
+ properties:
+ prop2: 1
+
+collection_with_prop2_5:
+ uuid: zzzzz-4zz18-withprop2value5
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with prop1 5
+ properties:
+ prop2: 5
+
+collection_with_uri_prop:
+ uuid: zzzzz-4zz18-withuripropval1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with RDF-style URI property key
+ properties:
+ "http://schema.org/example": "value1"
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
ram: 256000000
API: true
+cr_in_trashed_project:
+ uuid: zzzzz-xvhdp-cr5trashedcontr
+ owner_uuid: zzzzz-j7d0g-trashedproject1
+ name: completed container request
+ state: Final
+ priority: 1
+ created_at: <%= 2.minute.ago.to_s(:db) %>
+ updated_at: <%= 1.minute.ago.to_s(:db) %>
+ modified_at: <%= 1.minute.ago.to_s(:db) %>
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: test
+ cwd: test
+ output_path: test
+ command: ["echo", "hello"]
+ container_uuid: zzzzz-dz642-compltcontainer
+ log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 123
+
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_equal c.delete_at, c.trash_at + Rails.configuration.blob_signature_ttl
end
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_operator c.delete_at, :<, db_current_time
end
id: uuid,
}
assert_response 200
- c = Collection.unscoped.find_by_uuid(uuid)
+ c = Collection.find_by_uuid(uuid)
assert_operator c.trash_at, :<, db_current_time
assert_operator c.delete_at, :>=, time_before_trashing + Rails.configuration.default_trash_lifetime
end
assert_equal all_objects['arvados#pipelineInstance'], second_page['arvados#pipelineInstance']+5
assert_equal true, second_page['arvados#pipelineTemplate']>0
end
+
+ [['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
+ ['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
+ ['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
+ ['prop1', 'exists', false, [:collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1]],
+ ['prop1', 'in', ['value1', 'value2'], [:collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3, :collection_with_prop2_1]],
+ ['prop1', 'in', ['value1', 'valueX'], [:collection_with_prop1_value1], [:collection_with_prop1_value3, :collection_with_prop2_1]],
+ ['prop1', 'not in', ['value1', 'value2'], [:collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2]],
+ ['prop1', 'not in', ['value1', 'valueX'], [:collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1]],
+ ['prop1', '>', 'value2', [:collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
+ ['prop1', '<', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop1_value2]],
+ ['prop1', '<=', 'value2', [:collection_with_prop1_other1, :collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3]],
+ ['prop1', '>=', 'value2', [:collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1, :collection_with_prop1_value1]],
+ ['prop1', 'like', 'value%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
+ ['prop1', 'like', '%1', [:collection_with_prop1_value1, :collection_with_prop1_other1], [:collection_with_prop1_value2, :collection_with_prop1_value3]],
+ ['prop1', 'ilike', 'VALUE%', [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3], [:collection_with_prop1_other1]],
+ ['prop2', '>', 1, [:collection_with_prop2_5], [:collection_with_prop2_1]],
+ ['prop2', '<', 5, [:collection_with_prop2_1], [:collection_with_prop2_5]],
+ ['prop2', '<=', 5, [:collection_with_prop2_1, :collection_with_prop2_5], []],
+ ['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []],
+ ['<http://schema.org/example>', '=', "value1", [:collection_with_uri_prop], []],
+ ].each do |prop, op, opr, inc, ex|
+ test "jsonb filter properties.#{prop} #{op} #{opr})" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: SafeJSON.dump([ ["properties.#{prop}", op, opr] ]),
+ limit: 1000
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+
+ inc.each do |i|
+ assert_includes(found, collections(i).uuid)
+ end
+
+ ex.each do |e|
+ assert_not_includes(found, collections(e).uuid)
+ end
+ end
+ end
+
+ test "jsonb 'exists' and '!=' filter" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['properties.prop1', 'exists', true], ['properties.prop1', '!=', 'value1'] ]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found.length, 3
+ assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value2).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value3).uuid)
+ assert_includes(found, collections(:collection_with_prop1_other1).uuid)
+ end
+
+ test "jsonb alternate form 'exists' and '!=' filter" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['properties', 'exists', 'prop1'], ['properties.prop1', '!=', 'value1'] ]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found.length, 3
+ assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value2).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value3).uuid)
+ assert_includes(found, collections(:collection_with_prop1_other1).uuid)
+ end
+
+ test "jsonb 'exists' must be boolean" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['properties.prop1', 'exists', nil] ]
+ }
+ assert_response 422
+ assert_match(/Invalid operand '' for 'exists' must be true or false/,
+ json_response['errors'].join(' '))
+ end
+
+ test "jsonb checks column exists" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['puppies.prop1', '=', 'value1'] ]
+ }
+ assert_response 422
+ assert_match(/Invalid attribute 'puppies' for subproperty filter/,
+ json_response['errors'].join(' '))
+ end
+
+ test "jsonb checks column is valid" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['name.prop1', '=', 'value1'] ]
+ }
+ assert_response 422
+ assert_match(/Invalid attribute 'name' for subproperty filter/,
+ json_response['errors'].join(' '))
+ end
+
+ test "jsonb invalid operator" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['properties.prop1', '###', 'value1'] ]
+ }
+ assert_response 422
+ assert_match(/Invalid operator for subproperty search '###'/,
+ json_response['errors'].join(' '))
+ end
+
+ test "replication_desired = 2" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: SafeJSON.dump([ ['replication_desired', '=', 2] ])
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
+ assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
+ end
+
end
assert_response :success
assert_not_nil Group.readable_by(users(auth)).where(uuid: groups(:trashed_subproject).uuid).first
end
-
end
end
FileUtils.mkdir_p @tmpdir
system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
-
- # Initialize an empty internal git repo.
- intdir =
- Rails.configuration.git_internal_dir =
- Rails.root.join(@tmpdir, 'internal.git').to_s
- FileUtils.mkdir_p intdir
- IO.read("|git --git-dir #{intdir.shellescape} init")
- assert $?.success?
+ Rails.configuration.git_internal_dir = "#{@tmpdir}/internal.git"
end
base.teardown do
- FileUtils.remove_entry @tmpdir, true
FileUtils.remove_entry Commit.cache_dir_base, true
+ FileUtils.mkdir_p @tmpdir
+ system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
end
end
%w(GET HEAD POST PUT DELETE).each do |m|
assert_includes allowed, m, "A-C-A-Methods should include #{m}"
end
- assert_equal 'Authorization', response.headers['Access-Control-Allow-Headers']
+ assert_equal 'Authorization, Content-Type', response.headers['Access-Control-Allow-Headers']
end
def assert_no_cors_headers
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'webrick'
+require 'webrick/https'
+require 'test_helper'
+require 'helpers/users_test_helper'
+
+class RemoteUsersTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
+
+ def salted_active_token(remote:)
+ salt_token(fixture: :active, remote: remote).sub('/zzzzz-', '/'+remote+'-')
+ end
+
+ def auth(remote:)
+ token = salted_active_token(remote: remote)
+ {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ end
+
+ # For remote authentication tests, we bring up a simple stub server
+ # (on a port chosen by webrick) and configure the SUT so the stub is
+ # responsible for clusters "zbbbb" (a well-behaved cluster) and
+ # "zbork" (a misbehaving cluster).
+ #
+ # Test cases can override the stub's default response to
+ # .../users/current by changing @stub_status and @stub_content.
+ setup do
+ clnt = HTTPClient.new
+ clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ HTTPClient.stubs(:new).returns clnt
+
+ @controller = Arvados::V1::UsersController.new
+ ready = Thread::Queue.new
+ srv = WEBrick::HTTPServer.new(
+ Port: 0,
+ Logger: WEBrick::Log.new(
+ Rails.root.join("log", "webrick.log").to_s,
+ WEBrick::Log::INFO),
+ AccessLog: [[File.open(Rails.root.join(
+ "log", "webrick_access.log").to_s, 'a+'),
+ WEBrick::AccessLog::COMBINED_LOG_FORMAT]],
+ SSLEnable: true,
+ SSLVerifyClient: OpenSSL::SSL::VERIFY_NONE,
+ SSLPrivateKey: OpenSSL::PKey::RSA.new(
+ File.open(Rails.root.join("tmp", "self-signed.key")).read),
+ SSLCertificate: OpenSSL::X509::Certificate.new(
+ File.open(Rails.root.join("tmp", "self-signed.pem")).read),
+ SSLCertName: [["CN", WEBrick::Utils::getservername]],
+ StartCallback: lambda { ready.push(true) })
+ srv.mount_proc '/discovery/v1/apis/arvados/v1/rest' do |req, res|
+ Rails.cache.delete 'arvados_v1_rest_discovery'
+ res.body = Arvados::V1::SchemaController.new.send(:discovery_doc).to_json
+ end
+ srv.mount_proc '/arvados/v1/users/current' do |req, res|
+ res.status = @stub_status
+ res.body = @stub_content.is_a?(String) ? @stub_content : @stub_content.to_json
+ end
+ Thread.new do
+ srv.start
+ end
+ ready.pop
+ @remote_server = srv
+ @remote_host = "127.0.0.1:#{srv.config[:Port]}"
+ Rails.configuration.remote_hosts['zbbbb'] = @remote_host
+ Rails.configuration.remote_hosts['zbork'] = @remote_host
+ Arvados::V1::SchemaController.any_instance.stubs(:root_url).returns "https://#{@remote_host}"
+ @stub_status = 200
+ @stub_content = {
+ uuid: 'zbbbb-tpzed-000000000000000',
+ email: 'foo@example.com',
+ username: 'barney',
+ is_admin: true,
+ is_active: true,
+ }
+ end
+
+ teardown do
+ @remote_server.andand.stop
+ end
+
+ test 'authenticate with remote token' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid']
+ assert_equal false, json_response['is_admin']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+
+ # revoke original token
+ @stub_status = 401
+
+ # re-authorize before cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+
+ # simulate cache expiry
+ ApiClientAuthorization.where(
+ uuid: salted_active_token(remote: 'zbbbb').split('/')[1]).
+ update_all(expires_at: db_current_time - 1.minute)
+
+ # re-authorize after cache expires
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+
+ # revive original token and re-authorize
+ @stub_status = 200
+ @stub_content[:username] = 'blarney'
+ @stub_content[:email] = 'blarney@example.com'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'barney', json_response['username'], 'local username should not change once assigned'
+ assert_equal 'blarney@example.com', json_response['email']
+ end
+
+ test 'authenticate with remote token, remote username conflicts with local' do
+ @stub_content[:username] = 'active'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'active2', json_response['username']
+ end
+
+ test 'authenticate with remote token, remote username is nil' do
+ @stub_content.delete :username
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'foo', json_response['username']
+ end
+
+ test 'authenticate with remote token from misbhehaving remote cluster' do
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbork')
+ assert_response 401
+ end
+
+ test 'authenticate with remote token that fails validate' do
+ @stub_status = 401
+ @stub_content = {
+ error: 'not authorized',
+ }
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['v2',
+ 'v2/',
+ 'v2//',
+ 'v2///',
+ "v2/'; delete from users where 1=1; commit; select '/lol",
+ 'v2/foo/bar',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2/zzzzz-gj3su-077z32aux8dg2s1/',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v2/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi/zzzzz-gj3su-077z32aux8dg2s1',
+ 'v2//3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ 'v8/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi',
+ '"v2/zzzzz-gj3su-077z32aux8dg2s1/3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"',
+ '/',
+ '//',
+ '///',
+ ].each do |token|
+ test "authenticate with malformed remote token #{token}" do
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response 401
+ end
+ end
+
+ test "ignore extra fields in remote token" do
+ token = salted_active_token(remote: 'zbbbb') + '/foo/bar/baz/*'
+ get '/arvados/v1/users/current', {format: 'json'}, {"HTTP_AUTHORIZATION" => "Bearer #{token}"}
+ assert_response :success
+ end
+
+ test 'remote api server is not an api server' do
+ @stub_status = 200
+ @stub_content = '<html>bad</html>'
+ get '/arvados/v1/users/current', {format: 'json'}, auth(remote: 'zbbbb')
+ assert_response 401
+ end
+
+ ['zbbbb', 'z0000'].each do |token_valid_for|
+ test "validate #{token_valid_for}-salted token for remote cluster zbbbb" do
+ salted_token = salt_token(fixture: :active, remote: token_valid_for)
+ get '/arvados/v1/users/current', {format: 'json', remote: 'zbbbb'}, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ if token_valid_for == 'zbbbb'
+ assert_response 200
+ assert_equal(users(:active).uuid, json_response['uuid'])
+ else
+ assert_response 401
+ end
+ end
+ end
+
+ test "list readable groups with salted token" do
+ salted_token = salt_token(fixture: :active, remote: 'zbbbb')
+ get '/arvados/v1/groups', {
+ format: 'json',
+ remote: 'zbbbb',
+ limit: 10000,
+ }, {
+ "HTTP_AUTHORIZATION" => "Bearer #{salted_token}"
+ }
+ assert_response 200
+ group_uuids = json_response['items'].collect { |i| i['uuid'] }
+ assert_includes(group_uuids, 'zzzzz-j7d0g-fffffffffffffff')
+ refute_includes(group_uuids, 'zzzzz-j7d0g-000000000000000')
+ assert_includes(group_uuids, groups(:aproject).uuid)
+ refute_includes(group_uuids, groups(:trashed_project).uuid)
+ refute_includes(group_uuids, groups(:testusergroup_admins).uuid)
+ end
+end
"HTTP_AUTHORIZATION" => "OAuth2 #{t}")
end
+ def salt_token(fixture:, remote:)
+ auth = api_client_authorizations(fixture)
+ uuid = auth.uuid
+ token = auth.api_token
+ hmac = OpenSSL::HMAC.hexdigest('sha1', token, remote)
+ return "v2/#{uuid}/#{hmac}"
+ end
+
def self.skip_slow_tests?
!(ENV['RAILS_TEST_SHORT'] || '').empty?
end
assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
end
+
+ test 'jsonb column' do
+ set_user_from_auth :active
+
+ c = Collection.create!(properties: {})
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {'foo' => 'foo'})
+ c.reload
+ assert_equal({'foo' => 'foo'}, c.properties)
+
+ c.update_attributes(properties: nil)
+ c.reload
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {foo: 'bar'})
+ assert_equal({'foo' => 'bar'}, c.properties)
+ c.reload
+ assert_equal({'foo' => 'bar'}, c.properties)
+ end
end
assert c.valid?
uuid = c.uuid
+ c = Collection.readable_by(current_user).where(uuid: uuid)
+ assert_not_empty c, 'Should be able to find live collection'
+
# mark collection as expired
- c.update_attributes!(trash_at: Time.new.strftime("%Y-%m-%d"))
- c = Collection.where(uuid: uuid)
+ c.first.update_attributes!(trash_at: Time.new.strftime("%Y-%m-%d"))
+ c = Collection.readable_by(current_user).where(uuid: uuid)
assert_empty c, 'Should not be able to find expired collection'
# recreate collection with the same name
if fixture_name == :expired_collection
# Fixture-finder shorthand doesn't find trashed collections
# because they're not in the default scope.
- c = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3ih')
+ c = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3ih')
else
c = collections(fixture_name)
end
end
end
SweepTrashedCollections.sweep_now
- c = Collection.unscoped.where('uuid=? and is_trashed=true', c.uuid).first
+ c = Collection.where('uuid=? and is_trashed=true', c.uuid).first
assert c
act_as_user users(:active) do
assert Collection.create!(owner_uuid: c.owner_uuid,
test "delete in SweepTrashedCollections" do
uuid = 'zzzzz-4zz18-3u1p5umicfpqszp' # deleted_on_next_sweep
- assert_not_empty Collection.unscoped.where(uuid: uuid)
+ assert_not_empty Collection.where(uuid: uuid)
SweepTrashedCollections.sweep_now
- assert_empty Collection.unscoped.where(uuid: uuid)
+ assert_empty Collection.where(uuid: uuid)
end
test "delete referring links in SweepTrashedCollections" do
name: 'something')
end
past = db_current_time
- Collection.unscoped.where(uuid: uuid).
+ Collection.where(uuid: uuid).
update_all(is_trashed: true, trash_at: past, delete_at: past)
- assert_not_empty Collection.unscoped.where(uuid: uuid)
+ assert_not_empty Collection.where(uuid: uuid)
SweepTrashedCollections.sweep_now
- assert_empty Collection.unscoped.where(uuid: uuid)
+ assert_empty Collection.where(uuid: uuid)
end
end
c.lock
c.update_attributes! state: Container::Running
- output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
+ output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
assert output.is_trashed
assert c.update_attributes output: output.portable_data_hash
c.lock
c.update_attributes! state: Container::Running
- output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
+ output = Collection.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
assert Group.readable_by(users(:active)).where(uuid: g_baz.uuid).any?
end
+ test "trashed does not propagate across permission links" do
+ set_user_from_auth :admin
+
+ g_foo = Group.create!(name: "foo")
+ u_bar = User.create!(first_name: "bar")
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+
+ g_foo.update! is_trashed: false
+ ln = Link.create!(tail_uuid: g_foo.uuid,
+ head_uuid: u_bar.uuid,
+ link_class: "permission",
+ name: "can_read")
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ end
+
end
package main
import (
+ "errors"
"log"
"net/http"
"os"
+ "regexp"
"strings"
"sync"
"time"
log.Fatal(err)
}
h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
- log.Printf("%+v", h.clientPool.Prototype)
}
func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Nobody has called WriteHeader yet: that
// must be our job.
w.WriteHeader(statusCode)
- w.Write([]byte(statusText))
+ if statusCode >= 400 {
+ w.Write([]byte(statusText))
+ }
}
// If the given password is a valid token, log the first 10 characters of the token.
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
arv.ApiToken = apiToken
- reposFound := arvadosclient.Dict{}
- if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{"name", "=", repoName}},
- }, &reposFound); err != nil {
+ repoUUID, err := h.lookupRepo(arv, repoName)
+ if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
validApiToken = true
- if avail, ok := reposFound["items_available"].(float64); !ok {
- statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
- return
- } else if avail < 1 {
+ if repoUUID == "" {
statusCode, statusText = http.StatusNotFound, "not found"
return
- } else if avail > 1 {
- statusCode, statusText = http.StatusInternalServerError, "name collision"
- return
}
- repoUUID := reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string)
-
isWrite := strings.HasSuffix(r.URL.Path, "/git-receive-pack")
if !isWrite {
statusText = "read"
h.handler.ServeHTTP(w, r)
}
+
+var uuidRegexp = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
+
+func (h *authHandler) lookupRepo(arv *arvadosclient.ArvadosClient, repoName string) (string, error) {
+ reposFound := arvadosclient.Dict{}
+ var column string
+ if uuidRegexp.MatchString(repoName) {
+ column = "uuid"
+ } else {
+ column = "name"
+ }
+ err := arv.List("repositories", arvadosclient.Dict{
+ "filters": [][]string{{column, "=", repoName}},
+ }, &reposFound)
+ if err != nil {
+ return "", err
+ } else if avail, ok := reposFound["items_available"].(float64); !ok {
+ return "", errors.New("bad list response from API")
+ } else if avail < 1 {
+ return "", nil
+ } else if avail > 1 {
+ return "", errors.New("name collision")
+ }
+ return reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string), nil
+}
package main
import (
+ "io"
+ "log"
"net/http"
"net/http/httptest"
"net/url"
+ "path/filepath"
+ "strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
type AuthHandlerSuite struct{}
+func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+}
+
+func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopAPI()
+}
+
+func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
+ arvadostest.ResetEnv()
+ repoRoot, err := filepath.Abs("../api/tmp/git/test")
+ c.Assert(err, check.IsNil)
+ theConfig = &Config{
+ Client: arvados.Client{
+ APIHost: arvadostest.APIHost(),
+ Insecure: true,
+ },
+ Listen: ":0",
+ GitCommand: "/usr/bin/git",
+ RepoRoot: repoRoot,
+ ManagementToken: arvadostest.ManagementToken,
+ }
+}
+
+func (s *AuthHandlerSuite) TestPermission(c *check.C) {
+ h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ log.Printf("%v", r.URL)
+ io.WriteString(w, r.URL.Path)
+ })}
+ baseURL, err := url.Parse("http://git.example/")
+ c.Assert(err, check.IsNil)
+ for _, trial := range []struct {
+ label string
+ token string
+ pathIn string
+ pathOut string
+ status int
+ }{
+ {
+ label: "read repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "read repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "write repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "write repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "uuid not found",
+ token: arvadostest.ActiveToken,
+ pathIn: strings.Replace(arvadostest.Repository2UUID, "6", "z", -1) + ".git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "name not found",
+ token: arvadostest.ActiveToken,
+ pathIn: "nonexistent-bogus.git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "read read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-upload-pack",
+ pathOut: arvadostest.FooRepoUUID + "/.git/git-upload-pack",
+ },
+ {
+ label: "write read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-receive-pack",
+ status: http.StatusForbidden,
+ },
+ } {
+ c.Logf("trial label: %q", trial.label)
+ u, err := baseURL.Parse(trial.pathIn)
+ c.Assert(err, check.IsNil)
+ resp := httptest.NewRecorder()
+ req := &http.Request{
+ Method: "POST",
+ URL: u,
+ Header: http.Header{
+ "Authorization": {"Bearer " + trial.token}}}
+ h.ServeHTTP(resp, req)
+ if trial.status == 0 {
+ trial.status = http.StatusOK
+ }
+ c.Check(resp.Code, check.Equals, trial.status)
+ if trial.status < 400 {
+ if trial.pathOut != "" && !strings.HasPrefix(trial.pathOut, "/") {
+ trial.pathOut = "/" + trial.pathOut
+ }
+ c.Check(resp.Body.String(), check.Equals, trial.pathOut)
+ }
+ }
+}
+
func (s *AuthHandlerSuite) TestCORS(c *check.C) {
h := &authHandler{}
// Send a tiny script on stdin to execute the crunch-run
// command (slurm requires this to be a #! script)
- cmd.Stdin = strings.NewReader(execScript(append(crunchRunCommand, container.UUID)))
+
+ // append() here avoids modifying crunchRunCommand's
+ // underlying array, which is shared with other goroutines.
+ args := append([]string(nil), crunchRunCommand...)
+ args = append(args, container.UUID)
+ cmd.Stdin = strings.NewReader(execScript(args))
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
return fmt.Errorf("While creating keep mount temp dir: %v", err)
}
+ token, err := runner.ContainerToken()
+ if err != nil {
+ return fmt.Errorf("could not get container token: %s", err)
+ }
+
pdhOnly := true
tmpcount := 0
arvMountCmd := []string{
return fmt.Errorf("writing temp file: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+ case mnt.Kind == "git_tree":
+ tmpdir, err := runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("creating temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+ err = gitMount(mnt).extractTree(runner.ArvClient, tmpdir, token)
+ if err != nil {
+ return err
+ }
+ runner.Binds = append(runner.Binds, tmpdir+":"+bind+":ro")
}
}
}
arvMountCmd = append(arvMountCmd, runner.ArvMountPoint)
- token, err := runner.ContainerToken()
- if err != nil {
- return fmt.Errorf("could not get container token: %s", err)
- }
-
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
return fmt.Errorf("While trying to start arv-mount: %v", err)
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/manifest"
dockertypes "github.com/docker/docker/api/types"
cr.CleanupDirs()
checkEmpty()
}
+
+ // git_tree mounts
+ {
+ i = 0
+ cr.ArvMountPoint = ""
+ (*GitMountSuite)(nil).useTestGitServer(c)
+ cr.token = arvadostest.ActiveToken
+ cr.Container.Mounts = make(map[string]arvados.Mount)
+ cr.Container.Mounts = map[string]arvados.Mount{
+ "/tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ Path: "/",
+ },
+ "/non-tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ Path: "/",
+ },
+ }
+ cr.OutputPath = "/tmp"
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+
+ // dirMap[mountpoint] == tmpdir
+ dirMap := make(map[string]string)
+ for _, bind := range cr.Binds {
+ tokens := strings.Split(bind, ":")
+ dirMap[tokens[1]] = tokens[0]
+
+ if cr.Container.Mounts[tokens[1]].Writable {
+ c.Check(len(tokens), Equals, 2)
+ } else {
+ c.Check(len(tokens), Equals, 3)
+ c.Check(tokens[2], Equals, "ro")
+ }
+ }
+
+ data, err := ioutil.ReadFile(dirMap["/tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ _, err = ioutil.ReadFile(dirMap["/tip"] + "/file only on testbranch")
+ c.Check(err, FitsTypeOf, &os.PathError{})
+ c.Check(os.IsNotExist(err), Equals, true)
+
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/file only on testbranch")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "testfile\n")
+
+ cr.CleanupDirs()
+ checkEmpty()
+ }
}
func (s *TestSuite) TestStdout(c *C) {
--- /dev/null
+package main
+
+import (
+ "fmt"
+ "net/url"
+ "os"
+ "path/filepath"
+ "regexp"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "gopkg.in/src-d/go-billy.v3/osfs"
+ git "gopkg.in/src-d/go-git.v4"
+ git_config "gopkg.in/src-d/go-git.v4/config"
+ git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+ "gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type gitMount arvados.Mount
+
+var (
+ sha1re = regexp.MustCompile(`^[0-9a-f]{40}$`)
+ repoUUIDre = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
+)
+
+func (gm gitMount) validate() error {
+ if gm.Path != "" && gm.Path != "/" {
+ return fmt.Errorf("cannot mount git_tree with path %q -- only \"/\" is supported", gm.Path)
+ }
+ if !sha1re.MatchString(gm.Commit) {
+ return fmt.Errorf("cannot mount git_tree with commit %q -- must be a 40-char SHA1", gm.Commit)
+ }
+ if gm.RepositoryName != "" || gm.GitURL != "" {
+ return fmt.Errorf("cannot mount git_tree -- repository_name and git_url must be empty")
+ }
+ if !repoUUIDre.MatchString(gm.UUID) {
+ return fmt.Errorf("cannot mount git_tree with uuid %q -- must be a repository UUID", gm.UUID)
+ }
+ if gm.Writable {
+ return fmt.Errorf("writable git_tree mount is not supported")
+ }
+ return nil
+}
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm gitMount) extractTree(ac IArvadosClient, dir string, token string) error {
+ err := gm.validate()
+ if err != nil {
+ return err
+ }
+ baseURL, err := ac.Discovery("gitUrl")
+ if err != nil {
+ return fmt.Errorf("discover gitUrl from API: %s", err)
+ } else if _, ok := baseURL.(string); !ok {
+ return fmt.Errorf("discover gitUrl from API: expected string, found %T", baseURL)
+ }
+
+ u, err := url.Parse(baseURL.(string))
+ if err != nil {
+ return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
+ }
+ u, err = u.Parse("/" + gm.UUID + ".git")
+ if err != nil {
+ return fmt.Errorf("build git url from %q, %q: %s", baseURL, gm.UUID, err)
+ }
+ store := memory.NewStorage()
+ repo, err := git.Init(store, osfs.New(dir))
+ if err != nil {
+ return fmt.Errorf("init repo: %s", err)
+ }
+ _, err = repo.CreateRemote(&git_config.RemoteConfig{
+ Name: "origin",
+ URLs: []string{u.String()},
+ })
+ if err != nil {
+ return fmt.Errorf("create remote %q: %s", u.String(), err)
+ }
+ err = repo.Fetch(&git.FetchOptions{
+ RemoteName: "origin",
+ Auth: git_http.NewBasicAuth("none", token),
+ })
+ if err != nil {
+ return fmt.Errorf("git fetch %q: %s", u.String(), err)
+ }
+ wt, err := repo.Worktree()
+ if err != nil {
+ return fmt.Errorf("worktree failed: %s", err)
+ }
+ err = wt.Checkout(&git.CheckoutOptions{
+ Hash: git_plumbing.NewHash(gm.Commit),
+ })
+ if err != nil {
+ return fmt.Errorf("checkout failed: %s", err)
+ }
+ err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ // copy user rx bits to group and other, in case
+ // prevailing umask is more restrictive than 022
+ mode := info.Mode()
+ mode = mode | ((mode >> 3) & 050) | ((mode >> 6) & 5)
+ return os.Chmod(path, mode)
+ })
+ if err != nil {
+ return fmt.Errorf("chmod -R %q: %s", dir, err)
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+ git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+)
+
+type GitMountSuite struct {
+ tmpdir string
+}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) SetUpTest(c *check.C) {
+ s.useTestGitServer(c)
+
+ var err error
+ s.tmpdir, err = ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+}
+
+func (s *GitMountSuite) TearDownTest(c *check.C) {
+ err := os.RemoveAll(s.tmpdir)
+ c.Check(err, check.IsNil)
+}
+
+// Commit fd3531f is crunch-run-tree-test
+func (s *GitMountSuite) TestextractTree(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
+ fi, err := os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
+ }
+
+ fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
+ data, err = ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
+ fi, err = os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
+ }
+
+ // Ensure there's no extra stuff like a ".git" dir
+ s.checkTmpdirContents(c, []string{"dir1"})
+
+ // Ensure tmpdir is world-readable and world-executable so the
+ // UID inside the container can use it.
+ fi, err = os.Stat(s.tmpdir)
+ c.Check(err, check.IsNil)
+ c.Check(fi.Mode()&os.ModePerm, check.Equals, os.FileMode(0755))
+}
+
+// Commit 5ebfab0 is not the tip of any branch or tag, but is
+// reachable in branch "crunch-run-non-tip-test".
+func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
+ gm := gitMount{
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "file only on testbranch")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "testfile\n")
+}
+
+func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: "zzzzz-s0uqq-nonexistentrepo",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*repository not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*object not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestGitUrlDiscoveryFails(c *check.C) {
+ delete(discoveryMap, "gitUrl")
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.ErrorMatches, ".*gitUrl.*")
+}
+
+func (s *GitMountSuite) TestInvalid(c *check.C) {
+ for _, trial := range []struct {
+ gm gitMount
+ matcher string
+ }{
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "abc123",
+ },
+ matcher: ".*SHA1.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ RepositoryName: arvadostest.Repository2Name,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*repository_name.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ GitURL: "https://localhost:0/" + arvadostest.Repository2Name + ".git",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*git_url.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/dir1/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*path.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*UUID.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ Writable: true,
+ },
+ matcher: ".*writable.*",
+ },
+ } {
+ err := trial.gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ s.checkTmpdirContents(c, []string{})
+
+ err = trial.gm.validate()
+ c.Check(err, check.ErrorMatches, trial.matcher)
+ }
+}
+
+func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
+ f, err := os.Open(s.tmpdir)
+ c.Check(err, check.IsNil)
+ names, err := f.Readdirnames(-1)
+ c.Check(err, check.IsNil)
+ c.Check(names, check.DeepEquals, expect)
+}
+
+func (*GitMountSuite) useTestGitServer(c *check.C) {
+ git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+}
from setuptools import setup, find_packages
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name="arvados-docker-cleaner",
- version="0.1",
+ version=version,
description="Arvados Docker cleaner",
author="Arvados",
author_email="info@arvados.org",
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados_fuse',
- version='0.1',
+ version=version,
description='Arvados FUSE driver',
long_description=open(README).read(),
author='Arvados',
h := webdav.Handler{
Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
FileSystem: &webdavFS{
- collfs: fs,
- writing: writeMethod[r.Method],
+ collfs: fs,
+ writing: writeMethod[r.Method],
+ alwaysReadEOF: r.Method == "PROPFIND",
},
LockSystem: h.webdavLS,
Logger: func(_ *http.Request, err error) {
"crypto/rand"
"errors"
"fmt"
+ "io"
prand "math/rand"
"os"
"path"
type webdavFS struct {
collfs arvados.CollectionFileSystem
writing bool
+ // webdav PROPFIND reads the first few bytes of each file
+ // whose filename extension isn't recognized, which is
+ // prohibitively expensive: we end up fetching multiple 64MiB
+ // blocks. Avoid this by returning EOF on all reads when
+ // handling a PROPFIND.
+ alwaysReadEOF bool
}
func (fs *webdavFS) makeparents(name string) {
// have 405.
f = writeFailer{File: f, err: errReadOnly}
}
+ if fs.alwaysReadEOF {
+ f = readEOF{File: f}
+ }
return
}
return wf.err
}
+type readEOF struct {
+ webdav.File
+}
+
+func (readEOF) Read(p []byte) (int, error) {
+ return 0, io.EOF
+}
+
// noLockSystem implements webdav.LockSystem by returning success for
// every possible locking operation, even though it has no side
// effects such as actually locking anything. This works for a
}
func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
- var auth string
- if auth = req.Header.Get("Authorization"); auth == "" {
+ parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
+ if len(parts) < 2 || !(parts[0] == "OAuth2" || parts[0] == "Bearer") || len(parts[1]) == 0 {
return false, ""
}
+ tok = parts[1]
- _, err := fmt.Sscanf(auth, "OAuth2 %s", &tok)
- if err != nil {
- // Scanning error
- return false, ""
+ // Tokens are validated differently depending on what kind of
+ // operation is being performed. For example, tokens in
+ // collection-sharing links permit GET requests, but not
+ // PUT requests.
+ var op string
+ if req.Method == "GET" || req.Method == "HEAD" {
+ op = "read"
+ } else {
+ op = "write"
}
- if cache.RecallToken(tok) {
+ if cache.RecallToken(op + ":" + tok) {
// Valid in the cache, short circuit
return true, tok
}
+ var err error
arv := *kc.Arvados
arv.ApiToken = tok
- if err := arv.Call("HEAD", "users", "", "current", nil, nil); err != nil {
+ if op == "read" {
+ err = arv.Call("HEAD", "keep_services", "", "accessible", nil, nil)
+ } else {
+ err = arv.Call("HEAD", "users", "", "current", nil, nil)
+ }
+ if err != nil {
log.Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
return false, ""
}
// Success! Update cache
- cache.RememberToken(tok)
+ cache.RememberToken(op + ":" + tok)
return true, tok
}
kc := runProxy(c, nil, true)
defer closeListener()
- hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
+ hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
- {
- _, _, err := kc.Ask(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
- c.Log("Ask 1")
- }
+ _, _, err := kc.Ask(hash)
+ c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
- {
- hash2, rep, err := kc.PutB([]byte("bar"))
- c.Check(hash2, Equals, "")
- c.Check(rep, Equals, 0)
- c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
- c.Log("PutB")
- }
+ hash2, rep, err := kc.PutB([]byte("bar"))
+ c.Check(hash2, Equals, "")
+ c.Check(rep, Equals, 0)
+ c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
- {
- blocklen, _, err := kc.Ask(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
- c.Check(blocklen, Equals, int64(0))
- c.Log("Ask 2")
- }
+ blocklen, _, err := kc.Ask(hash)
+ c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
+ c.Check(err, ErrorMatches, ".*not found.*")
+ c.Check(blocklen, Equals, int64(0))
+
+ _, blocklen, _, err = kc.Get(hash)
+ c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
+ c.Check(err, ErrorMatches, ".*not found.*")
+ c.Check(blocklen, Equals, int64(0))
- {
- _, blocklen, _, err := kc.Get(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
- c.Check(blocklen, Equals, int64(0))
- c.Log("Get")
- }
}
func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
c.Assert((err != nil), Equals, true)
}
+func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
+ kc := runProxy(c, nil, false)
+ defer closeListener()
+ hash, _, err := kc.PutB([]byte("shareddata"))
+ c.Check(err, IsNil)
+ kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
+ rdr, _, _, err := kc.Get(hash)
+ c.Assert(err, IsNil)
+ data, err := ioutil.ReadAll(rdr)
+ c.Check(err, IsNil)
+ c.Check(data, DeepEquals, []byte("shareddata"))
+}
+
func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
kc := runProxy(c, nil, false)
defer closeListener()
// Put a test block
hash, rep, err := kc.PutB([]byte("foo"))
- c.Check(err, Equals, nil)
+ c.Check(err, IsNil)
c.Check(rep, Equals, 2)
- for _, token := range []string{
+ for _, badToken := range []string{
"nosuchtoken",
"2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
} {
- // Change token to given bad token
- kc.Arvados.ApiToken = token
+ kc.Arvados.ApiToken = badToken
+
+ // Ask and Get will fail only if the upstream
+ // keepstore server checks for valid signatures.
+ // Without knowing the blob signing key, there is no
+ // way for keepproxy to know whether a given token is
+ // permitted to read a block. So these tests fail:
+ if false {
+ _, _, err = kc.Ask(hash)
+ c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
+ c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
+ c.Check(err, ErrorMatches, ".*HTTP 403.*")
+
+ _, _, _, err = kc.Get(hash)
+ c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
+ c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
+ c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
+ }
- // Ask should result in error
- _, _, err = kc.Ask(hash)
- c.Check(err, NotNil)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound.Temporary(), Equals, false)
- c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
-
- // Get should result in error
- _, _, _, err = kc.Get(hash)
- c.Check(err, NotNil)
- errNotFound, _ = err.(keepclient.ErrNotFound)
- c.Check(errNotFound.Temporary(), Equals, false)
- c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
+ _, _, err = kc.PutB([]byte("foo"))
+ c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
}
}
('unpaired', 'open', 'boot exceeded', 'not idle'): "START_SHUTDOWN",
('unpaired', 'open', 'boot wait', 'idle exceeded'): None,
('unpaired', 'open', 'boot wait', 'idle wait'): None,
- ('unpaired', 'open', 'boot wait', 'not idle'): None}
+ ('unpaired', 'open', 'boot wait', 'not idle'): None,
+
+ ('fail', 'closed', 'boot exceeded', 'idle exceeded'): "START_SHUTDOWN",
+ ('fail', 'closed', 'boot exceeded', 'idle wait'): "START_SHUTDOWN",
+ ('fail', 'closed', 'boot exceeded', 'not idle'): "START_SHUTDOWN",
+ ('fail', 'closed', 'boot wait', 'idle exceeded'): "START_SHUTDOWN",
+ ('fail', 'closed', 'boot wait', 'idle wait'): "START_SHUTDOWN",
+ ('fail', 'closed', 'boot wait', 'not idle'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot exceeded', 'idle exceeded'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot exceeded', 'idle wait'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot exceeded', 'not idle'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot wait', 'idle exceeded'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot wait', 'idle wait'): "START_SHUTDOWN",
+ ('fail', 'open', 'boot wait', 'not idle'): "START_SHUTDOWN"}
"unpaired": 0,
"busy": 0,
"idle": 0,
+ "fail": 0,
"down": 0,
"shutdown": 0
}
counts["unpaired"],
counts["idle"],
busy_count,
- counts["down"],
+ counts["down"]+counts["fail"],
counts["shutdown"])
if over_max >= 0:
# grace period without a ping, so shut it down so we can boot a new
# node in its place.
self._begin_node_shutdown(node_actor, cancellable=False)
- elif node_actor.in_state('down').get():
+ elif node_actor.in_state('down', 'fail').get():
# Node is down and unlikely to come back.
self._begin_node_shutdown(node_actor, cancellable=False)
except pykka.ActorDeadError as e:
'mix', 'mix*',
'drng', 'drng*'):
nodestates[nodename] = 'busy'
- elif state == 'idle':
- nodestates[nodename] = 'idle'
+ elif state in ('idle', 'fail'):
+ nodestates[nodename] = state
else:
nodestates[nodename] = 'down'
except ValueError:
SETUP_DIR = os.path.dirname(__file__) or "."
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-node-manager',
- version='0.1',
+ version=version,
description='Arvados compute node manager',
long_description=open(README).read(),
author='Arvados',
self.assertEquals((True, "node state is ('idle', 'open', 'boot wait', 'idle exceeded')"),
self.node_actor.shutdown_eligible().get(self.TIMEOUT))
+ def test_shutdown_when_node_state_fail(self):
+ self.make_actor(5, testutil.arvados_node_mock(
+ 5, crunch_worker_state='fail'))
+ self.shutdowns._set_state(True, 600)
+ self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'idle exceeded')"),
+ self.node_actor.shutdown_eligible().get(self.TIMEOUT))
+
def test_no_shutdown_when_node_state_stale(self):
self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
self.shutdowns._set_state(True, 600)
+++ /dev/null
-arv-sync-groups
auto_setup_new_users_with_repository: true
default_collection_replication: 1
docker_image_formats: ["v2"]
+ keep_web_service_url: http://$localip:${services[keep-web]}/
EOF
(cd config && /usr/local/lib/arvbox/application_yml_override.py)
else
flock /var/lib/gems/gems.lock gem install --no-document bundler
fi
+ ln -sf /var/lib/gems/bin/bundle /usr/local/bin
fi
if ! flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
cd /usr/src/composer
-npm install yarn
+npm -d install yarn
PATH=$PATH:/usr/src/composer/node_modules/.bin
yarn install
if test "$1" != "--only-deps" ; then
- echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/arvados-configuration.yml
+ echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
exec ng serve --host 0.0.0.0 --port 4200 --env=webdev
fi
keep_web_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
arvados_docsite: http://$localip:${services[doc]}/
force_ssl: false
+ composer_url: http://$localip:${services[composer]}
EOF
bundle exec rake assets:precompile
SETUP_DIR = os.path.dirname(__file__) or '.'
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='crunchstat_summary',
- version='0.1',
+ version=version,
description='read crunch log files and summarize resource usage',
author='Arvados',
author_email='info@arvados.org',
--- /dev/null
+sync-groups
"username": true,
}
- flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError)
+ flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
// Set up usage message
flags.Usage = func() {
// Print version information if requested
if *getVersion {
- fmt.Printf("arv-sync-groups %s\n", version)
+ fmt.Printf("%s %s\n", os.Args[0], version)
os.Exit(0)
}
}
defer f.Close()
- log.Printf("arv-sync-groups %s started. Using %q as users id and parent group UUID %q", version, cfg.UserID, cfg.ParentGroupUUID)
+ log.Printf("%s %s started. Using %q as users id and parent group UUID %q", os.Args[0], version, cfg.UserID, cfg.ParentGroupUUID)
// Get the complete user list to minimize API Server requests
allUsers := make(map[string]arvados.User)
projects / arvados.git / commitdiff