*.asc
sdk/java-v2/build.gradle
sdk/java-v2/settings.gradle
-sdk/cwl/tests/wf/feddemo
\ No newline at end of file
+sdk/cwl/tests/wf/feddemo
+go.mod
+go.sum
multi_json (~> 1.0)
websocket-driver (>= 0.2.0)
public_suffix (4.0.3)
- rack (2.0.7)
+ rack (2.2.3)
rack-mini-profiler (1.0.2)
rack (>= 1.2.0)
rack-test (0.6.3)
json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
xpath (2.1.0)
nokogiri (~> 1.3)
uglifier (~> 2.0)
BUNDLED WITH
- 1.11
+ 1.16.6
<meta name="author" content="">
<% if current_user %>
<% content_for :js do %>
- window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.gsub(/\/?$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
+ window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
<% end %>
<% end %>
<% if current_user and $arvados_api_client.discovery[:websocketUrl] %>
<%= yield :head %>
<%= javascript_tag do %>
angular.module('Arvados').value('arvadosApiToken', '<%=Thread.current[:arvados_api_token]%>');
- angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s + '/discovery/v1/apis/arvados/v1/rest' %>');
+ angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/') + 'discovery/v1/apis/arvados/v1/rest' %>');
<%= yield :js %>
<% end %>
<style>
end
test 'view file' do
+ need_selenium "phantomjs does not follow redirects reliably, maybe https://github.com/ariya/phantomjs/issues/10389"
magic = rand(2**512).to_s 36
owner = api_fixture('groups')['anonymously_accessible_project']['uuid']
col = upload_data_and_get_collection(magic, 'admin', "Hello\\040world.txt", owner)
end
test "can download an entire collection with a reader token" do
+ need_selenium "phantomjs does not follow redirects reliably, maybe https://github.com/ariya/phantomjs/issues/10389"
+
token = api_token('active')
data = "foo\nfile\n"
datablock = `echo -n #{data.shellescape} | ARVADOS_API_TOKEN=#{token.shellescape} arv-put --no-progress --raw -`.strip
token = api_fixture('api_client_authorizations')['active_all_collections']['api_token']
url_head = "/collections/download/#{uuid}/#{token}/"
visit url_head
+ assert_text "You can download individual files listed below"
# It seems that Capybara can't inspect tags outside the body, so this is
# a very blunt approach.
assert_no_match(/<\s*meta[^>]+\bnofollow\b/i, page.html,
"wget prohibited from recursing the collection page")
# Look at all the links that wget would recurse through using our
# recommended options, and check that it's exactly the file list.
- hrefs = page.all('a').map do |anchor|
- link = anchor[:href] || ''
- if link.start_with? url_head
- link[url_head.size .. -1]
- elsif link.start_with? '/'
- nil
- else
- link
- end
- end
- assert_equal(['./foo'], hrefs.compact.sort,
- "download page did provide strictly file links")
+ hrefs = []
+ page.html.scan(/href="(.*?)"/) { |m| hrefs << m[0] }
+ assert_equal(['./foo'], hrefs, "download page did provide strictly file links")
click_link "foo"
assert_text "foo\nfile\n"
end
show_object_using('active', 'groups', 'aproject', 'A Project')
click_on "Sharing"
click_on "Share with groups"
- good_uuid = api_fixture("groups")["private"]["uuid"]
+ good_uuid = api_fixture("groups")["future_project_viewing_group"]["uuid"]
assert(page.has_selector?(".selectable[data-object-uuid=\"#{good_uuid}\"]"),
"'share with groups' listing missing owned user group")
bad_uuid = api_fixture("groups")["asubproject"]["uuid"]
end
assert_text ":active/workbenchtest.git"
assert_match /git@git.*:active\/workbenchtest.git/, page.text
- assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}\/active\/workbenchtest.git/, page.text
+ assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}active\/workbenchtest.git/, page.text
end
[
. build/run-library.sh
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
set -x
docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
--- /dev/null
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arvados-cwl-runner --version
--- /dev/null
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
--- /dev/null
+test-package-python27-python-arvados-fuse.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/rh-python36-python-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
PACKAGE_NAME=arvados-sso-server
INSTALL_PATH=/var/www/arvados-sso
CONFIG_PATH=/etc/arvados/sso
-DOC_URL="http://doc.arvados.org/install/install-sso.html#configure"
+DOC_URL="https://doc.arvados.org/v2.0/install/install-sso.html#configure"
RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
RAILSPKG_SUPPORTS_CONFIG_CHECK=0
ARVADOS_BUILDING_ITERATION="1"
fi
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
fi
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
+ set +e
+ /usr/bin/which dpkg-scanpackages >/dev/null
+ if [[ "$?" != "0" ]]; then
+ echo >&2
+ echo >&2 "Error: please install dpkg-dev. E.g. sudo apt-get install dpkg-dev"
+ echo >&2
+ exit 1
+ fi
+ /usr/bin/which apt-ftparchive >/dev/null
+ if [[ "$?" != "0" ]]; then
+ echo >&2
+ echo >&2 "Error: please install apt-utils. E.g. sudo apt-get install apt-utils"
+ echo >&2
+ exit 1
+ fi
+ set -e
(cd $WORKSPACE/packages/$TARGET
dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
keepstore
keep-web
libarvados-perl
+ libpam-arvados
+ libpam-arvados-go
python-arvados-fuse
python-arvados-python-client
python-arvados-cwl-runner"
--upload
If the build and test steps are successful, upload the python
packages to pypi and the gems to rubygems (default: false)
+--ruby <true|false>
+ Build ruby gems (default: true)
+--python <true|false>
+ Build python packages (default: true)
WORKSPACE=path Path to the Arvados source tree to build packages from
TARGET=
UPLOAD=0
+RUBY=1
+PYTHON=1
DEBUG=${ARVADOS_DEBUG:-0}
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,target: \
+ help,debug,ruby:,python:,upload,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --ruby)
+ RUBY="$2"; shift
+ if [ "$RUBY" != "true" ] && [ "$RUBY" != "1" ]; then
+ RUBY=0
+ else
+ RUBY=1
+ fi
+ ;;
+ --python)
+ PYTHON="$2"; shift
+ if [ "$PYTHON" != "true" ] && [ "$PYTHON" != "1" ]; then
+ PYTHON=0
+ else
+ PYTHON=1
+ fi
+ ;;
--upload)
UPLOAD=1
;;
debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
debug_echo "Workspace is $WORKSPACE"
+if [ $RUBY -eq 0 ] && [ $PYTHON -eq 0 ]; then
+ echo "Nothing to do!"
+ exit 0
+fi
+
if [[ -f /etc/profile.d/rvm.sh ]]; then
source /etc/profile.d/rvm.sh
GEM="rvm-exec default gem"
debug_echo "umask is" `umask`
-gem_wrapper arvados "$WORKSPACE/sdk/ruby"
-gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
-gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
-
GEM_BUILD_FAILURES=0
-if [ ${#failures[@]} -ne 0 ]; then
- GEM_BUILD_FAILURES=${#failures[@]}
+if [ $RUBY -eq 1 ]; then
+ debug_echo "Building Ruby gems"
+ gem_wrapper arvados "$WORKSPACE/sdk/ruby"
+ gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
+ gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
+ if [ ${#failures[@]} -ne 0 ]; then
+ GEM_BUILD_FAILURES=${#failures[@]}
+ fi
fi
-python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
-python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
-python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
-python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
-python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
-
PYTHON_BUILD_FAILURES=0
-if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
- PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
+if [ $PYTHON -eq 1 ]; then
+ debug_echo "Building Python packages"
+ python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
+ python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
+ python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
+ python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+ python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
+
+ if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
+ PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
+ fi
fi
-if [[ "$UPLOAD" != 0 ]]; then
+if [ $UPLOAD -ne 0 ]; then
+ echo "Uploading"
- if [[ $DEBUG > 0 ]]; then
+ if [ $DEBUG > 0 ]; then
EXTRA_UPLOAD_FLAGS=" --verbose"
else
EXTRA_UPLOAD_FLAGS=""
fi
- if [[ ! -e "$WORKSPACE/packages" ]]; then
+ if [ ! -e "$WORKSPACE/packages" ]; then
mkdir -p "$WORKSPACE/packages"
fi
- title "Start upload python packages"
- timer_reset
-
- if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
- else
- echo "Skipping python packages upload, there were errors building the packages"
+ if [ $PYTHON -eq 1 ]; then
+ title "Start upload python packages"
+ timer_reset
+
+ if [ $PYTHON_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
+ else
+ echo "Skipping python packages upload, there were errors building the packages"
+ fi
+ checkexit $? "upload python packages"
+ title "End of upload python packages (`timer`)"
fi
- checkexit $? "upload python packages"
- title "End of upload python packages (`timer`)"
- title "Start upload ruby gems"
- timer_reset
-
- if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
- else
- echo "Skipping ruby gem upload, there were errors building the packages"
+ if [ $RUBY -eq 1 ]; then
+ title "Start upload ruby gems"
+ timer_reset
+
+ if [ $GEM_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
+ else
+ echo "Skipping ruby gem upload, there were errors building the packages"
+ fi
+ checkexit $? "upload ruby gems"
+ title "End of upload ruby gems (`timer`)"
fi
- checkexit $? "upload ruby gems"
- title "End of upload ruby gems (`timer`)"
-
fi
exit_cleanly
#
# SPDX-License-Identifier: AGPL-3.0
-. `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
+. `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
+. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados packages
perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
- fpm_build install/lib/=/usr/share libarvados-perl \
+ fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
dir "$(version_from_git)" install/man/=/usr/share/man \
"$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
cd "$SRC_BUILD_DIR"
PKG_VERSION=$(version_from_git)
cd $WORKSPACE/packages/$TARGET
- fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+ fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
rm -rf "$SRC_BUILD_DIR"
fi
"Copy all data from one set of Keep servers to another"
package_go_binary tools/keep-exercise keep-exercise \
"Performance testing tool for Arvados Keep"
+package_go_so lib/pam pam_arvados.so libpam-arvados-go \
+ "Arvados PAM authentication module"
# The Python SDK - Should be built first because it's needed by others
fpm_build_virtualenv "arvados-python-client" "sdk/python"
# The FUSE driver
fpm_build_virtualenv "arvados-fuse" "services/fuse"
+# The FUSE driver - Python3 package
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+
# The node manager
fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
# Output the version being built, or if we're building a
# dev/prerelease, output a version number based on the git log for
# the given $subdir.
- local minorversion="$1"; shift # unused
local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git $1 | cut -d. -f1-4
+ version_from_git | cut -d. -f1-4
}
timestamp_from_git() {
format_last_commit_here "%ct" "$subdir"
}
+calculate_python_sdk_cwl_package_versions() {
+ python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+ cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+ python_sdk_version=$(cd sdk/python && nohash_version_from_git)
+ cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
+
+ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ cwl_runner_version=$python_sdk_version
+ fi
+}
+
handle_python_package () {
# This function assumes the current working directory is the python package directory
if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
cd "$WORKSPACE"
ts="$(timestamp_from_git "$dir")"
if [[ "$ts" -gt "$timestamp" ]]; then
- version=$(version_from_git "" "$dir")
+ version=$(version_from_git "$dir")
timestamp="$ts"
fi
done
__returnvar="$version"
}
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt]
package_go_binary() {
local src_path="$1"; shift
local prog="$1"; shift
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+}
+
+# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
+package_go_so() {
+ local src_path="$1"; shift
+ local sofile="$1"; shift
+ local pkg="$1"; shift
+ local description="$1"; shift
+
+ debug_echo "package_go_so $src_path as $pkg"
+
+ calculate_go_package_version go_package_version $src_path
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence $pkg $go_package_version go || return 1
+ cd $WORKSPACE/$src_path
+ go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
+ cd $WORKSPACE/packages/$TARGET
+ local -a fpmargs=(
+ "--url=https://arvados.org"
+ "--license=Apache License, Version 2.0"
+ "--description=${description}"
+ "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
+ )
+ if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
+ fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/pam-configs/arvados-go")
+ fi
+ if [[ -e "$WORKSPACE/$src_path/README" ]]; then
+ fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
+ fi
+ fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
}
default_iteration() {
for exclude in ${exclude_list[@]}; do
switches+=(-x "$exclude_root/$exclude")
done
- fpm_build "${pos_args[@]}" "${switches[@]}" \
+ fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
-x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
rm -rf "$scripts_dir"
fi
# arvados-python-client sdist should always be built, to be available
- # for other dependant packages.
+ # for other dependent packages.
if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
return 0
fi
cd build/usr/share/$python/dist/$PYTHON_PKG/
# Replace the shebang lines in all python scripts, and handle the activate
- # scripts too This is a functional replacement of the 237 line
+ # scripts too. This is a functional replacement of the 237 line
# virtualenv_tools.py script that doesn't work in python3 without serious
# patching, minus the parts we don't need (modifying pyc files, etc).
for binfile in `ls bin/`; do
# 12271 - As FPM-generated packages don't include scripts by default, the
# packages cleanup on upgrade depends on files being listed on the %files
# section in the generated SPEC files. To remove DIRECTORIES, they need to
- # be listed in that sectiontoo, so we need to add this parameter to properly
+ # be listed in that section too, so we need to add this parameter to properly
# remove lingering dirs. But this only works for python2: if used on
# python33, it includes dirs like /opt/rh/python33 that belong to
# other packages.
COMMAND_ARR+=('--rpm-auto-add-directories')
fi
- if [[ "$PKG" == "arvados-python-client" ]]; then
+ if [[ "$PKG" == "arvados-python-client" ]] || [[ "$PKG" == "arvados-fuse" ]]; then
if [[ "$python" == "python2.7" ]]; then
COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
else
COMMAND_ARR+=('-n' "$PYTHON_PKG")
COMMAND_ARR+=('-C' "build")
- if [[ -e "$WORKSPACE/$PKG_DIR/$PKG.service" ]]; then
+ systemd_unit="$WORKSPACE/$PKG_DIR/$PKG.service"
+ if [[ -e "${systemd_unit}" ]]; then
COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
fi
COMMAND_ARR+=('--depends' "$i")
done
+ # make sure the systemd service file ends up in the right place
+ # used by arvados-docker-cleaner and arvados-node-manager
+ if [[ -e "${systemd_unit}" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
+ fi
+
COMMAND_ARR+=("${fpm_args[@]}")
# Make sure to install all our package binaries in /usr/bin.
# Build packages for everything
fpm_build () {
+ # Source dir where fpm-info.sh (if any) will be found.
+ SRC_DIR=$1
+ shift
# The package source. Depending on the source type, this can be a
# path, or the name of the package in an upstream repository (e.g.,
# pip).
declare -a build_depends=()
declare -a fpm_depends=()
declare -a fpm_exclude=()
- declare -a fpm_dirs=(
- # source dir part of 'dir' package ("/source=/dest" => "/source"):
- "${PACKAGE%%=/*}")
- for pkgdir in "${fpm_dirs[@]}"; do
- fpminfo="$pkgdir/fpm-info.sh"
- if [[ -e "$fpminfo" ]]; then
- debug_echo "Loading fpm overrides from $fpminfo"
- source "$fpminfo"
- break
- fi
- done
+ if [[ ! -d "$SRC_DIR" ]]; then
+ echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
+ exit 1
+ fi
+ fpminfo="${SRC_DIR}/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ debug_echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ fi
for pkg in "${build_depends[@]}"; do
if [[ $TARGET =~ debian|ubuntu ]]; then
pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
echo "== Interactive commands:"
echo "TARGET (short for 'test DIR')"
echo "test TARGET"
+ echo "10 test TARGET (run test 10 times)"
echo "test TARGET:py3 (test with python3)"
echo "test TARGET -check.vv (pass arguments to test)"
echo "install TARGET"
while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
history -s "$nextcmd"
history -w
+ count=1
+ if [[ "${nextcmd}" =~ ^[0-9] ]]; then
+ read count nextcmd <<<"${nextcmd}"
+ fi
read verb target opts <<<"${nextcmd}"
target="${target%/}"
target="${target/\/:/:}"
${verb}_${target}
;;
*)
- argstarget=${target%:py3}
+ argstarget=${target%:py3}
testargs["$argstarget"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
- do_$verb $tt
+ while [ $count -gt 0 ]; do
+ do_$verb $tt
+ let "count=count-1"
+ done
;;
esac
;;
"git.arvados.org/arvados.git/lib/cli"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/deduplicationreport"
"git.arvados.org/arvados.git/lib/mount"
)
"virtual_machine": cli.APICall,
"workflow": cli.APICall,
- "mount": mount.Command,
+ "mount": mount.Command,
+ "deduplication-report": deduplicationreport.Command,
})
)
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
"git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/lib/recovercollection"
"git.arvados.org/arvados.git/services/ws"
)
"-version": cmd.Version,
"--version": cmd.Version,
- "boot": boot.Command,
- "cloudtest": cloudtest.Command,
- "config-check": config.CheckCommand,
- "config-defaults": config.DumpDefaultsCommand,
- "config-dump": config.DumpCommand,
- "controller": controller.Command,
- "crunch-run": crunchrun.Command,
- "dispatch-cloud": dispatchcloud.Command,
- "install": install.Command,
- "ws": ws.Command,
+ "boot": boot.Command,
+ "cloudtest": cloudtest.Command,
+ "config-check": config.CheckCommand,
+ "config-defaults": config.DumpDefaultsCommand,
+ "config-dump": config.DumpCommand,
+ "controller": controller.Command,
+ "crunch-run": crunchrun.Command,
+ "dispatch-cloud": dispatchcloud.Command,
+ "install": install.Command,
+ "recover-collection": recovercollection.Command,
+ "ws": ws.Command,
})
)
- admin/index.html.textile.liquid
- Users and Groups:
- admin/user-management.html.textile.liquid
- - admin/reassign-ownership.html.textile.liquid
- admin/user-management-cli.html.textile.liquid
+ - admin/reassign-ownership.html.textile.liquid
+ - admin/link-accounts.html.textile.liquid
- admin/group-management.html.textile.liquid
- admin/federation.html.textile.liquid
- admin/merge-remote-account.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
- admin/workbench2-vocabulary.html.textile.liquid
- admin/storage-classes.html.textile.liquid
+ - admin/recovering-deleted-collections.html.textile.liquid
- Cloud:
- admin/spot-instances.html.textile.liquid
- admin/cloudtest.html.textile.liquid
- install/arvbox.html.textile.liquid
- Arvados on Kubernetes:
- install/arvados-on-kubernetes.html.textile.liquid
+ - install/arvados-on-kubernetes-minikube.html.textile.liquid
+ - install/arvados-on-kubernetes-GKE.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- install/packages.html.textile.liquid
- install/install-keep-balance.html.textile.liquid
- User interface:
- install/setup-login.html.textile.liquid
- - install/install-sso.html.textile.liquid
- install/install-workbench-app.html.textile.liquid
- install/install-workbench2-app.html.textile.liquid
- install/install-composer.html.textile.liquid
- install/install-ws.html.textile.liquid
- install/install-arv-git-httpd.html.textile.liquid
- install/install-shell-server.html.textile.liquid
+ - install/install-webshell.html.textile.liquid
- Containers API:
- install/crunch2-slurm/install-compute-node.html.textile.liquid
- install/install-jobs-image.html.textile.liquid
- install/install-postgresql.html.textile.liquid
- install/ruby.html.textile.liquid
- install/nginx.html.textile.liquid
- - install/google-auth.html.textile.liquid
- install/install-docker.html.textile.liquid
A federation of clusters can be configured to use a separate user database per cluster, or delegate a central cluster to manage the database.
-If clusters belong to separate organizations, each cluster will have its own user database for the members of that organization. Through federation, a user from one organization can be granted access to the cluster of another organization. The admin of the seond cluster controls access on a individual basis by choosing to activate or deactivate accounts from other organizations (with the default policy the value of @ActivateUsers@).
+If clusters belong to separate organizations, each cluster will have its own user database for the members of that organization. Through federation, a user from one organization can be granted access to the cluster of another organization. The admin of the second cluster controls access on a individual basis by choosing to activate or deactivate accounts from other organizations (with the default policy the value of @ActivateUsers@).
On the other hand, if all clusters belong to the same organization, and users in that organization should have access to all the clusters, user management can be simplified by setting the @LoginCluster@ which manages the user database used by all other clusters in the federation. To do this, choose one cluster in the federation which will be the 'login cluster'. Set the the @Login.LoginCluster@ configuration value on all clusters in the federation to the cluster id of the login cluster. After setting @LoginCluster@, restart arvados-api-server and arvados-controller.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: "Link user accounts"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+If a user needs to log in to Arvados with a upstream account or provider, they may end up with two Arvados user accounts. If the user still has the ability to log in with the old account, they can use the "self-serve account linking":{{site.baseurl}}/user/topics/link-accounts.html feature of workbench. However, if the user does not have the ability to log in with both upstream accounts, the admin can also link the accounts using the command line.
+
+h3. Step 1: Determine user uuids
+
+User uuids can be determined by browsing workbench or using @arv user list@ at the command line.
+
+Account linking works by recording in the database that a log in to the "old" account should redirected and treated as a login to the "new" account.
+
+The "old" account is the Arvados account that will be redirected.
+
+The "new" account is the user that the "old" account is redirected to. As part of account linking any Arvados records owned by the "old" account is also transferred to the "new" account.
+
+Counter-intuitively, if you do not want the account uuid of the user to change, the "new" account should be the pre-existing account, and the "old" account should be the redundant second account that was more recently created. This means "old" and "new" are opposite from their expected chronological meaning. In this case, the use of "old" and "new" reflect the direction of transfer of ownership -- the login was associated with the "old" user account, but will be associated with the "new" user account.
+
+In the example below, @zzzzz-tpzed-3kz0nwtjehhl0u4@ is the "old" account (the pre-existing account we want to keep) and @zzzzz-tpzed-fr97h9t4m5jffxs@ is the "new" account (the redundant account we want to merge into the existing account).
+
+h3. Step 2: Create a project
+
+Create a project owned by the "new" account that will hold any data owned by the "old" account.
+
+<pre>
+$ arv --format=uuid group create --group '{"group_class": "project", "name": "Data from old user", "owner_uuid": "zzzzz-tpzed-fr97h9t4m5jffxs"}'
+zzzzz-j7d0g-mczqiguhil13083
+</pre>
+
+h3. Step 3: Merge "old" user to "new" user
+
+The @user merge@ method redirects login and reassigns data from the "old" account to the "new" account.
+
+<pre>
+$ arv user merge --redirect-to-new-user \
+ --old-user-uuid=zzzzz-tpzed-3kz0nwtjehhl0u4 \
+ --new-user-uuid=zzzzz-tpzed-fr97h9t4m5jffxs \
+ --new-owner-uuid=zzzzz-j7d0g-mczqiguhil13083 \
+</pre>
+
+Note that authorization credentials (API tokens, ssh keys) are also transferred to the "new" account, so credentials used to access the "old" account work with the "new" account.
|keepstore|✓|
|keep-balance|✓|
|keep-web|✓|
-|sso-provider||
|workbench1||
|workbench2||
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This page describes how to enable users to use more than one upstream identity provider to log into the same Arvados account. This can be used to migrate account providers, for example, from LDAP to Google. In order to do this, users must be able to log into both the "old" and "new" providers.
+When a user logs in to Arvados, their email address (as returned by the authentication provider) is used as the primary key for their Arvados account.
-h2. Configure multiple or alternate provider in SSO
+If you reconfigure Arvados to use a different authentication provider after some users have created accounts, you should either ensure the new provider returns the same email addresses as the old one, or update your Arvados users' @email@ attributes to match the email addresses returned by the new provider.
-In @application.yml@ for the SSO server, you can enable both @google_oauth2@ and @ldap@ providers:
-
-<pre>
-production:
- google_oauth2_client_id: abcd
- google_oauth2_client_secret: abcd
-
- use_ldap:
- title: Example LDAP
- host: ldap.example.com
- port: 636
- method: ssl
- base: "ou=Users, dc=example, dc=com"
- uid: uid
- username: uid
-</pre>
-
-Restart the SSO server after changing the configuration.
-
-h2. Matching on email address
-
-If the new account provider supplies an email address (primary or alternate) that matches an existing user account, the user will be logged into that account. No further migration is necessary, and the old provider can be removed from the SSO configuration.
-
-h2. Link accounts
-
-If the new provider cannot provide matching email addresses, users will have to migrate manually by "linking accounts":{{site.baseurl}}/user/topics/link-accounts.html
-
-After linking accounts, users can use the new provider to access their existing Arvados account.
-
-Once all users have migrated, the old account provider can be removed from the SSO configuration.
+Otherwise, next time users log in, they will be given new accounts instead of logging in to their existing accounts.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Recovering deleted collections
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+In some cases, it is possible to recover files that have been lost by modifying or deleting a collection.
+
+Possibility of recovery depends on many factors, including:
+* Whether the collection manifest is still available, e.g., in an audit log entry
+* Whether the data blocks are also referenced by other collections
+* Whether the data blocks have been unreferenced long enough to be marked for deletion/trash by keep-balance
+* Blob signature TTL, trash lifetime, trash check interval, and other config settings
+
+To attempt recovery of a previous version of a deleted/modified collection, use the @arvados-server recover-collection@ command. It should be run on one of your server nodes where the @arvados-server@ package is installed and the @/etc/arvados/config.yml@ file is up to date.
+
+Specify the collection you want to recover by passing either the UUID of an audit log entry, or a file containing the manifest.
+
+If recovery is successful, the @recover-collection@ program saves the recovered data a new collection belonging to the system user, and prints the new collection's UUID on stdout.
+
+<pre>
+# arvados-server recover-collection 9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.557761245Z] loaded log entry logged_event_time="2020-06-05 16:48:01.438791 +0000 UTC" logged_event_type=update old_collection_uuid=9tee4-4zz18-1ex26g95epmgw5w src=9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.642145127Z] recovery succeeded UUID=9tee4-4zz18-5trfp4k4xxg97f1 src=9tee4-57u5n-nb5awmk1pahac2t
+9tee4-4zz18-5trfp4k4xxg97f1
+INFO[2020-06-05T19:52:29.644699436Z] exiting
+</pre>
+
+In this example, the original data has been restored and saved in a new collection with UUID @9tee4-4zz18-5trfp4k4xxg97f1@.
+
+For more options, run @arvados-server recover-collection -help@.
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-02-07)
+h2(#master). development master (as of 2020-06-17)
"Upgrading from 2.0.0":#v2_0_0
-None in current development master.
+h3. Removing sso-provider
+
+The SSO (single sign-on) component is deprecated and will not be supported in future releases. Existing configurations will continue to work in this release, but you should switch to one of the built-in authentication mechanisms as soon as possible. See "setting up web based login":{{site.baseurl}}/install/setup-login.html for details.
+
+After migrating your configuration, uninstall the @arvados-sso-provider@ package.
+
+h3. S3 signatures
+
+Keepstore now uses "V4 signatures":https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html by default for S3 requests. If you are using Amazon S3, no action is needed; all regions support V4 signatures. If you are using a different S3-compatible service that does not support V4 signatures, add @V2Signature: true@ to your volume driver parameters to preserve the old behavior. See "configuring S3 object storage":{{site.baseurl}}/install/configure-s3-object-storage.html.
+
+h3. New permission system constraints
+
+Some constraints on the permission system have been added, in particular @role@ and @project@ group types now have distinct behavior. These constraints were already de-facto imposed by the Workbench UI, so on most installations the only effect of this migration will be to reassign @role@ groups to the system user and create a @can_manage@ permission link for the previous owner.
+
+# The @group_class@ field must be either @role@ or @project@. Invalid group_class are migrated to @role@.
+# A @role@ cannot own things. Anything owned by a role is migrated to a @can_manage@ link and reassigned to the system user.
+# Only @role@ and @user@ can have outgoing permission links. Permission links originating from projects are deleted by the migration.
+# A @role@ is always owned by the system_user. When a group is created, it creates a @can_manage@ link for the object that would have been assigned to @owner_uuid@. Migration adds @can_manage@ links and reassigns roles to the system user. This also has the effect of requiring that all @role@ groups have unique names on the system. If there is a name collision during migration, roles will be renamed to ensure they are unique.
+# A permission link can have the permission level (@name@) updated but not @head_uuid@, @tail_uuid@ or @link_class@.
+
+The @arvados-sync-groups@ tool has been updated to reflect these constraints, so it is important to use the version of @arvados-sync-groups@ that matches the API server version.
+
+Before upgrading, use the following commands to find out which groups and permissions in your database will be automatically modified or deleted during the upgrade.
+
+To determine which groups have invalid @group_class@ (these will be migrated to @role@ groups):
+
+<pre>
+arv group list --filters '[["group_class", "not in", ["project", "role"]]]'
+</pre>
+
+To list all @role@ groups, which will be reassigned to the system user (unless @owner_uuid@ is already the system user):
+
+<pre>
+arv group list --filters '[["group_class", "=", "role"]]'
+</pre>
+
+To list which @project@ groups have outgoing permission links (such links are now invalid and will be deleted by the migration):
+
+<pre>
+for uuid in $(arv link list --filters '[["link_class", "=", "permission"], ["tail_uuid", "like", "%-j7d0g-%"]]' |
+ jq -r .items[].tail_uuid | sort | uniq) ; do
+ arv group list --filters '[["group_class", "=", "project"], ["uuid", "=", "'$uuid'"]]' | jq .items
+done
+</pre>
h2(#v2_0_0). v2.0.0 (2020-02-07)
If a provider identifier is given, the API server searches for a matching user record.
-If a provider identifier is not given, no match is found, it next searches by primary email and then alternate email address. This enables "provider migration":migrating-providers.html and a "pre-activated accounts.":#pre-activated
+If a provider identifier is not given, no match is found, it next searches by primary email and then alternate email address. This enables "provider migration":migrating-providers.html and "pre-activated accounts.":#pre-activated
If no user account is found, a new user account is created with the information from the identity provider.
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
{background:#ccffcc}.|uuid|string|The UUID of the User in question.|path||
+
+h3. merge
+
+Transfer ownership of data from the "old" user account to the "new" user account. When @redirect_to_new_user@ is @true@ this also causes logins to the "old" account to be redirected to the "new" account. The "old" user account that was redirected becomes invisible in user listings.
+
+See "Merge user accounts":{{site.baseurl}}/admin/link-accounts.html , "Reassign user data ownership":{{site.baseurl}}/admin/reassign-ownership.html and "Linking alternate login accounts":{{site.baseurl}}/user/topics/link-accounts.html for examples of how this method is used.
+
+Must supply either @new_user_token@ (the currently authorized user will be the "old" user), or both @new_user_uuid@ and @old_user_uuid@ (the currently authorized user must be an admin).
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+|new_user_token|string|A valid token for the "new" user|query||
+|new_user_uuid|uuid|The uuid of the "new" account|query||
+|old_user_uuid|uuid|The uuid of the "old" account|query||
+|new_owner_uuid|uuid|The uuid of a project to which objects owned by the "old" user will be reassigned.|query||
+|redirect_to_new_user|boolean|If true, also redirect login and reassign authorization credentials from "old" user to the "new" user|query||
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-* There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
-** *none* is the default state when there are no other permission grants.
-*** the object is not included in any list query response.
-*** direct queries of the object by uuid return 404 Not Found.
-*** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
-** *can_read* grants read-only access to the record. Attempting to update or delete the record returns an error. *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
-** *can_write* permits changes to the record (but not permission links). *can_write* permits the user to delete the object. *can_write* also implies *can_read*.
-** *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@. *can_manage* also implies *can_write* and *can_read*.
+There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
+
+* *none* is the default state when there are no other permission grants.
+** the object is not included in any list query response.
+** direct queries of the object by uuid return 404 Not Found.
+** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
+* *can_read* grants read-only access to the record. Attempting to update or delete the record returns an error.
+** *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
+* *can_write* permits changes to the record, including changing ownership and deleting the object.
+** *can_write* cannot read, create, update or delete permission links associated with the object.
+** *can_write* also implies *can_read*.
+* *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@.
+** *can_manage* also implies *can_write* and *can_read*.
h2. Ownership
-* All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group".
-* The User or Group specified by @owner_uuid@ has *can_manage* permission on the object.
-** This permission is one way: A User or Group's @owner_uuid@ being equal to @X@ does not imply any permission for that User/Group to read, write, or manage an object whose @uuid@ is equal to @X@.
-* Applications should represent each object as belonging to, or being "inside", the Group/User referenced by its @owner_uuid@.
-** A "project" is a subtype of Group that is treated as a "Project" in Workbench, and as a directory by @arv-mount@.
-** A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
-* To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
+All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group". For Group, the @group_class@ must be a "project".
+
+The User or Group specified by @owner_uuid@ has *can_manage* permission on the object. This permission is one way: an object that is owned does not get any special permissions on the User or Group that owns it.
+
+To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
h2(#links). Permission links
-A link object with
+A permission link is a link object with:
* @owner_uuid@ of the system user.
* @link_class@ "permission"
* @name@ one of *can_read*, *can_write* or *can_manage*
* @head_uuid@ of some Arvados object
-* @tail_uuid@ of a User or Group
+* @tail_uuid@ of a User or Group. For Group, the @group_class@ must be a "role".
-grants the @name@ permission for @tail_uuid@ accessing @head_uuid@
+This grants the permission in @name@ for @tail_uuid@ accessing @head_uuid@.
-* If a User has *can_manage* permission on some object, this grants permission to read, create, update and delete permission links where the @head_uuid@ is the object under management.
+If a User has *can_manage* permission on some object, the user has the ability to read, create, update and delete permission links with @head_uuid@ of the managed object. In other words, the user has the ability to modify the permission grants on the object.
h3. Transitive permissions
-Permissions can be obtained indirectly through Groups.
-* If a User X *can_read* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
+Permissions can be obtained indirectly through nested ownership (*can_manage*) or by following multiple permission links.
+
+* If a User X owns project A, and project A owns project B, then User X *can_manage* project B.
+* If a User X *can_read* role A, and role A *can_read* Object B, then User X *can_read* Object B.
* Permissions are narrowed to the least powerful permission on the path.
-** If User X *can_write* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
-** If User X *can_read* Group A, and Group A *can_write* Object B, then User X *can_read* Object B.
+** If User X *can_write* role A, and role A *can_read* Object B, then User X *can_read* Object B.
+** If User X *can_read* role A, and role A *can_write* Object B, then User X *can_read* Object B.
+
+h2. Projects and Roles
+
+A "project" is a subtype of Group that is displayed as a "Project" in Workbench, and as a directory by @arv-mount@.
+* A project can own things (appear in @owner_uuid@)
+* A project can be owned by a user or another project.
+* The name of a project is unique only among projects with the same owner_uuid.
+* Projects can be targets (@head_uuid@) of permission links, but not origins (@tail_uuid@). Putting a project in a @tail_uuid@ field is an error.
+
+A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
+* A role cannot own things (cannot appear in @owner_uuid@). Putting a role in an @owner_uuid@ field is an error.
+* All roles are owned by the system user.
+* The name of a role is unique across a single Arvados cluster.
+* Roles can be both targets (@head_uuid@) and origins (@tail_uuid@) of permission links.
+
+h3. Access through Roles
-h2. Group Membership
+A "role" consists of a set of users or other roles that have that role, and a set of permissions (primarily read/write/manage access to projects) the role grants.
-Group membership is determined by whether the group has *can_read* permission on an object. If a group G *can_read* an object A, then we say A is a member of G.
+If there is a permission link stating that user A *can_write* role R, then we say A has role R. This means user A has up to *can_write* access to everything the role has access to.
-For some kinds of groups, like roles, it is natural for users who are members of a group to also have *can_manage* permission on the group, i.e., G *can_read* A and A *can_manage* G ("A can do anything G can do"). However, this is not necessary: A can be a member of a group while being unable to even read it.
+Because permissions are one-way, the links A *can_write* R and B *can_write* R does not imply that user A and B will be able to see each other. For users in a role to see each other, read permission should be added going in the opposite direction: R *can_read* A and R *can_read* B.
+
+If a user needs to be able to manipulate permissions of objects that are accessed through the role (for example, to share project P with a user outside the role), then role R must have *can_manage* permission on project P (R *can_manage* P) and the user must be granted *can_manage* permission on R (A *can_manage* R).
h2. Special cases
-* Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read). To retain the integrity of the log, the log table should deny all update or delete operations.
-* Permission links where @tail_uuid@ is a User permit @can_read@ on the link by that user. (User can discover her own permission grants.)
-* *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks)
-* If User or Group X *can_FOO* Group A, and Group A *can_manage* User B, then X *can_FOO* _everything that User B can_FOO_.
+Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read). To retain the integrity of the log, the log table denies all update or delete operations.
+
+Permission links where @tail_uuid@ is a User allow *can_read* on the link record by that user (User can discover her own permission grants.)
+
+At least *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks).
+
+A user can only read a container record if the user has read permission to a container_request with that container_uuid.
+
+*can_read* and *can_write* access on a user grants access to the user record, but not anything owned by the user.
+*can_manage* access to a user grants can_manage access to the user, _and everything owned by that user_ .
+If a user A *can_read* role R, and role R *can_manage* user B, then user A *can_read* user B _and everything owned by that user_ .
h2(#system). System user and group
h2. Anoymous user and group
-An Arvados site may be configured to allow users to browse resources without requiring a login. In this case, permissions for non-logged-in users are associated with the "anonymous" user. To make objects visible to the public, they can be shared with the "anonymous" group. The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@. The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
+An Arvados site may be configured to allow users to browse resources without requiring a login. In this case, permissions for non-logged-in users are associated with the "anonymous" user. To make objects visible to anyone (both logged-in and non-logged-in users), they can be shared with the "anonymous" role. Note that objects shared with the "anonymous" user will only be visible to non-logged-in users!
+
+The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@. The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
h2. Example
|keep-block-check|Given a list of keep block locators, check that each block exists on one of the configured keepstore servers and verify the block hash.|
|keep-exercise|Benchmarking tool to test throughput and reliability of keepstores under various usage patterns.|
|keep-rsync|Get lists of blocks from two clusters, copy blocks which exist on source cluster but are missing from destination cluster.|
-|sync-groups|Take a CSV file listing (group, username) pairs and synchronize membership in Arvados groups.|
+|sync-groups|Take a CSV file listing with (group, user, permission) records and synchronize membership in Arvados groups.|
+AutoReloadConfig: true
Clusters:
zzzzz:
ManagementToken: e687950a23c3a9bceec28c6223a06c79
<a name="Support"></a>
<p><strong>Support and Community</strong></p>
- <p>The <a href="https://gitter.im/arvados/community">arvados community channel</a> at gitter.im is available for live discussion and community support. There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>.
+ <p>Interact with the Arvados community on the <a href="https://forum.arvados.org">Arvados Forum</a>
+ and the <a href="https://gitter.im/arvados/community">arvados/community</a> channel at gitter.im.
</p>
<p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info@curii.com">info@curii.com</a> for more information.</p>
---
layout: default
navsection: installguide
-title: Arvados on Kubernetes - Google Kubernetes Engine
+title: Arvados on GKE
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
+This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+h2. Prerequisites
h3. Install tooling
* Follow the instructions at "https://docs.helm.sh/using_helm/#installing-helm":https://docs.helm.sh/using_helm/#installing-helm
-h3. Boot the GKE cluster
+h3. Create the GKE cluster
This can be done via the "cloud console":https://console.cloud.google.com/kubernetes/ or via the command line:
<pre>
-$ gcloud container clusters create <CLUSTERNAME> --zone us-central1-a --machine-type n1-standard-2 --cluster-version 1.10
+$ gcloud container clusters create <CLUSTERNAME> --zone us-central1-a --machine-type n1-standard-2 --cluster-version 1.15
</pre>
It takes a few minutes for the cluster to be initialized.
$ kubectl get nodes
</pre>
-Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
+Test @helm@ by running
+
+<pre>
+$ helm ls
+</pre>
+
+There should be no errors. The command will return nothing.
+
+h2(#git). Clone the repository
+
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
+
+<pre>
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
+</pre>
+
+h2(#Start). Start the Arvados cluster
+
+Next, determine the IP address that the Arvados cluster will use to expose its API, Workbench, etc. If you want this Arvados cluster to be reachable from places other than the local machine, the IP address will need to be routable as appropriate.
+
+<pre>
+$ ./cert-gen.sh <IP ADDRESS>
+</pre>
+
+The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
+
+<pre>
+ adminUserEmail
+ adminUserPassword
+ superUserSecret
+ anonymousUserSecret
+</pre>
+
+Now start the Arvados cluster:
+
+<pre>
+$ helm install arvados . --set externalIP=<IP ADDRESS>
+</pre>
+
+At this point, you can use kubectl to see the Arvados cluster boot:
+
+<pre>
+$ kubectl get pods
+$ kubectl get svc
+</pre>
+
+After a few minutes, there shouldn't be any services listed with a 'Pending' external IP address. At that point you can access Arvados Workbench at the IP address specified
+
+* https://<IP ADDRESS>
+
+with the username and password specified in the @values.yaml@ file.
+
+Alternatively, use the Arvados cli tools or SDKs. First set the environment variables:
+
+<pre>
+$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
+$ export ARVADOS_API_HOST=<STATIC IP>:444
+$ export ARVADOS_API_HOST_INSECURE=true
+</pre>
+
+Test access with:
+
+<pre>
+$ arv user current
+</pre>
+
+h2(#reload). Reload
+
+If you make changes to the Helm chart (e.g. to @values.yaml@), you can reload Arvados with
+
+<pre>
+$ helm upgrade arvados .
+</pre>
+
+h2. Shut down
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+<pre>
+$ helm del arvados
+</pre>
+
+<pre>
+$ gcloud container clusters delete <CLUSTERNAME> --zone us-central1-a
+</pre>
---
layout: default
navsection: installguide
-title: Arvados on Kubernetes - Minikube
+title: Arvados on Minikube
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
+This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+h2. Prerequisites
h3. Install tooling
$ kubectl get nodes
</pre>
-Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
+Test @helm@ by running
+
+<pre>
+$ helm ls
+</pre>
+
+There should be no errors. The command will return nothing.
+
+h2(#git). Clone the repository
+
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
+
+<pre>
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
+</pre>
+
+h2(#Start). Start the Arvados cluster
+
+All Arvados services will be accessible on Minikube's IP address. This will be a local IP address, you can see what it is by running
+
+<pre>
+$ minikube ip
+192.168.39.15
+</pre>
+
+Generate self-signed SSL certificates for the Arvados services:
+
+<pre>
+$ ./cert-gen.sh `minikube ip`
+</pre>
+
+The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
+
+<pre>
+ adminUserEmail
+ adminUserPassword
+ superUserSecret
+ anonymousUserSecret
+</pre>
+
+Now start the Arvados cluster:
+
+<pre>
+$ helm install arvados . --set externalIP=`minikube ip`
+</pre>
+
+And update the Kubernetes services to have the Minikube IP as their 'external' IP:
+
+<pre>
+$ ./minikube-external-ip.sh
+</pre>
+
+At this point, you can use kubectl to see the Arvados cluster boot:
+
+<pre>
+$ kubectl get pods
+$ kubectl get svc
+</pre>
+
+After a few minutes, you can access Arvados Workbench at the Minikube IP address:
+
+* https://<MINIKUBE IP>
+
+with the username and password specified in the @values.yaml@ file.
+
+Alternatively, use the Arvados cli tools or SDKs. First set the environment variables:
+
+<pre>
+$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
+$ export ARVADOS_API_HOST=<MINIKUBE IP>:444
+$ export ARVADOS_API_HOST_INSECURE=true
+</pre>
+
+Test access with:
+
+<pre>
+$ arv user current
+</pre>
+
+h2(#reload). Reload
+
+If you make changes to the Helm chart (e.g. to @values.yaml@), you can reload Arvados with
+
+<pre>
+$ helm upgrade arvados .
+</pre>
+
+h2. Shut down
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+<pre>
+$ helm del arvados
+</pre>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados on Kubernetes is implemented as a Helm Chart.
+Arvados on Kubernetes is implemented as a @Helm 3@ chart.
{% include 'notebox_begin_warning' %}
-This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
{% include 'notebox_end' %}
h2(#overview). Overview
-This Helm Chart provides a basic, small Arvados cluster.
+This Helm chart provides a basic, small Arvados cluster.
Current limitations, to be addressed in the future:
-* An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down.
-* No dynamic scaling of compute nodes (but you can adjust @values.yaml@ and "reload the Helm Chart":#reload
+* An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down.
+* No dynamic scaling of compute nodes (but you can adjust @values.yaml@ and reload the Helm chart)
* All compute nodes are the same size
* Compute nodes have no cpu/memory/disk constraints yet
* No git server
h2. Requirements
-* Kubernetes 1.10+ cluster with at least 3 nodes, 2 or more cores per node
-* @kubectl@ and @helm@ installed locally, and able to connect to your Kubernetes cluster
+* Minikube or Google Kubernetes Engine (Kubernetes 1.10+ with at least 3 nodes, 2+ cores per node)
+* @kubectl@ and @Helm 3@ installed locally, and able to connect to your Kubernetes cluster
-If you do not have a Kubernetes cluster already set up, you can use "Google Kubernetes Engine":/install/arvados-on-kubernetes-GKE.html for multi-node development and testing or "another Kubernetes solution":https://kubernetes.io/docs/setup/pick-right-solution/. Minikube is not supported yet.
+Please refer to "Arvados on Minikube":/install/arvados-on-kubernetes-minikube.html or "Arvados on GKE":/install/arvados-on-kubernetes-GKE.html for detailed installation instructions.
-h2(#helm). Initialize helm on the Kubernetes cluster
-
-If you already have helm running on the Kubernetes cluster, proceed directly to "Start the Arvados cluster":#Start below.
-
-<pre>
-$ helm init
-$ kubectl create serviceaccount --namespace kube-system tiller
-$ kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
-$ kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'
-</pre>
-
-Test @helm@ by running
-
-<pre>
-$ helm ls
-</pre>
-
-There should be no errors. The command will return nothing.
-
-h2(#git). Clone the repository
-
-Clone the repository and nagivate to the @arvados-kubernetes/charts/arvados@ directory:
-
-<pre>
-$ git clone https://github.com/arvados/arvados-kubernetes.git
-$ cd arvados-kubernetes/charts/arvados
-</pre>
-
-h2(#Start). Start the Arvados cluster
-
-Next, determine the IP address that the Arvados cluster will use to expose its API, Workbench, etc. If you want this Arvados cluster to be reachable from places other than the local machine, the IP address will need to be routable as appropriate.
-
-<pre>
-$ ./cert-gen.sh <IP ADDRESS>
-</pre>
-
-The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
-
-<pre>
- adminUserEmail
- adminUserPassword
- superUserSecret
- anonymousUserSecret
-</pre>
-
-Now start the Arvados cluster:
-
-<pre>
-$ helm install --name arvados . --set externalIP=<IP ADDRESS>
-</pre>
-
-At this point, you can use kubectl to see the Arvados cluster boot:
-
-<pre>
-$ kubectl get pods
-$ kubectl get svc
-</pre>
-
-After a few minutes, you can access Arvados Workbench at the IP address specified
-
-* https://<IP ADDRESS>
-
-with the username and password specified in the @values.yaml@ file.
-
-Alternatively, use the Arvados cli tools or SDKs:
-
-Set the environment variables:
-
-<pre>
-$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
-$ export ARVADOS_API_HOST=<STATIC IP>:444
-$ export ARVADOS_API_HOST_INSECURE=true
-</pre>
-
-Test access with:
-
-<pre>
-$ arv user current
-</pre>
-
-h2(#reload). Reload
-
-If you make changes to the Helm Chart (e.g. to @values.yaml@), you can reload Arvados with
-
-<pre>
-$ helm upgrade arvados .
-</pre>
-
-h2. Shut down
-
-{% include 'notebox_begin_warning' %}
-This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
-<pre>
-$ helm del arvados --purge
-</pre>
# If the AccessViaHosts section is empty or omitted, all
# keepstore servers will have read/write access to the
# volume.
- "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {ReadOnly: true}
Driver: <span class="userinput">Azure</span>
DriverParameters:
# If the AccessViaHosts section is empty or omitted, all
# keepstore servers will have read/write access to the
# volume.
- "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {ReadOnly: true}
Driver: <span class="userinput">S3</span>
DriverParameters:
# declaration.
LocationConstraint: false
+ # Use V2 signatures instead of the default V4. Amazon S3
+ # supports V4 signatures in all regions, but this option
+ # might be needed for other S3-compatible services.
+ V2Signature: false
+
# Requested page size for "list bucket contents" requests.
IndexPageSize: 1000
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Setting up Google auth
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-In order to use Google for authentication, you must use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
-
-# Go to the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> and select or create a project; this will take you to the project page.
-# Click on *+ Enable APIs and Services*
-## Search for *People API* and click on *Enable API*.
-# Navigate back to the main "APIs & Services" page
-# On the sidebar, click on *OAuth consent screen*
-## On consent screen settings, enter your identifying details
-## Under *Authorized domains* add @example.com@
-## Click on *Save*.
-# On the sidebar, click on *Credentials*; then click on *Create credentials*→*OAuth Client ID*
-# Under *Application type* select *Web application*.
-# You must set the authorization origins. Edit @auth.example.com@ to the appropriate hostname that you will use to access the SSO service:
-## JavaScript origin should be @https://ClusterID.example.com/@ (using Arvados-controller based login) or @https://auth.example.com/@ (for the SSO server)
-## Redirect URI should be @https://ClusterID.example.com/login@ (using Arvados-controller based login) or @https://auth.example.com/users/auth/google_oauth2/callback@ (for the SSO server)
-# Copy the values of *Client ID* and *Client secret* from the Google Developers Console and add them to the appropriate configuration.
# This configures the public https port that clients will actually connect to,
# the request is reverse proxied to the upstream 'controller'
- listen *:443 ssl;
- server_name <span class="userinput">xxxxx.example.com</span>;
+ listen 443 ssl;
+ server_name <span class="userinput">ClusterID.example.com</span>;
- ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
server 127.0.0.1:<span class="userinput">9001</span>;
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name git.<span class="userinput">ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
- ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Choosing which components to install
-...
-
-Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.) It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture to understand how these components interact.
-
-table(table table-bordered table-condensed).
-|\3=. *Core*|
-|"Postgres database":install-postgresql.html |Stores data for the API server.|Required.|
-|"API server":install-api-server.html |Core Arvados logic for managing users, groups, collections, containers, and enforcing permissions.|Required.|
-|\3=. *Keep (storage)*|
-|"Keepstore":install-keepstore.html |Stores content-addressed blocks in a variety of backends (local filesystem, cloud object storage).|Required.|
-|"Keepproxy":install-keepproxy.html |Gateway service to access keep servers from external networks.|Required to be able to use arv-put, arv-get, or arv-mount outside the private Arvados network.|
-|"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to be able to download files from Keep over plain HTTP in Workbench.|
-|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
-|\3=. *User interface*|
-|"Single Sign On server":install-sso.html |Login server.|Required for web based login to Workbench.|
-|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, SSO server, keep-web, websockets server.|
-|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).|
-|\3=. *Additional services*|
-|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
-|"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.|
-|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
-|\3=. *Crunch (running containers)*|
-|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a SLURM cluster.|Optional if you wish to use Arvados for data management only.|
-|"Node Manager":install-nodemanager.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static SLURM cluster (such as on-premise HPC).|
<pre><code> Services:
WebDAV:
InternalURLs:
- http://"<span class="userinput">localhost:9002</span>": {}
+ http://<span class="userinput">localhost:9002</span>: {}
</code></pre>
</notextile>
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name <span class="userinput">download.ClusterID.example.com</span>
<span class="userinput">collections.ClusterID.example.com</span>
<span class="userinput">*.collections.ClusterID.example.com</span>
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name <span class="userinput">keep.ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_http_version 1.1;
proxy_request_buffering off;
- ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
Keepstore:
# No ExternalURL because they are only accessed by the internal subnet.
InternalURLs:
- "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {}
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {}
# and so forth
</code></pre>
</notextile>
|"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to access files from Workbench.|
|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
|\3=. *User interface*|
-|"Single Sign On server":install-sso.html |Web based login to Workbench.|Depends on identity provider. Not required for Google. Required for LDAP or standalone database.|
-|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, SSO server, keep-web, websockets server.|
+|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, keep-web, websockets server.|
|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).|
|\3=. *Additional services*|
|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
Choose which backend you will use to authenticate users.
-* Google login to authenticate users with a Google account. Note: if you only use this identity provider, login can be handled by @arvados-controller@ (recommended), and you do not need to install the Arvados Single Sign-On server (SSO).
-* LDAP login to authenticate users using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory. Supports username/password authentication.
-* Standalone SSO server user database. Supports username/password authentication. Supports new user sign-up.
+* Google login to authenticate users with a Google account.
+* LDAP login to authenticate users by username/password using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory.
+* PAM login to authenticate users by username/password according to the PAM configuration on the controller node.
h2(#storage). Storage backend
table(table table-bordered table-condensed).
|_. Function|_. Number of nodes|_. Recommended specs|
|Postgres database, Arvados API server, Arvados controller, Git, Websockets, Container dispatcher|1|16+ GiB RAM, 4+ cores, fast disk for database|
-|Single Sign-On (SSO) server ^1^|1|2 GiB RAM|
|Workbench, Keepproxy, Keep-web, Keep-balance|1|8 GiB RAM, 2+ cores|
-|Keepstore servers ^2^|2+|4 GiB RAM|
-|Compute worker nodes ^2^|0+ |Depends on workload; scaled dynamically in the cloud|
-|User shell nodes ^3^|0+|Depends on workload|
+|Keepstore servers ^1^|2+|4 GiB RAM|
+|Compute worker nodes ^1^|0+ |Depends on workload; scaled dynamically in the cloud|
+|User shell nodes ^2^|0+|Depends on workload|
</div>
-^1^ May be omitted when using Google login support in @arvados-controller@
-^2^ Should be scaled up as needed
-^3^ Refers to shell nodes managed by Arvados, that provide ssh access for users to interact with Arvados at the command line. Optional.
+^1^ Should be scaled up as needed
+^2^ Refers to shell nodes managed by Arvados, that provide ssh access for users to interact with Arvados at the command line. Optional.
{% include 'notebox_begin' %}
For a small demo installation, it is possible to run all the Arvados services on a single node. Special considerations for single-node installs will be noted in boxes like this.
|Arvados API|@ClusterID.example.com@|
|Arvados Git server|git.@ClusterID.example.com@|
|Arvados Websockets endpoint|ws.@ClusterID.example.com@|
-|Arvados SSO Server|@auth.example.com@|
|Arvados Workbench|workbench.@ClusterID.example.com@|
|Arvados Workbench 2|workbench2.@ClusterID.example.com@|
|Arvados Keepproxy server|keep.@ClusterID.example.com@|
<notextile>
<pre>
-<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'</span>
{
...
"api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install the Single Sign On (SSO) server
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% include 'notebox_begin_warning' %}
-Skip this section if you are using Google login via @arvados-controller@.
-{% include 'notebox_end' %}
-
-# "Install dependencies":#dependencies
-# "Set up database":#database-setup
-# "Update config.yml":#update-config
-# "Configure the SSO server":#create-application-yml
-# "Update Nginx configuration":#update-nginx
-# "Install arvados-sso-server":#install-packages
-# "Create arvados-server client record":#client
-# "Restart the API server and controller":#restart-api
-
-h2(#dependencies). Install dependencies
-
-# "Install PostgreSQL":install-postgresql.html
-# "Install Ruby and Bundler":ruby.html Important! The Single Sign On server only supports Ruby 2.3, to avoid version conflicts we recommend installing it on a different server from the API server. When installing Ruby, ensure that you get the right version by installing the "ruby2.3" package, or by using RVM with @--ruby=2.3@
-# "Install nginx":nginx.html
-# "Install Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html
-
-h2(#database-setup). Set up the database
-
-{% assign service_role = "arvados_sso" %}
-{% assign service_database = "arvados_sso_production" %}
-{% assign use_contrib = false %}
-{% include 'install_postgres_database' %}
-
-Now create @/etc/arvados/sso/database.yml@
-
-<pre>
-production:
- adapter: postgresql
- encoding: utf8
- database: arvados_sso_production
- username: arvados_sso
- password: $password
- host: localhost
- template: template0
-</pre>
-
-h2(#update-config). Update config.yml
-
-<pre>
- Services:
- SSO:
- ExternalURL: auth.ClusterID.example.com
- Login:
- ProviderAppID: "arvados-server"
- ProviderAppSecret: $app_secret
-</pre>
-
-Generate @ProviderAppSecret@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</code></pre></notextile>
-
-h2(#create-application-yml). Configure the SSO server
-
-The SSO server runs from the @/var/www/arvados-sso/current/@ directory. The files @/var/www/arvados-sso/current/config/application.yml@ and @/var/www/arvados-sso/current/config/database.yml@ will be symlinked to the configuration files in @/etc/arvados/sso/@.
-
-The SSO server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the SSO server and is provided for installation convenience only.
-
-Consult @config/application.default.yml@ for a full list of configuration options. Local configuration goes in @/etc/arvados/sso/application.yml@, do not edit @config/application.default.yml@.
-
-Create @/etc/arvados/sso/application.yml@ and add these keys:
-
-<pre>
-production:
- uuid_prefix: xxxxx
- secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</pre>
-
-h3(#uuid_prefix). uuid_prefix
-
-Most of the time, you want this to be the same as your @ClusterID@. If not, generate a new one from the command line listed previously.
-
-h3(#secret_token). secret_token
-
-Generate a new secret token for signing cookies:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</code></pre></notextile>
-
-h3(#authentication_methods). Authentication methods
-
-Authentication methods are configured in @application.yml@. Currently three authentication methods are supported: local accounts, LDAP, and Google. If neither Google nor LDAP are enabled, the SSO server defaults to local user accounts. Only one authentication mechanism should be in use at a time. Choose your authentication method and add the listed configuration items to the @production@ section.
-
-h4(#local_accounts). Local account authentication
-
-There are two configuration options for local accounts:
-
-<pre>
- # If true, allow new creation of new accounts in the SSO server's internal
- # user database.
- allow_account_registration: false
-
- # If true, send an email confirmation before activating new accounts in the
- # SSO server's internal user database (otherwise users are activated immediately.)
- require_email_confirmation: false
-</pre>
-
-For more information about configuring backend support for sending email (required to send email confirmations) see "Configuring Action Mailer":http://guides.rubyonrails.org/configuring.html#configuring-action-mailer
-
-If @allow_account_registration@ is false, you may manually create local accounts on the SSO server from the Rails console. {% include 'install_rails_command' %}
-
-Enter the following commands at the console.
-
-<notextile>
-<pre><code>:001 > <span class="userinput">user = User.new(:email => "test@example.com")</span>
-:002 > <span class="userinput">user.password = "passw0rd"</span>
-:003 > <span class="userinput">user.save!</span>
-:004 > <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h4(#ldap). LDAP authentication
-
-The following options are available to configure LDAP authentication. Note that you must preserve the indentation of the fields listed under @use_ldap@.
-
-<pre>
- use_ldap:
- title: Example LDAP
- host: ldap.example.com
- port: 636
- method: ssl
- base: "ou=Users, dc=example, dc=com"
- uid: uid
- email_domain: example.com
- #bind_dn: "some_user"
- #password: "some_password"
-</pre>
-
-table(table).
-|_. Option|_. Description|
-|title |Title displayed to the user on the login page|
-|host |LDAP server hostname|
-|port |LDAP server port|
-|method|One of "plain", "ssl", "tls"|
-|base |Directory lookup base|
-|uid |User id field used for directory lookup|
-|email_domain|Strip off specified email domain from login and perform lookup on bare username|
-|bind_dn|If required by server, username to log with in before performing directory lookup|
-|password|If required by server, password to log with before performing directory lookup|
-
-h4(#google). Google authentication
-
-First, visit "Setting up Google auth.":google-auth.html
-
-Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
-
-<notextile>
-<pre><code> # Google API tokens required for OAuth2 login.
- google_oauth2_client_id: <span class="userinput">"---YOUR---CLIENT---ID---HERE--"-</span>
- google_oauth2_client_secret: <span class="userinput">"---YOUR---CLIENT---SECRET---HERE--"-</span></code></pre></notextile>
-
-h2(#update-nginx). Update nginx configuration
-
-Use a text editor to create a new file @/etc/nginx/conf.d/arvados-sso.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
-
-<notextile>
-<pre><code>server {
- listen <span class="userinput">auth.ClusterID.example.com</span>:443 ssl;
- server_name <span class="userinput">auth.ClusterID.example.com</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- root /var/www/arvados-sso/current/public;
- index index.html;
-
- passenger_enabled on;
-
- # <span class="userinput">If you are using RVM, uncomment the line below.</span>
- # <span class="userinput">If you're using system ruby, leave it commented out.</span>
- #passenger_ruby /usr/local/rvm/wrappers/default/ruby;
-}
-</code></pre>
-</notextile>
-
-h2(#install-packages). Install arvados-sso-server package
-
-h3. Centos 7
-
-<notextile>
-<pre><code># <span class="userinput">yum install arvados-sso-server</span>
-</code></pre>
-</notextile>
-
-h3. Debian and Ubuntu
-
-<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends arvados-sso-server</span>
-</code></pre>
-</notextile>
-
-h2(#client). Create arvados-server client record
-
-{% assign railshost = "" %}
-{% assign railsdir = "/var/www/arvados-sso/current" %}
-Use @rails console@ to create a @Client@ record that will be used by the Arvados API server. {% include 'install_rails_command' %}
-
-Enter the following commands at the console. The values that appear after you assign @app_id@ and @app_secret@ will be copied to @Login.ProviderAppID@ and @Login.ProviderAppSecret@ in @config.yml@.
-
-<notextile>
-<pre><code>:001 > <span class="userinput">c = Client.new</span>
-:002 > <span class="userinput">c.name = "joshid"</span>
-:003 > <span class="userinput">c.app_id = "arvados-server"</span>
-:004 > <span class="userinput">c.app_secret = "the value of Login.ProviderAppSecret"</span>
-:005 > <span class="userinput">c.save!</span>
-:006 > <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h2(#restart-api). Restart the API server and controller
-
-After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
-
-<notextile>
-<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
-</code></pre>
-</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Configure webshell
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Introduction":#introduction
+# "Prerequisites":#prerequisites
+# "Update config.yml":#configure
+# "Update nginx configuration":#update-nginx
+# "Install packages":#install-packages
+# "Configure shellinabox":#config-shellinabox
+# "Configure pam":#config-pam
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
+Arvados supports @webshell@, which allows ssh access to shell nodes via the browser. This functionality is integrated in @Workbench@.
+
+@Webshell@ is provided by the @shellinabox@ package which runs on each shell node for which webshell is enabled. For authentication, a supported @pam library@ that allows authentication against Arvados is also required. One Nginx (or similar web server) virtualhost is also needed to expose all the @shellinabox@ instances via https.
+
+h2(#prerequisites). Prerequisites
+
+# "Install workbench":{{site.baseurl}}/install/install-workbench-app.html
+# "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html
+
+h2(#configure). Update config.yml
+
+Edit the cluster config at @config.yml@ and set @Services.WebShell.ExternalURL@. Replace @zzzzz@ with your cluster id. Workbench will use this information to activate its support for webshell.
+
+<notextile>
+<pre><code> Services:
+ WebShell:
+ InternalURLs: {}
+ ExternalURL: <span class="userinput">https://webshell.ClusterID.example.com/</span>
+</span></code></pre>
+</notextile>
+
+h2(#update-nginx). Update Nginx configuration
+
+The arvados-webshell service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption. This Nginx virtualhost could live on your Workbench server, or any other server that is reachable by your Workbench users and can access the @shell-in-a-box@ service on the shell node(s) on port 4200.
+
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-webshell.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+upstream arvados-webshell {
+ server <span class="userinput">shell.ClusterID.example.com</span>:<span class="userinput">4200</span>;
+}
+
+server {
+ listen 443 ssl;
+ server_name webshell.<span class="userinput">ClusterID.example.com</span>;
+
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+ location /<span class="userinput">shell.ClusterID</span> {
+ if ($request_method = 'OPTIONS') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ add_header 'Access-Control-Max-Age' 1728000;
+ add_header 'Content-Type' 'text/plain charset=UTF-8';
+ add_header 'Content-Length' 0;
+ return 204;
+ }
+ if ($request_method = 'POST') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+ if ($request_method = 'GET') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+
+ proxy_ssl_session_reuse off;
+ proxy_read_timeout 90;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_pass http://arvados-webshell;
+ }
+}
+</pre></notextile>
+
+Note that the location line in the nginx config matches your shell node hostname *without domain*, because that is how the shell node was defined in the "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html#vm-record instructions. It makes for a more user friendly experience in Workbench.
+
+For additional shell nodes with @shell-in-a-box@, add @location@ and @upstream@ sections as needed.
+
+{% assign arvados_component = 'shellinabox libpam-arvados-go' %}
+
+{% include 'install_packages' %}
+
+h2(#config-shellinabox). Configure shellinabox
+
+h3. Red Hat and Centos
+
+Edit @/etc/sysconfig/shellinaboxd@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+OPTS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+Edit @/etc/default/shellinabox@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+SHELLINABOX_PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+
+h2(#config-pam). Configure pam
+
+Use a text editor to create a new file @/etc/pam.d/shellinabox@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+# This example is a stock debian "login" file with pam_arvados
+# replacing pam_unix. It can be installed as /etc/pam.d/shellinabox .
+
+auth optional pam_faildelay.so delay=3000000
+auth [success=ok new_authtok_reqd=ok ignore=ignore user_unknown=bad default=die] pam_securetty.so
+auth requisite pam_nologin.so
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so close
+session required pam_env.so readenv=1
+session required pam_env.so readenv=1 envfile=/etc/default/locale
+
+auth [success=1 default=ignore] /usr/lib/pam_arvados.so <span class="userinput">ClusterID.example.com</span> <span class="userinput">shell.ClusterID.example.com</span>
+auth requisite pam_deny.so
+auth required pam_permit.so
+
+auth optional pam_group.so
+session required pam_limits.so
+session optional pam_lastlog.so
+session optional pam_motd.so motd=/run/motd.dynamic
+session optional pam_motd.so
+session optional pam_mail.so standard
+
+@include common-account
+@include common-session
+@include common-password
+
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so open
+</pre></notextile>
+
+h2(#confirm-working). Confirm working installation
+
+A user should be able to log in to the shell server, using webshell via workbench. Please refer to "Accessing an Arvados VM with Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html
+
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name workbench.<span class="userinput">ClusterID.example.com</span>;
- ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name workbench2.<span class="userinput">ClusterID.example.com</span>;
- ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
}
server {
- listen *:443 ssl;
+ listen 443 ssl;
server_name ws.<span class="userinput">ClusterID.example.com</span>;
proxy_connect_timeout 90s;
{% include 'restart_api' %}
-h2(#restart-api). Restart the API server and controller
-
-After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
-
-<notextile>
-<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
-</code></pre>
-</notextile>
-
h2(#confirm). Confirm working installation
Confirm the service is listening on its assigned port and responding to requests.
<notextile>
-<pre><code>~$ <span class="userinput">curl https://<span class="userinput">ws.ClusterID.example.com</span>/status.json</span>
-{"Clients":1}
+<pre><code>~$ <span class="userinput">curl https://<span class="userinput">ws.ClusterID.example.com</span>/websocket</span>
+not websocket protocol
</code></pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-# "Option 1: Google login through Arvados controller":#controller
-# "Option 2: Separate single-sign-on (SSO) server (Google, LDAP, local database)":#sso
+Select one of the following login mechanisms for your cluster.
-h2(#controller). Option 1: Google login through Arvados controller
+# If all users will authenticate with Google, "configure Google login":#google.
+# If all users will authenticate with an OpenID Connect provider (other than Google), "configure OpenID Connect":#oidc.
+# If all users will authenticate with an existing LDAP service, "configure LDAP":#ldap.
+# If all users will authenticate using PAM as configured on your controller node, "configure PAM":#pam.
-First, visit "Setting up Google auth.":google-auth.html
+h2(#google). Google login
-Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into @Login.GoogleClientID@ and @Login.GoogleClientSecret@ of @config.yml@ :
+With this configuration, users will sign in with their Google accounts.
+
+Use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
+# Select or create a project.
+# Click *+ Enable APIs and Services*.
+#* Search for *People API* and click *Enable API*.
+#* Navigate back to the main "APIs & Services" page.
+# On the sidebar, click *OAuth consent screen*.
+#* On consent screen settings, enter your identifying details.
+#* Under *Authorized domains* add your domain (@example.com@).
+#* Click *Save*.
+# On the sidebar, click *Credentials*, then click *Create credentials*→*OAuth client ID*
+# Under *Application type* select *Web application*.
+# Add the JavaScript origin: @https://ClusterID.example.com/@
+# Add the Redirect URI: @https://ClusterID.example.com/login@
+# Copy the values of *Client ID* and *Client secret* to the @Login.Google@ section of @config.yml@.
+
+<pre>
+ Login:
+ Google:
+ Enable: true
+ ClientID: "0000000000000-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz.apps.googleusercontent.com"
+ ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
+</pre>
+
+h2(#oidc). OpenID Connect
+
+With this configuration, users will sign in with a third-party OpenID Connect provider. The provider will supply appropriate values for the issuer URL, client ID, and client secret config entries.
<pre>
Login:
- GoogleClientID: ""
- GoogleClientSecret: ""
+ OpenIDConnect:
+ Enable: true
+ Issuer: https://accounts.example.com/
+ ClientID: "0123456789abcdef"
+ ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
</pre>
-h2(#sso). Option 2: Separate single-sign-on (SSO) server (supports Google, LDAP, local database)
+Check the OpenIDConnect section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
+
+h2(#ldap). LDAP
+
+With this configuration, authentication uses an external LDAP service like OpenLDAP or Active Directory.
+
+Enable LDAP authentication and provide your LDAP server's host, port, and credentials (if needed to search the directory) in @config.yml@:
+
+<pre>
+ Login:
+ LDAP:
+ Enable: true
+ URL: ldap://ldap.example.com:389
+ SearchBindUser: cn=lookupuser,dc=example,dc=com
+ SearchBindPassword: xxxxxxxx
+ SearchBase: ou=Users,dc=example,dc=com
+</pre>
+
+The email address reported by LDAP will be used as primary key for Arvados accounts. This means *users must not be able to edit their own email addresses* in the directory.
+
+Additional configuration settings are available:
+* @StartTLS@ is enabled by default.
+* @StripDomain@ and @AppendDomain@ modify the username entered by the user before searching for it in the directory.
+* @SearchAttribute@ (default @uid@) is the LDAP attribute used when searching for usernames.
+* @SearchFilters@ accepts LDAP filter expressions to control which users can log in.
+
+Check the LDAP section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
+
+h2(#pam). PAM (experimental)
+
+With this configuration, authentication is done according to the Linux PAM ("Pluggable Authentication Modules") configuration on your controller host.
+
+Enable PAM authentication in @config.yml@:
+
+<pre>
+ Login:
+ PAM:
+ Enable: true
+</pre>
+
+Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
+
+The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a shell account and password on the controller host itself. This can be convenient for a single-user or test cluster.
+
+PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
-See "Install the Single Sign On (SSO) server":install-sso.html
+For information about configuring PAM, refer to the "PAM System Administrator's Guide":http://www.linux-pam.org/Linux-PAM-html/Linux-PAM_SAG.html.
h2. On the web
-The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org . The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components. The "Arvados blog":https://dev.arvados.org/projects/arvados/blogs posts articles of interest about Arvados.
+The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org . The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components.
-h2. Mailing lists
+h2. Forum
-The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a forum for general discussion, questions, and news about Arvados development. The "Arvados developer mailing list":http://lists.arvados.org/mailman/listinfo/arvados-dev is a forum for more technical discussion, intended for developers and contributers to Arvados.
+The "Arvados Forum":https://forum.arvados.org has topic-based discussion, Q&A and community support.
h2. Chat
-The "arvados community channel":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+The "arvados/community":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+
+h2. Mailing list
+
+The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a low-volume list used mainly to announce new releases of Arvados.
h2. Bug tracking
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The Arvados API token is a secret key that enables the @arv@ command line client to access Arvados with the proper permissions.
+The Arvados API token is a secret key that enables the Arvados command line tools to authenticate themselves.
Access the Arvados Workbench using this link: "{{site.arvados_workbench_host}}/":{{site.arvados_workbench_host}}/ (Replace the hostname portion with the hostname of your local Arvados instance if necessary.)
This tool reads a CSV (comma-separated values) file having information about external groups and their members. When running it for the first time, it'll create a special group named 'Externally synchronized groups' meant to be the parent of all the remote groups.
-Every line on the file should have 2 values: a group name and a local user identifier, meaning that the named user is a member of the group. The tool will create the group if it doesn't exist, and add the user to it. If group member is not present on the input file, the account will be removed from the group.
+Every line on the file should have 3 values: a group name, a local user identifier and a permission level, meaning that the named user is a member of the group with the provided permission. The tool will create the group if it doesn't exist, and add the user to it. If any group member is not present on the input file, it will be removed from the group.
Users can be identified by their email address or username: the tool will check if every user exist on the system, and report back when not found. Groups on the other hand, are identified by their name.
+Permission level can be one of the following: @can_read@, @can_write@ or @can_manage@, giving the group member read, read/write or managing privileges on the group. For backwards compatibility purposes, if any record omits the third (permission) field, it will default to @can_write@ permission. You can read more about permissions on the "group management admin guide":/admin/group-management.html.
+
This tool is designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
# SPDX-License-Identifier: Apache-2.0
# Based on Debian Stretch
-FROM debian:stretch-slim
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
# apt.arvados.org
-deb http://apt.arvados.org/ stretch-dev main
+deb http://apt.arvados.org/ buster-dev main
# apt.arvados.org
-deb http://apt.arvados.org/ stretch main
+deb http://apt.arvados.org/ buster main
# apt.arvados.org
-deb http://apt.arvados.org/ stretch-testing main
+deb http://apt.arvados.org/ buster-testing main
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
github.com/arvados/cgofuse v1.2.0-arvados1
github.com/aws/aws-sdk-go v1.25.30
+ github.com/bgentry/speakeasy v0.1.0 // indirect
+ github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
github.com/coreos/go-oidc v2.1.0+incompatible
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
github.com/dgrijalva/jwt-go v3.1.0+incompatible // indirect
github.com/docker/docker v1.4.2-0.20180109013817-94b8a116fbf1
github.com/docker/go-connections v0.3.0 // indirect
github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d // indirect
+ github.com/dustin/go-humanize v1.0.0
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+ github.com/fsnotify/fsnotify v1.4.9
github.com/ghodss/yaml v1.0.0
github.com/gliderlabs/ssh v0.2.2 // indirect
+ github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
+ github.com/go-ldap/ldap v3.0.3+incompatible
github.com/gogo/protobuf v1.1.1
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/mux v1.6.1-0.20180107155708-5bbbb5b2b572
github.com/imdario/mergo v0.3.8-0.20190415133143-5ef87b449ca7
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
+ github.com/jmoiron/sqlx v1.2.0
github.com/julienschmidt/httprouter v1.2.0
github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 // indirect
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd
google.golang.org/api v0.13.0
+ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
gopkg.in/square/go-jose.v2 v2.3.1
gopkg.in/src-d/go-billy.v4 v4.0.1
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 h1:0Di2onNnlN5PAyWPbqlPyN45eOQ+QW/J9eqLynt4IV4=
+github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092/go.mod h1:8IzBjZCRSnsvM6MJMG8HNNtnzMl48H22rbJL2kRUJ0Y=
github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d h1:dVaNRYvaGV23AdNdsm+4y1mPN0tj3/1v6taqKMmM6Ko=
github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
+github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
+github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff/go.mod h1:ddfPX8Z28YMjiqoaJhNBzWHapTHXejnB5cDCUWDwriw=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c h1:ouxemItv3B/Zh008HJkEXDYCN3BIRyNHxtUN7ThJ5Js=
github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 h1:eQox4Rh4ewJF+mqYPxCkmBAirRnPaHEB26UkNuPyjlk=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII=
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405 h1:829vOVxxusYHC+IqBtkX5mbKtsY9fheQiQn0MZRVLfQ=
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
String() string
}
+var errNeedConfigReload = errors.New("config changed, restart needed")
+
type bootCommand struct{}
-func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- super := &Supervisor{
- Stderr: stderr,
- logger: ctxlog.New(stderr, "json", "info"),
+func (bcmd bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "json", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ for {
+ err := bcmd.run(ctx, prog, args, stdin, stdout, stderr)
+ if err == errNeedConfigReload {
+ continue
+ } else if err != nil {
+ logger.WithError(err).Info("exiting")
+ return 1
+ } else {
+ return 0
+ }
}
+}
- ctx := ctxlog.Context(context.Background(), super.logger)
+func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
-
- var err error
- defer func() {
- if err != nil {
- super.logger.WithError(err).Info("exiting")
- }
- }()
+ super := &Supervisor{
+ Stderr: stderr,
+ logger: ctxlog.FromContext(ctx),
+ }
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.SetOutput(stderr)
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
- err = flags.Parse(args)
+ err := flags.Parse(args)
if err == flag.ErrHelp {
- err = nil
- return 0
+ return nil
} else if err != nil {
- return 2
+ return err
} else if *versionFlag {
- return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ return nil
} else if super.ClusterType != "development" && super.ClusterType != "test" && super.ClusterType != "production" {
- err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
- return 2
+ return fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
}
loader.SkipAPICalls = true
cfg, err := loader.Load()
if err != nil {
- return 1
+ return err
}
- super.Start(ctx, cfg)
+ super.Start(ctx, cfg, loader.Path)
defer super.Stop()
var timer *time.Timer
url, ok := super.WaitReady()
if timer != nil && !timer.Stop() {
- err = errors.New("boot timed out")
- return 1
+ return errors.New("boot timed out")
} else if !ok {
- err = errors.New("boot failed")
- return 1
- }
- // Write controller URL to stdout. Nothing else goes to
- // stdout, so this provides an easy way for a calling script
- // to discover the controller URL when everything is ready.
- fmt.Fprintln(stdout, url)
- if *shutdown {
- super.Stop()
+ super.logger.Error("boot failed")
+ } else {
+ // Write controller URL to stdout. Nothing else goes
+ // to stdout, so this provides an easy way for a
+ // calling script to discover the controller URL when
+ // everything is ready.
+ fmt.Fprintln(stdout, url)
+ if *shutdown {
+ super.Stop()
+ }
}
// Wait for signal/crash + orderly shutdown
- <-super.done
- return 0
+ return super.Wait()
}
{"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
{"KEEPPROXY", super.cluster.Services.Keepproxy},
{"GIT", super.cluster.Services.GitHTTP},
+ {"HEALTH", super.cluster.Services.Health},
{"WORKBENCH1", super.cluster.Services.Workbench1},
{"WS", super.cluster.Services.Websocket},
} {
"os/signal"
"os/user"
"path/filepath"
+ "reflect"
"strings"
"sync"
"syscall"
"time"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
+ "github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
)
ctx context.Context
cancel context.CancelFunc
- done chan struct{}
+ done chan struct{} // closed when child procs/services have shut down
+ err error // error that caused shutdown (valid when done is closed)
healthChecker *health.Aggregator
tasksReady map[string]chan bool
waitShutdown sync.WaitGroup
environ []string // for child processes
}
-func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
+func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config, cfgPath string) {
super.ctx, super.cancel = context.WithCancel(ctx)
super.done = make(chan struct{})
go func() {
+ defer close(super.done)
+
sigch := make(chan os.Signal)
signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(sigch)
go func() {
for sig := range sigch {
super.logger.WithField("signal", sig).Info("caught signal")
+ if super.err == nil {
+ super.err = fmt.Errorf("caught signal %s", sig)
+ }
+ super.cancel()
+ }
+ }()
+
+ hupch := make(chan os.Signal)
+ signal.Notify(hupch, syscall.SIGHUP)
+ defer signal.Stop(hupch)
+ go func() {
+ for sig := range hupch {
+ super.logger.WithField("signal", sig).Info("caught signal")
+ if super.err == nil {
+ super.err = errNeedConfigReload
+ }
super.cancel()
}
}()
+ if cfgPath != "" && cfgPath != "-" && cfg.AutoReloadConfig {
+ go watchConfig(super.ctx, super.logger, cfgPath, copyConfig(cfg), func() {
+ if super.err == nil {
+ super.err = errNeedConfigReload
+ }
+ super.cancel()
+ })
+ }
+
err := super.run(cfg)
if err != nil {
super.logger.WithError(err).Warn("supervisor shut down")
+ if super.err == nil {
+ super.err = err
+ }
}
- close(super.done)
}()
}
+func (super *Supervisor) Wait() error {
+ <-super.done
+ return super.err
+}
+
func (super *Supervisor) run(cfg *arvados.Config) error {
+ defer super.cancel()
+
cwd, err := os.Getwd()
if err != nil {
return err
if p == "0" {
p = nextPort(h)
}
- cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
+ cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
if svc.ExternalURL.Host == "" {
if svc == &cluster.Services.Controller ||
svc == &cluster.Services.GitHTTP ||
+ svc == &cluster.Services.Health ||
svc == &cluster.Services.Keepproxy ||
svc == &cluster.Services.WebDAV ||
svc == &cluster.Services.WebDAVDownload ||
svc == &cluster.Services.Workbench1 {
- svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
} else if svc == &cluster.Services.Websocket {
- svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
}
}
if len(svc.InternalURLs) == 0 {
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
- arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
+ arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{},
}
}
}
}
if super.ClusterType == "test" {
// Add a second keepstore process.
- cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
// Create a directory-backed volume for each keepstore
// process.
}
return ctx.Err()
}
+
+func copyConfig(cfg *arvados.Config) *arvados.Config {
+ pr, pw := io.Pipe()
+ go func() {
+ err := json.NewEncoder(pw).Encode(cfg)
+ if err != nil {
+ panic(err)
+ }
+ pw.Close()
+ }()
+ cfg2 := new(arvados.Config)
+ err := json.NewDecoder(pr).Decode(cfg2)
+ if err != nil {
+ panic(err)
+ }
+ return cfg2
+}
+
+func watchConfig(ctx context.Context, logger logrus.FieldLogger, cfgPath string, prevcfg *arvados.Config, fn func()) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ logger.WithError(err).Error("fsnotify setup failed")
+ return
+ }
+ defer watcher.Close()
+
+ err = watcher.Add(cfgPath)
+ if err != nil {
+ logger.WithError(err).Error("fsnotify watcher failed")
+ return
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ logger.WithError(err).Warn("fsnotify watcher reported error")
+ case _, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ for len(watcher.Events) > 0 {
+ <-watcher.Events
+ }
+ loader := config.NewLoader(&bytes.Buffer{}, &logrus.Logger{Out: ioutil.Discard})
+ loader.Path = cfgPath
+ loader.SkipAPICalls = true
+ cfg, err := loader.Load()
+ if err != nil {
+ logger.WithError(err).Warn("error reloading config file after change detected; ignoring new config for now")
+ } else if reflect.DeepEqual(cfg, prevcfg) {
+ logger.Debug("config file changed but is still DeepEqual to the existing config")
+ } else {
+ logger.Debug("config changed, notifying supervisor")
+ fn()
+ prevcfg = cfg
+ }
+ }
+ }
+}
code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
c.Check(stdout.String(), check.Matches, `(?ms).*TimeoutBooting: 10m\n.*`)
- c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345: {}\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345/: {}\n.*`)
}
func (s *CommandSuite) TestDump_UnknownKey(c *check.C) {
code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.UnknownKey.*`)
- c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n z1234:\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms)(.*\n)?Clusters:\n z1234:\n.*`)
c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
}
# or omitted, pages are processed serially.
BalanceCollectionBuffers: 1000
+ # Maximum time for a rebalancing run. This ensures keep-balance
+ # eventually gives up and retries if, for example, a network
+ # error causes a hung connection that is never closed by the
+ # OS. It should be long enough that it doesn't interrupt a
+ # long-running balancing operation.
+ BalanceTimeout: 6h
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
MaxUUIDEntries: 1000
Login:
- # These settings are provided by your OAuth2 provider (eg
- # Google) used to perform upstream authentication.
- ProviderAppID: ""
- ProviderAppSecret: ""
-
- # (Experimental) Authenticate with Google, bypassing the
- # SSO-provider gateway service. Use the Google Cloud console to
- # enable the People API (APIs and Services > Enable APIs and
- # services > Google People API > Enable), generate a Client ID
- # and secret (APIs and Services > Credentials > Create
- # credentials > OAuth client ID > Web application) and add your
- # controller's /login URL (e.g.,
- # "https://zzzzz.example.com/login") as an authorized redirect
- # URL.
- #
- # Incompatible with ForceLegacyAPI14. ProviderAppID must be
- # blank.
- GoogleClientID: ""
- GoogleClientSecret: ""
-
- # Allow users to log in to existing accounts using any verified
- # email address listed by their Google account. If true, the
- # Google People API must be enabled in order for Google login to
- # work. If false, only the primary email address will be used.
- GoogleAlternateEmailAddresses: true
-
- # (Experimental) Use PAM to authenticate logins, using the
- # specified PAM service name.
- #
- # Cannot be used in combination with OAuth2 (ProviderAppID) or
- # Google (GoogleClientID). Cannot be used on a cluster acting as
- # a LoginCluster.
- PAM: false
- PAMService: arvados
-
- # Domain name (e.g., "example.com") to use to construct the
- # user's email address if PAM authentication returns a username
- # with no "@". If empty, use the PAM username as the user's
- # email address, whether or not it contains "@".
- #
- # Note that the email address is used as the primary key for
- # user records when logging in. Therefore, if you change
- # PAMDefaultEmailDomain after the initial installation, you
- # should also update existing user records to reflect the new
- # domain. Otherwise, next time those users log in, they will be
- # given new accounts instead of accessing their existing
- # accounts.
- PAMDefaultEmailDomain: ""
+ # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+ # LoginCluster) should be enabled; see
+ # https://doc.arvados.org/install/setup-login.html
+
+ Google:
+ # Authenticate with Google.
+ Enable: false
+
+ # Use the Google Cloud console to enable the People API (APIs
+ # and Services > Enable APIs and services > Google People API
+ # > Enable), generate a Client ID and secret (APIs and
+ # Services > Credentials > Create credentials > OAuth client
+ # ID > Web application) and add your controller's /login URL
+ # (e.g., "https://zzzzz.example.com/login") as an authorized
+ # redirect URL.
+ #
+ # Incompatible with ForceLegacyAPI14. ProviderAppID must be
+ # blank.
+ ClientID: ""
+ ClientSecret: ""
+
+ # Allow users to log in to existing accounts using any verified
+ # email address listed by their Google account. If true, the
+ # Google People API must be enabled in order for Google login to
+ # work. If false, only the primary email address will be used.
+ AlternateEmailAddresses: true
+
+ OpenIDConnect:
+ # Authenticate with an OpenID Connect provider.
+ Enable: false
+
+ # Issuer URL, e.g., "https://login.example.com".
+ #
+ # This must be exactly equal to the URL returned by the issuer
+ # itself in its config response ("isser" key). If the
+ # configured value is "https://example" and the provider
+ # returns "https://example:443" or "https://example/" then
+ # login will fail, even though those URLs are equivalent
+ # (RFC3986).
+ Issuer: ""
+
+ # Your client ID and client secret (supplied by the provider).
+ ClientID: ""
+ ClientSecret: ""
+
+ # OpenID claim field containing the user's email
+ # address. Normally "email"; see
+ # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+ EmailClaim: "email"
+
+ # OpenID claim field containing the email verification
+ # flag. Normally "email_verified". To accept every returned
+ # email address without checking a "verified" field at all,
+ # use the empty string "".
+ EmailVerifiedClaim: "email_verified"
+
+ # OpenID claim field containing the user's preferred
+ # username. If empty, use the mailbox part of the user's email
+ # address.
+ UsernameClaim: ""
+
+ PAM:
+ # (Experimental) Use PAM to authenticate users.
+ Enable: false
+
+ # PAM service name. PAM will apply the policy in the
+ # corresponding config file (e.g., /etc/pam.d/arvados) or, if
+ # there is none, the default "other" config.
+ Service: arvados
+
+ # Domain name (e.g., "example.com") to use to construct the
+ # user's email address if PAM authentication returns a
+ # username with no "@". If empty, use the PAM username as the
+ # user's email address, whether or not it contains "@".
+ #
+ # Note that the email address is used as the primary key for
+ # user records when logging in. Therefore, if you change
+ # PAMDefaultEmailDomain after the initial installation, you
+ # should also update existing user records to reflect the new
+ # domain. Otherwise, next time those users log in, they will
+ # be given new accounts instead of accessing their existing
+ # accounts.
+ DefaultEmailDomain: ""
+
+ LDAP:
+ # Use an LDAP service to authenticate users.
+ Enable: false
+
+ # Server URL, like "ldap://ldapserver.example.com:389" or
+ # "ldaps://ldapserver.example.com:636".
+ URL: "ldap://ldap:389"
+
+ # Use StartTLS upon connecting to the server.
+ StartTLS: true
+
+ # Skip TLS certificate name verification.
+ InsecureTLS: false
+
+ # Strip the @domain part if a user supplies an email-style
+ # username with this domain. If "*", strip any user-provided
+ # domain. If "", never strip the domain part. Example:
+ # "example.com"
+ StripDomain: ""
+
+ # If, after applying StripDomain, the username contains no "@"
+ # character, append this domain to form an email-style
+ # username. Example: "example.com"
+ AppendDomain: ""
+
+ # The LDAP attribute to filter on when looking up a username
+ # (after applying StripDomain and AppendDomain).
+ SearchAttribute: uid
+
+ # Bind with this username (DN or UPN) and password when
+ # looking up the user record.
+ #
+ # Example user: "cn=admin,dc=example,dc=com"
+ SearchBindUser: ""
+ SearchBindPassword: ""
+
+ # Directory base for username lookup. Example:
+ # "ou=Users,dc=example,dc=com"
+ SearchBase: ""
+
+ # Additional filters to apply when looking up users' LDAP
+ # entries. This can be used to restrict access to a subset of
+ # LDAP users, or to disambiguate users from other directory
+ # entries that have the SearchAttribute present.
+ #
+ # Special characters in assertion values must be escaped (see
+ # RFC4515).
+ #
+ # Example: "(objectClass=person)"
+ SearchFilters: ""
+
+ # LDAP attribute to use as the user's email address.
+ #
+ # Important: This must not be an attribute whose value can be
+ # edited in the directory by the users themselves. Otherwise,
+ # users can take over other users' Arvados accounts trivially
+ # (email address is the primary key for Arvados accounts.)
+ EmailAttribute: mail
+
+ # LDAP attribute to use as the preferred Arvados username. If
+ # no value is found (or this config is empty) the username
+ # originally supplied by the user will be used.
+ UsernameAttribute: uid
+
+ SSO:
+ # Authenticate with a separate SSO server. (Deprecated)
+ Enable: false
+
+ # ProviderAppID and ProviderAppSecret are generated during SSO
+ # setup; see
+ # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
+ ProviderAppID: ""
+ ProviderAppSecret: ""
# The cluster ID to delegate the user database. When set,
# logins on this cluster will be redirected to the login cluster
Region: us-east-1a
Bucket: aaaaa
LocationConstraint: false
+ V2Signature: false
IndexPageSize: 1000
ConnectTimeout: 1m
ReadTimeout: 10m
RunningJobLogRecordsToFetch: 2000
# In systems with many shared projects, loading of dashboard and topnav
- # cab be slow due to collections indexing; use the following parameters
+ # can be slow due to collections indexing; use the following parameters
# to suppress these properties
ShowRecentCollectionsOnDashboard: true
ShowUserNotifications: true
# implementation. Note that it also disables some new federation
# features and will be removed in a future release.
ForceLegacyAPI14: false
+
+# (Experimental) Restart services automatically when config file
+# changes are detected. Only supported by `arvados-server boot` in
+# dev/test mode.
+AutoReloadConfig: false
type deprCluster struct {
RequestLimits deprRequestLimits
NodeProfiles map[string]nodeProfile
+ Login struct {
+ GoogleClientID *string
+ GoogleClientSecret *string
+ GoogleAlternateEmailAddresses *bool
+ ProviderAppID *string
+ ProviderAppSecret *string
+ }
}
type deprecatedConfig struct {
if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
*dst = *n
}
+
+ // Google* moved to Google.*
+ if dst, n := &cluster.Login.Google.ClientID, dcluster.Login.GoogleClientID; n != nil && *n != *dst {
+ *dst = *n
+ if *n != "" {
+ // In old config, non-empty ClientID meant enable
+ cluster.Login.Google.Enable = true
+ }
+ }
+ if dst, n := &cluster.Login.Google.ClientSecret, dcluster.Login.GoogleClientSecret; n != nil && *n != *dst {
+ *dst = *n
+ }
+ if dst, n := &cluster.Login.Google.AlternateEmailAddresses, dcluster.Login.GoogleAlternateEmailAddresses; n != nil && *n != *dst {
+ *dst = *n
+ }
+
+ // Provider* moved to SSO.Provider*
+ if dst, n := &cluster.Login.SSO.ProviderAppID, dcluster.Login.ProviderAppID; n != nil && *n != *dst {
+ *dst = *n
+ if *n != "" {
+ // In old config, non-empty ID meant enable
+ cluster.Login.SSO.Enable = true
+ }
+ }
+ if dst, n := &cluster.Login.SSO.ProviderAppSecret, dcluster.Login.ProviderAppSecret; n != nil && *n != *dst {
+ *dst = *n
+ }
+
cfg.Clusters[id] = cluster
}
return nil
if strings.HasPrefix(host, ":") {
host = hostname + host
}
- svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+ svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host, Path: "/"}] = arvados.ServiceInstance{}
}
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
}
if client.APIHost != "" {
cluster.Services.Controller.ExternalURL.Host = client.APIHost
+ cluster.Services.Controller.ExternalURL.Path = "/"
}
if client.Scheme != "" {
cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
}
if oc.Listen != nil {
- cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.LogLevel != nil {
cluster.SystemLogs.LogLevel = *oc.LogLevel
loadOldClientConfig(cluster, oc.Client)
if oc.Listen != nil {
- cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.DefaultReplicas != nil {
cluster.Collections.DefaultReplication = *oc.DefaultReplicas
loadOldClientConfig(cluster, oc.Client)
if oc.Listen != nil {
- cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
- cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.AttachmentOnlyHost != nil {
- cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost}
+ cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost, Path: "/"}
}
if oc.ManagementToken != nil {
cluster.ManagementToken = *oc.ManagementToken
return err
}
- myURL := arvados.URL{Scheme: "http"}
+ myURL := arvados.URL{Scheme: "http", Path: "/"}
if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
myURL.Scheme = "https"
}
cluster.TLS.Key = "file://" + *v
}
if v := oc.Listen; v != nil {
- if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+ if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v, Path: "/"}]; ok {
// already listed
myURL.Host = *v
} else if len(*v) > 1 && (*v)[0] == ':' {
url := arvados.URL{
Scheme: "http",
Host: net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+ Path: "/",
}
if ks.ServiceSSLFlag {
url.Scheme = "https"
TLS: {Insecure: true}
Services:
Controller:
- ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
`, `
Clusters:
z1111:
InternalURLs:
"http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
Controller:
- ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
SystemLogs:
Format: text
LogLevel: debug
ReadOnly: true
StorageAccountName: storageacctname
StorageAccountKeyFile: `+secretkeyfile.Name()+`
- StorageBaseURL: https://example.example
+ StorageBaseURL: https://example.example/
ContainerName: testctr
LocationConstraint: true
AzureReplication: 4
}, &arvados.AzureVolumeDriverParameters{
StorageAccountName: "storageacctname",
StorageAccountKey: "secretkeydata",
- StorageBaseURL: "https://example.example",
+ StorageBaseURL: "https://example.example/",
ContainerName: "testctr",
RequestTimeout: arvados.Duration(time.Minute * 3),
ListBlobsRetryDelay: arvados.Duration(time.Minute * 4),
c.Check(v.Driver, check.Equals, expectvol.Driver)
c.Check(v.Replication, check.Equals, expectvol.Replication)
- avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+ avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345", Path: "/"}]
c.Check(ok, check.Equals, true)
c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
url := arvados.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", hostname, port),
+ Path: "/",
}
_, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
c.Check(ok, check.Equals, false)
url := arvados.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", hostname, port),
+ Path: "/",
}
_, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
c.Check(ok, check.Equals, false)
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
- _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+ _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port), Path: "/"}]
c.Check(ok, check.Equals, true)
}
c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
- c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107 does not have access to any volumes.*`)
+ c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107/ does not have access to any volumes.*`)
c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
}
`)
}
+func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
+ checkEquivalent(c, `
+Clusters:
+ z1111:
+ Login:
+ GoogleClientID: aaaa
+ GoogleClientSecret: bbbb
+ GoogleAlternateEmailAddresses: true
+`, `
+Clusters:
+ z1111:
+ Login:
+ Google:
+ Enable: true
+ ClientID: aaaa
+ ClientSecret: bbbb
+ AlternateEmailAddresses: true
+`)
+ checkEquivalent(c, `
+Clusters:
+ z1111:
+ Login:
+ ProviderAppID: aaaa
+ ProviderAppSecret: bbbb
+`, `
+Clusters:
+ z1111:
+ Login:
+ SSO:
+ Enable: true
+ ProviderAppID: aaaa
+ ProviderAppSecret: bbbb
+`)
+}
+
func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
content := []byte(`
{
cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
c.Check(err, check.IsNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
- c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com"})
+ c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
"Collections.WebDAVCache": false,
"Collections.BalanceCollectionBatch": false,
"Collections.BalancePeriod": false,
+ "Collections.BalanceTimeout": false,
"Collections.BlobMissingReport": false,
"Collections.BalanceCollectionBuffers": false,
"Containers": true,
"InstanceTypes.*": true,
"InstanceTypes.*.*": true,
"Login": true,
- "Login.GoogleClientID": false,
- "Login.GoogleClientSecret": false,
- "Login.GoogleAlternateEmailAddresses": false,
- "Login.PAM": true,
- "Login.PAMService": false,
- "Login.PAMDefaultEmailDomain": false,
- "Login.ProviderAppID": false,
- "Login.ProviderAppSecret": false,
+ "Login.Google": true,
+ "Login.Google.AlternateEmailAddresses": false,
+ "Login.Google.ClientID": false,
+ "Login.Google.ClientSecret": false,
+ "Login.Google.Enable": true,
+ "Login.LDAP": true,
+ "Login.LDAP.AppendDomain": false,
+ "Login.LDAP.EmailAttribute": false,
+ "Login.LDAP.Enable": true,
+ "Login.LDAP.InsecureTLS": false,
+ "Login.LDAP.SearchAttribute": false,
+ "Login.LDAP.SearchBase": false,
+ "Login.LDAP.SearchBindPassword": false,
+ "Login.LDAP.SearchBindUser": false,
+ "Login.LDAP.SearchFilters": false,
+ "Login.LDAP.StartTLS": false,
+ "Login.LDAP.StripDomain": false,
+ "Login.LDAP.URL": false,
+ "Login.LDAP.UsernameAttribute": false,
"Login.LoginCluster": true,
+ "Login.OpenIDConnect": true,
+ "Login.OpenIDConnect.ClientID": false,
+ "Login.OpenIDConnect.ClientSecret": false,
+ "Login.OpenIDConnect.Enable": true,
+ "Login.OpenIDConnect.Issuer": false,
+ "Login.OpenIDConnect.EmailClaim": false,
+ "Login.OpenIDConnect.EmailVerifiedClaim": false,
+ "Login.OpenIDConnect.UsernameClaim": false,
+ "Login.PAM": true,
+ "Login.PAM.DefaultEmailDomain": false,
+ "Login.PAM.Enable": true,
+ "Login.PAM.Service": false,
+ "Login.SSO": true,
+ "Login.SSO.Enable": true,
+ "Login.SSO.ProviderAppID": false,
+ "Login.SSO.ProviderAppSecret": false,
"Login.RemoteTokenRefresh": true,
"Mail": true,
"Mail.MailchimpAPIKey": false,
# or omitted, pages are processed serially.
BalanceCollectionBuffers: 1000
+ # Maximum time for a rebalancing run. This ensures keep-balance
+ # eventually gives up and retries if, for example, a network
+ # error causes a hung connection that is never closed by the
+ # OS. It should be long enough that it doesn't interrupt a
+ # long-running balancing operation.
+ BalanceTimeout: 6h
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
MaxUUIDEntries: 1000
Login:
- # These settings are provided by your OAuth2 provider (eg
- # Google) used to perform upstream authentication.
- ProviderAppID: ""
- ProviderAppSecret: ""
-
- # (Experimental) Authenticate with Google, bypassing the
- # SSO-provider gateway service. Use the Google Cloud console to
- # enable the People API (APIs and Services > Enable APIs and
- # services > Google People API > Enable), generate a Client ID
- # and secret (APIs and Services > Credentials > Create
- # credentials > OAuth client ID > Web application) and add your
- # controller's /login URL (e.g.,
- # "https://zzzzz.example.com/login") as an authorized redirect
- # URL.
- #
- # Incompatible with ForceLegacyAPI14. ProviderAppID must be
- # blank.
- GoogleClientID: ""
- GoogleClientSecret: ""
-
- # Allow users to log in to existing accounts using any verified
- # email address listed by their Google account. If true, the
- # Google People API must be enabled in order for Google login to
- # work. If false, only the primary email address will be used.
- GoogleAlternateEmailAddresses: true
-
- # (Experimental) Use PAM to authenticate logins, using the
- # specified PAM service name.
- #
- # Cannot be used in combination with OAuth2 (ProviderAppID) or
- # Google (GoogleClientID). Cannot be used on a cluster acting as
- # a LoginCluster.
- PAM: false
- PAMService: arvados
-
- # Domain name (e.g., "example.com") to use to construct the
- # user's email address if PAM authentication returns a username
- # with no "@". If empty, use the PAM username as the user's
- # email address, whether or not it contains "@".
- #
- # Note that the email address is used as the primary key for
- # user records when logging in. Therefore, if you change
- # PAMDefaultEmailDomain after the initial installation, you
- # should also update existing user records to reflect the new
- # domain. Otherwise, next time those users log in, they will be
- # given new accounts instead of accessing their existing
- # accounts.
- PAMDefaultEmailDomain: ""
+ # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+ # LoginCluster) should be enabled; see
+ # https://doc.arvados.org/install/setup-login.html
+
+ Google:
+ # Authenticate with Google.
+ Enable: false
+
+ # Use the Google Cloud console to enable the People API (APIs
+ # and Services > Enable APIs and services > Google People API
+ # > Enable), generate a Client ID and secret (APIs and
+ # Services > Credentials > Create credentials > OAuth client
+ # ID > Web application) and add your controller's /login URL
+ # (e.g., "https://zzzzz.example.com/login") as an authorized
+ # redirect URL.
+ #
+ # Incompatible with ForceLegacyAPI14. ProviderAppID must be
+ # blank.
+ ClientID: ""
+ ClientSecret: ""
+
+ # Allow users to log in to existing accounts using any verified
+ # email address listed by their Google account. If true, the
+ # Google People API must be enabled in order for Google login to
+ # work. If false, only the primary email address will be used.
+ AlternateEmailAddresses: true
+
+ OpenIDConnect:
+ # Authenticate with an OpenID Connect provider.
+ Enable: false
+
+ # Issuer URL, e.g., "https://login.example.com".
+ #
+ # This must be exactly equal to the URL returned by the issuer
+ # itself in its config response ("isser" key). If the
+ # configured value is "https://example" and the provider
+ # returns "https://example:443" or "https://example/" then
+ # login will fail, even though those URLs are equivalent
+ # (RFC3986).
+ Issuer: ""
+
+ # Your client ID and client secret (supplied by the provider).
+ ClientID: ""
+ ClientSecret: ""
+
+ # OpenID claim field containing the user's email
+ # address. Normally "email"; see
+ # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+ EmailClaim: "email"
+
+ # OpenID claim field containing the email verification
+ # flag. Normally "email_verified". To accept every returned
+ # email address without checking a "verified" field at all,
+ # use the empty string "".
+ EmailVerifiedClaim: "email_verified"
+
+ # OpenID claim field containing the user's preferred
+ # username. If empty, use the mailbox part of the user's email
+ # address.
+ UsernameClaim: ""
+
+ PAM:
+ # (Experimental) Use PAM to authenticate users.
+ Enable: false
+
+ # PAM service name. PAM will apply the policy in the
+ # corresponding config file (e.g., /etc/pam.d/arvados) or, if
+ # there is none, the default "other" config.
+ Service: arvados
+
+ # Domain name (e.g., "example.com") to use to construct the
+ # user's email address if PAM authentication returns a
+ # username with no "@". If empty, use the PAM username as the
+ # user's email address, whether or not it contains "@".
+ #
+ # Note that the email address is used as the primary key for
+ # user records when logging in. Therefore, if you change
+ # PAMDefaultEmailDomain after the initial installation, you
+ # should also update existing user records to reflect the new
+ # domain. Otherwise, next time those users log in, they will
+ # be given new accounts instead of accessing their existing
+ # accounts.
+ DefaultEmailDomain: ""
+
+ LDAP:
+ # Use an LDAP service to authenticate users.
+ Enable: false
+
+ # Server URL, like "ldap://ldapserver.example.com:389" or
+ # "ldaps://ldapserver.example.com:636".
+ URL: "ldap://ldap:389"
+
+ # Use StartTLS upon connecting to the server.
+ StartTLS: true
+
+ # Skip TLS certificate name verification.
+ InsecureTLS: false
+
+ # Strip the @domain part if a user supplies an email-style
+ # username with this domain. If "*", strip any user-provided
+ # domain. If "", never strip the domain part. Example:
+ # "example.com"
+ StripDomain: ""
+
+ # If, after applying StripDomain, the username contains no "@"
+ # character, append this domain to form an email-style
+ # username. Example: "example.com"
+ AppendDomain: ""
+
+ # The LDAP attribute to filter on when looking up a username
+ # (after applying StripDomain and AppendDomain).
+ SearchAttribute: uid
+
+ # Bind with this username (DN or UPN) and password when
+ # looking up the user record.
+ #
+ # Example user: "cn=admin,dc=example,dc=com"
+ SearchBindUser: ""
+ SearchBindPassword: ""
+
+ # Directory base for username lookup. Example:
+ # "ou=Users,dc=example,dc=com"
+ SearchBase: ""
+
+ # Additional filters to apply when looking up users' LDAP
+ # entries. This can be used to restrict access to a subset of
+ # LDAP users, or to disambiguate users from other directory
+ # entries that have the SearchAttribute present.
+ #
+ # Special characters in assertion values must be escaped (see
+ # RFC4515).
+ #
+ # Example: "(objectClass=person)"
+ SearchFilters: ""
+
+ # LDAP attribute to use as the user's email address.
+ #
+ # Important: This must not be an attribute whose value can be
+ # edited in the directory by the users themselves. Otherwise,
+ # users can take over other users' Arvados accounts trivially
+ # (email address is the primary key for Arvados accounts.)
+ EmailAttribute: mail
+
+ # LDAP attribute to use as the preferred Arvados username. If
+ # no value is found (or this config is empty) the username
+ # originally supplied by the user will be used.
+ UsernameAttribute: uid
+
+ SSO:
+ # Authenticate with a separate SSO server. (Deprecated)
+ Enable: false
+
+ # ProviderAppID and ProviderAppSecret are generated during SSO
+ # setup; see
+ # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
+ ProviderAppID: ""
+ ProviderAppSecret: ""
# The cluster ID to delegate the user database. When set,
# logins on this cluster will be redirected to the login cluster
Region: us-east-1a
Bucket: aaaaa
LocationConstraint: false
+ V2Signature: false
IndexPageSize: 1000
ConnectTimeout: 1m
ReadTimeout: 10m
RunningJobLogRecordsToFetch: 2000
# In systems with many shared projects, loading of dashboard and topnav
- # cab be slow due to collections indexing; use the following parameters
+ # can be slow due to collections indexing; use the following parameters
# to suppress these properties
ShowRecentCollectionsOnDashboard: true
ShowUserNotifications: true
# implementation. Note that it also disables some new federation
# features and will be removed in a future release.
ForceLegacyAPI14: false
+
+# (Experimental) Restart services automatically when config file
+# changes are detected. Only supported by ` + "`" + `arvados-server boot` + "`" + ` in
+# dev/test mode.
+AutoReloadConfig: false
`)
// // ldr.Path == "/tmp/c.yaml"
func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
- flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
- flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
- flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
- flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
- flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
- flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
- flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
- flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+ if !ldr.SkipLegacy {
+ flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
+ flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
+ flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
+ flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+ flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
+ flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+ flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
+ flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+ }
}
// MungeLegacyConfigArgs checks args for a -config flag whose argument
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// Package api provides types used by controller/server-component
+// packages.
+package api
+
+import "context"
+
+// A RoutableFunc calls an API method (sometimes via a wrapped
+// RoutableFunc) that has real argument types.
+//
+// (It is used by ctrlctx to manage database transactions, so moving
+// it to the router package would cause a circular dependency
+// router->arvadostest->ctrlctx->router.)
+type RoutableFunc func(ctx context.Context, opts interface{}) (interface{}, error)
return true
}
- if *clusterId == "" {
- *clusterId = h.handler.Cluster.ClusterID
- }
-
- if strings.HasPrefix(currentUser.Authorization.UUID, h.handler.Cluster.ClusterID) &&
- *clusterId == h.handler.Cluster.ClusterID {
- // local user submitting container request to local cluster
+ if *clusterId == "" || *clusterId == h.handler.Cluster.ClusterID {
+ // Submitting container request to local cluster. No
+ // need to set a runtime_token (rails api will create
+ // one when the container runs) or do a remote cluster
+ // request.
return false
}
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmcvetta/randutil"
)
// non-nil, true, nil -- if the token is valid
func (h *Handler) validateAPItoken(req *http.Request, token string) (*CurrentUser, bool, error) {
user := CurrentUser{Authorization: arvados.APIClientAuthorization{APIToken: token}}
- db, err := h.db(req)
+ db, err := h.db(req.Context())
if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): database error", token)
return nil, false, err
}
}
user.Authorization.APIToken = token
var scopes string
- err = db.QueryRowContext(req.Context(), `SELECT api_client_authorizations.uuid, api_client_authorizations.scopes, users.uuid FROM api_client_authorizations JOIN users on api_client_authorizations.user_id=users.id WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp) LIMIT 1`, token).Scan(&user.Authorization.UUID, &scopes, &user.UUID)
+ err = db.QueryRowContext(req.Context(), `SELECT api_client_authorizations.uuid, api_client_authorizations.scopes, users.uuid FROM api_client_authorizations JOIN users on api_client_authorizations.user_id=users.id WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp AT TIME ZONE 'UTC') LIMIT 1`, token).Scan(&user.Authorization.UUID, &scopes, &user.UUID)
if err == sql.ErrNoRows {
+ ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): not found in database", token)
return nil, false, nil
} else if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): database error", token)
return nil, false, err
}
if uuid != "" && user.Authorization.UUID != uuid {
// secret part matches, but UUID doesn't -- somewhat surprising
+ ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): secret part found, but with different UUID: %s", token, user.Authorization.UUID)
return nil, false, nil
}
err = json.Unmarshal([]byte(scopes), &user.Authorization.Scopes)
if err != nil {
+ ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): error parsing scopes from db", token)
return nil, false, err
}
+ ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): ok", token)
return &user, true, nil
}
func (h *Handler) createAPItoken(req *http.Request, userUUID string, scopes []string) (*arvados.APIClientAuthorization, error) {
- db, err := h.db(req)
+ db, err := h.db(req.Context())
if err != nil {
return nil, err
}
(uuid, api_token, expires_at, scopes,
user_id,
api_client_id, created_at, updated_at)
-VALUES ($1, $2, CURRENT_TIMESTAMP + INTERVAL '2 weeks', $3,
+VALUES ($1, $2, CURRENT_TIMESTAMP AT TIME ZONE 'UTC' + INTERVAL '2 weeks', $3,
(SELECT id FROM users WHERE users.uuid=$4 LIMIT 1),
-0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
+0, CURRENT_TIMESTAMP AT TIME ZONE 'UTC', CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`,
uuid, token, string(scopesjson), userUUID)
if err != nil {
func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend backend) {
srv := httpserver.Server{Addr: ":"}
- srv.Handler = router.New(backend)
+ srv.Handler = router.New(backend, nil)
c.Check(srv.Start(), check.IsNil)
s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
Scheme: "http",
func (s *LoginSuite) TestLogout(c *check.C) {
s.cluster.Services.Workbench1.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench1.example.com"}
s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench2.example.com"}
- s.cluster.Login.GoogleClientID = "zzzzzzzzzzzzzz"
+ s.cluster.Login.Google.Enable = true
+ s.cluster.Login.Google.ClientID = "zzzzzzzzzzzzzz"
s.addHTTPRemote(c, "zhome", &arvadostest.APIStub{})
s.cluster.Login.LoginCluster = "zhome"
// s.fed is already set by SetUpTest, but we need to
cluster.TLS.Insecure = true
cluster.API.MaxItemsPerResponse = 1000
cluster.API.MaxRequestAmplification = 4
+ cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
import (
"context"
- "database/sql"
"errors"
"fmt"
"net/http"
"git.arvados.org/arvados.git/lib/controller/federation"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/router"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
+ "github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
)
proxy *proxy
secureClient *http.Client
insecureClient *http.Client
- pgdb *sql.DB
+ pgdb *sqlx.DB
pgdbMtx sync.Mutex
}
func (h *Handler) CheckHealth() error {
h.setupOnce.Do(h.setup)
- _, _, err := railsproxy.FindRailsAPI(h.Cluster)
+ _, err := h.db(context.TODO())
+ if err != nil {
+ return err
+ }
+ _, _, err = railsproxy.FindRailsAPI(h.Cluster)
return err
}
mux.Handle("/_health/", &health.Handler{
Token: h.Cluster.ManagementToken,
Prefix: "/_health/",
- Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
+ Routes: health.Routes{"ping": func() error { _, err := h.db(context.TODO()); return err }},
})
- rtr := router.New(federation.New(h.Cluster))
+ rtr := router.New(federation.New(h.Cluster), ctrlctx.WrapCallsInTransactions(h.db))
mux.Handle("/arvados/v1/config", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr)
var errDBConnection = errors.New("database connection error")
-func (h *Handler) db(req *http.Request) (*sql.DB, error) {
+func (h *Handler) db(ctx context.Context) (*sqlx.DB, error) {
h.pgdbMtx.Lock()
defer h.pgdbMtx.Unlock()
if h.pgdb != nil {
return h.pgdb, nil
}
- db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
+ db, err := sqlx.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
if err != nil {
- httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
+ ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect failed")
return nil, errDBConnection
}
if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
db.SetMaxOpenConns(p)
}
if err := db.Ping(); err != nil {
- httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
+ ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect scuceeded but ping failed")
return nil, errDBConnection
}
h.pgdb = db
PostgreSQL: integrationTestCluster().PostgreSQL,
ForceLegacyAPI14: forceLegacyAPI14,
}
+ s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
req := httptest.NewRequest(method, "/arvados/v1/config", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Log(resp.Body.String())
+ if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+ continue
+ }
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
continue
}
var cluster arvados.Cluster
- c.Log(resp.Body.String())
err := json.Unmarshal(resp.Body.Bytes(), &cluster)
c.Check(err, check.IsNil)
c.Check(cluster.ManagementToken, check.Equals, "")
c.Check(cluster.SystemRootToken, check.Equals, "")
- c.Check(cluster.Collections.BlobSigning, check.DeepEquals, true)
+ c.Check(cluster.Collections.BlobSigning, check.Equals, true)
c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
}
}
}
func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
- s.cluster.Login.ProviderAppID = "test"
- s.cluster.Login.ProviderAppSecret = "test"
+ s.cluster.Login.SSO.Enable = true
+ s.cluster.Login.SSO.ProviderAppID = "test"
+ s.cluster.Login.SSO.ProviderAppSecret = "test"
req := httptest.NewRequest("GET", "https://0.0.0.0:1/login?return_to=foo", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
}
func (s *HandlerSuite) TestLogoutSSO(c *check.C) {
- s.cluster.Login.ProviderAppID = "test"
+ s.cluster.Login.SSO.Enable = true
+ s.cluster.Login.SSO.ProviderAppID = "test"
req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
// Google login N/A
return
}
- s.cluster.Login.GoogleClientID = "test"
+ s.cluster.Login.Google.Enable = true
+ s.cluster.Login.Google.ClientID = "test"
req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
import (
"bytes"
"context"
+ "encoding/json"
"io"
"math"
"net"
+ "net/http"
"net/url"
"os"
"path/filepath"
Insecure: true
Proxy: true
ActivateUsers: true
- z2222:
+`
+ if id != "z2222" {
+ yaml += ` z2222:
Host: ` + hostport["z2222"] + `
Scheme: https
Insecure: true
Proxy: true
ActivateUsers: true
- z3333:
+`
+ }
+ if id != "z3333" {
+ yaml += ` z3333:
Host: ` + hostport["z3333"] + `
Scheme: https
Insecure: true
Proxy: true
ActivateUsers: true
`
+ }
+
loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
loader.Path = "-"
loader.SkipLegacy = true
},
config: *cfg,
}
- s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config)
+ s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config, "-")
}
for _, tc := range s.testClusters {
au, ok := tc.super.WaitReady()
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+// Get a token from the login cluster (z1111), use it to submit a
+// container request on z2222.
+func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ _, ac1, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+
+ // Use ac2 to get the discovery doc with a blank token, so the
+ // SDK doesn't magically pass the z1111 token to z2222 before
+ // we're ready to start our test.
+ _, ac2, _ := s.clientsWithToken("z2222", "")
+ var dd map[string]interface{}
+ err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
+ c.Assert(err, check.IsNil)
+
+ var (
+ body bytes.Buffer
+ req *http.Request
+ resp *http.Response
+ u arvados.User
+ cr arvados.ContainerRequest
+ )
+ json.NewEncoder(&body).Encode(map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "command": []string{"echo"},
+ "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
+ "cwd": "/",
+ "output_path": "/",
+ },
+ })
+ ac2.AuthToken = ac1.AuthToken
+
+ c.Log("...post CR with good (but not yet cached) token")
+ cr = arvados.ContainerRequest{}
+ req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Content-Type", "application/json")
+ err = ac2.DoAndDecode(&cr, req)
+ c.Logf("err == %#v", err)
+
+ c.Log("...get user with good token")
+ u = arvados.User{}
+ req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
+ c.Assert(err, check.IsNil)
+ err = ac2.DoAndDecode(&u, req)
+ c.Check(err, check.IsNil)
+ c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
+
+ c.Log("...post CR with good cached token")
+ cr = arvados.ContainerRequest{}
+ req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Content-Type", "application/json")
+ err = ac2.DoAndDecode(&cr, req)
+ c.Check(err, check.IsNil)
+ c.Check(cr.UUID, check.Matches, "z2222-.*")
+
+ c.Log("...post with good cached token ('OAuth2 ...')")
+ cr = arvados.ContainerRequest{}
+ req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
+ resp, err = arvados.InsecureHTTPClient.Do(req)
+ if c.Check(err, check.IsNil) {
+ err = json.NewDecoder(resp.Body).Decode(&cr)
+ c.Check(cr.UUID, check.Matches, "z2222-.*")
+ }
+}
+
// Test for bug #16263
func (s *IntegrationSuite) TestListUsers(c *check.C) {
rootctx1, _, _ := s.rootClients("z1111")
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
+ userctx1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
// Make sure LoginCluster is properly configured
for cls := range s.testClusters {
check.Commentf("incorrect LoginCluster config on cluster %q", cls))
}
// Make sure z1111 has users with NULL usernames
- lst, err := conn1.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+ lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
+ Limit: math.MaxInt64, // check that large limit works (see #16263)
+ })
nullUsername := false
c.Assert(err, check.IsNil)
c.Assert(len(lst.Items), check.Not(check.Equals), 0)
}
}
c.Assert(nullUsername, check.Equals, true)
+
+ user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
+ c.Assert(err, check.IsNil)
+ c.Check(user1.IsActive, check.Equals, true)
+
// Ask for the user list on z3333 using z1111's system root token
- _, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
- c.Assert(err, check.IsNil, check.Commentf("getting user list: %q", err))
-}
+ lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+ c.Assert(err, check.IsNil)
+ found := false
+ for _, user := range lst.Items {
+ if user.UUID == user1.UUID {
+ c.Check(user.IsActive, check.Equals, true)
+ found = true
+ break
+ }
+ }
+ c.Check(found, check.Equals, true)
-// Test for bug #16263
-func (s *IntegrationSuite) TestListUsersWithMaxLimit(c *check.C) {
- rootctx1, _, _ := s.rootClients("z1111")
- conn3 := s.conn("z3333")
- maxLimit := int64(math.MaxInt64)
+ // Deactivate user acct on z1111
+ _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
+ c.Assert(err, check.IsNil)
- // Make sure LoginCluster is properly configured
- for cls := range s.testClusters {
- c.Check(
- s.testClusters[cls].config.Clusters[cls].Login.LoginCluster,
- check.Equals, "z1111",
- check.Commentf("incorrect LoginCluster config on cluster %q", cls))
+ // Get user list from z3333, check the returned z1111 user is
+ // deactivated
+ lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+ c.Assert(err, check.IsNil)
+ found = false
+ for _, user := range lst.Items {
+ if user.UUID == user1.UUID {
+ c.Check(user.IsActive, check.Equals, false)
+ found = true
+ break
+ }
}
+ c.Check(found, check.Equals, true)
- // Ask for the user list on z3333 using z1111's system root token and
- // limit: max int64 value.
- _, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: maxLimit})
- c.Assert(err, check.IsNil, check.Commentf("getting user list: %q", err))
+ // Deactivated user can see is_active==false via "get current
+ // user" API
+ user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
+ c.Assert(err, check.IsNil)
+ c.Check(user1.IsActive, check.Equals, false)
}
func NewConn(cluster *arvados.Cluster) *Conn {
railsProxy := railsproxy.NewConn(cluster)
- return &Conn{
+ var conn Conn
+ conn = Conn{
cluster: cluster,
railsProxy: railsProxy,
loginController: chooseLoginController(cluster, railsProxy),
}
+ return &conn
}
func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "io"
+ "net"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ check "gopkg.in/check.v1"
+)
+
+type pgproxy struct {
+ net.Listener
+}
+
+// newPgProxy sets up a TCP proxy, listening on all interfaces, that
+// forwards all connections to the cluster's PostgreSQL server. This
+// allows the caller to run a docker container that can connect to a
+// postgresql instance that listens on the test host's loopback
+// interface.
+//
+// Caller is responsible for calling Close() on the returned pgproxy.
+func newPgProxy(c *check.C, cluster *arvados.Cluster) *pgproxy {
+ host := cluster.PostgreSQL.Connection["host"]
+ if host == "" {
+ host = "localhost"
+ }
+ port := cluster.PostgreSQL.Connection["port"]
+ if port == "" {
+ port = "5432"
+ }
+ target := net.JoinHostPort(host, port)
+
+ ln, err := net.Listen("tcp", ":")
+ c.Assert(err, check.IsNil)
+ go func() {
+ for {
+ downstream, err := ln.Accept()
+ if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
+ return
+ }
+ c.Assert(err, check.IsNil)
+ go func() {
+ c.Logf("pgproxy accepted connection from %s", downstream.RemoteAddr().String())
+ defer downstream.Close()
+ upstream, err := net.Dial("tcp", target)
+ if err != nil {
+ c.Logf("net.Dial(%q): %s", target, err)
+ return
+ }
+ defer upstream.Close()
+ go io.Copy(downstream, upstream)
+ io.Copy(upstream, downstream)
+ }()
+ }
+ }()
+ c.Logf("pgproxy listening at %s", ln.Addr().String())
+ return &pgproxy{Listener: ln}
+}
+
+func (proxy *pgproxy) Port() string {
+ _, port, _ := net.SplitHostPort(proxy.Addr().String())
+ return port
+}
import (
"context"
+ "database/sql"
+ "encoding/json"
"errors"
+ "fmt"
"net/http"
+ "net/url"
+ "strings"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
}
func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) loginController {
- wantGoogle := cluster.Login.GoogleClientID != ""
- wantSSO := cluster.Login.ProviderAppID != ""
- wantPAM := cluster.Login.PAM
+ wantGoogle := cluster.Login.Google.Enable
+ wantOpenIDConnect := cluster.Login.OpenIDConnect.Enable
+ wantSSO := cluster.Login.SSO.Enable
+ wantPAM := cluster.Login.PAM.Enable
+ wantLDAP := cluster.Login.LDAP.Enable
switch {
- case wantGoogle && !wantSSO && !wantPAM:
- return &googleLoginController{Cluster: cluster, RailsProxy: railsProxy}
- case !wantGoogle && wantSSO && !wantPAM:
+ case wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+ return &oidcLoginController{
+ Cluster: cluster,
+ RailsProxy: railsProxy,
+ Issuer: "https://accounts.google.com",
+ ClientID: cluster.Login.Google.ClientID,
+ ClientSecret: cluster.Login.Google.ClientSecret,
+ UseGooglePeopleAPI: cluster.Login.Google.AlternateEmailAddresses,
+ EmailClaim: "email",
+ EmailVerifiedClaim: "email_verified",
+ }
+ case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+ return &oidcLoginController{
+ Cluster: cluster,
+ RailsProxy: railsProxy,
+ Issuer: cluster.Login.OpenIDConnect.Issuer,
+ ClientID: cluster.Login.OpenIDConnect.ClientID,
+ ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
+ EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
+ UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
+ }
+ case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP:
return &ssoLoginController{railsProxy}
- case !wantGoogle && !wantSSO && wantPAM:
+ case !wantGoogle && !wantOpenIDConnect && !wantSSO && wantPAM && !wantLDAP:
return &pamLoginController{Cluster: cluster, RailsProxy: railsProxy}
+ case !wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && wantLDAP:
+ return &ldapLoginController{Cluster: cluster, RailsProxy: railsProxy}
default:
return errorLoginController{
- error: errors.New("configuration problem: exactly one of Login.GoogleClientID, Login.ProviderAppID, or Login.PAM must be configured"),
+ error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, and Login.LDAP must be enabled"),
}
}
}
}
return arvados.LogoutResponse{RedirectLocation: target}, nil
}
+
+func createAPIClientAuthorization(ctx context.Context, conn *rpc.Conn, rootToken string, authinfo rpc.UserSessionAuthInfo) (resp arvados.APIClientAuthorization, err error) {
+ ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{rootToken}})
+ newsession, err := conn.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{
+ // Send a fake ReturnTo value instead of the caller's
+ // opts.ReturnTo. We won't follow the resulting
+ // redirect target anyway.
+ ReturnTo: ",https://none.invalid",
+ AuthInfo: authinfo,
+ })
+ if err != nil {
+ return
+ }
+ target, err := url.Parse(newsession.RedirectLocation)
+ if err != nil {
+ return
+ }
+ token := target.Query().Get("api_token")
+ tx, err := ctrlctx.CurrentTx(ctx)
+ if err != nil {
+ return
+ }
+ tokensecret := token
+ if strings.Contains(token, "/") {
+ tokenparts := strings.Split(token, "/")
+ if len(tokenparts) >= 3 {
+ tokensecret = tokenparts[2]
+ }
+ }
+ var exp sql.NullString
+ var scopes []byte
+ err = tx.QueryRowxContext(ctx, "select uuid, api_token, expires_at, scopes from api_client_authorizations where api_token=$1", tokensecret).Scan(&resp.UUID, &resp.APIToken, &exp, &scopes)
+ if err != nil {
+ return
+ }
+ resp.ExpiresAt = exp.String
+ if len(scopes) > 0 {
+ err = json.Unmarshal(scopes, &resp.Scopes)
+ if err != nil {
+ return resp, fmt.Errorf("unmarshal scopes: %s", err)
+ }
+ }
+ return
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
+ "github.com/go-ldap/ldap"
+)
+
+type ldapLoginController struct {
+ Cluster *arvados.Cluster
+ RailsProxy *railsProxy
+}
+
+func (ctrl *ldapLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+ return noopLogout(ctrl.Cluster, opts)
+}
+
+func (ctrl *ldapLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+ return arvados.LoginResponse{}, errors.New("interactive login is not available")
+}
+
+func (ctrl *ldapLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+ log := ctxlog.FromContext(ctx)
+ conf := ctrl.Cluster.Login.LDAP
+ errFailed := httpserver.ErrorWithStatus(fmt.Errorf("LDAP: Authentication failure (with username %q and password)", opts.Username), http.StatusUnauthorized)
+
+ if conf.SearchAttribute == "" {
+ return arvados.APIClientAuthorization{}, errors.New("config error: SearchAttribute is blank")
+ }
+ if opts.Password == "" {
+ log.WithField("username", opts.Username).Error("refusing to authenticate with empty password")
+ return arvados.APIClientAuthorization{}, errFailed
+ }
+
+ log = log.WithField("URL", conf.URL.String())
+ l, err := ldap.DialURL(conf.URL.String())
+ if err != nil {
+ log.WithError(err).Error("ldap connection failed")
+ return arvados.APIClientAuthorization{}, err
+ }
+ defer l.Close()
+
+ if conf.StartTLS {
+ var tlsconfig tls.Config
+ if conf.InsecureTLS {
+ tlsconfig.InsecureSkipVerify = true
+ } else {
+ if host, _, err := net.SplitHostPort(conf.URL.Host); err != nil {
+ // Assume SplitHostPort error means
+ // port was not specified
+ tlsconfig.ServerName = conf.URL.Host
+ } else {
+ tlsconfig.ServerName = host
+ }
+ }
+ err = l.StartTLS(&tlsconfig)
+ if err != nil {
+ log.WithError(err).Error("ldap starttls failed")
+ return arvados.APIClientAuthorization{}, err
+ }
+ }
+
+ username := opts.Username
+ if at := strings.Index(username, "@"); at >= 0 {
+ if conf.StripDomain == "*" || strings.ToLower(conf.StripDomain) == strings.ToLower(username[at+1:]) {
+ username = username[:at]
+ }
+ }
+ if conf.AppendDomain != "" && !strings.Contains(username, "@") {
+ username = username + "@" + conf.AppendDomain
+ }
+
+ if conf.SearchBindUser != "" {
+ err = l.Bind(conf.SearchBindUser, conf.SearchBindPassword)
+ if err != nil {
+ log.WithError(err).WithField("user", conf.SearchBindUser).Error("ldap authentication failed")
+ return arvados.APIClientAuthorization{}, err
+ }
+ }
+
+ search := fmt.Sprintf("(%s=%s)", ldap.EscapeFilter(conf.SearchAttribute), ldap.EscapeFilter(username))
+ if conf.SearchFilters != "" {
+ search = fmt.Sprintf("(&%s%s)", conf.SearchFilters, search)
+ }
+ log = log.WithField("search", search)
+ req := ldap.NewSearchRequest(
+ conf.SearchBase,
+ ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 1, 0, false,
+ search,
+ []string{"DN", "givenName", "SN", conf.EmailAttribute, conf.UsernameAttribute},
+ nil)
+ resp, err := l.Search(req)
+ if ldap.IsErrorWithCode(err, ldap.LDAPResultNoResultsReturned) ||
+ ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) ||
+ (err == nil && len(resp.Entries) == 0) {
+ log.WithError(err).Info("ldap lookup returned no results")
+ return arvados.APIClientAuthorization{}, errFailed
+ } else if err != nil {
+ log.WithError(err).Error("ldap lookup failed")
+ return arvados.APIClientAuthorization{}, err
+ }
+ userdn := resp.Entries[0].DN
+ if userdn == "" {
+ log.Warn("refusing to authenticate with empty dn")
+ return arvados.APIClientAuthorization{}, errFailed
+ }
+ log = log.WithField("DN", userdn)
+
+ attrs := map[string]string{}
+ for _, attr := range resp.Entries[0].Attributes {
+ if attr == nil || len(attr.Values) == 0 {
+ continue
+ }
+ attrs[strings.ToLower(attr.Name)] = attr.Values[0]
+ }
+ log.WithField("attrs", attrs).Debug("ldap search succeeded")
+
+ // Now that we have the DN, try authenticating.
+ err = l.Bind(userdn, opts.Password)
+ if err != nil {
+ log.WithError(err).Info("ldap user authentication failed")
+ return arvados.APIClientAuthorization{}, errFailed
+ }
+ log.Debug("ldap authentication succeeded")
+
+ email := attrs[strings.ToLower(conf.EmailAttribute)]
+ if email == "" {
+ log.Errorf("ldap returned no email address in %q attribute", conf.EmailAttribute)
+ return arvados.APIClientAuthorization{}, errors.New("authentication succeeded but ldap returned no email address")
+ }
+
+ return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{
+ Email: email,
+ FirstName: attrs["givenname"],
+ LastName: attrs["sn"],
+ Username: attrs[strings.ToLower(conf.UsernameAttribute)],
+ })
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "os"
+ "os/exec"
+ "testing"
+
+ check "gopkg.in/check.v1"
+)
+
+func haveDocker() bool {
+ _, err := exec.Command("docker", "info").CombinedOutput()
+ return err == nil
+}
+
+func (s *LDAPSuite) TestLoginLDAPViaPAM(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker test in short mode")
+ }
+ if !haveDocker() {
+ c.Skip("skipping docker test because docker is not available")
+ }
+ pgproxy := newPgProxy(c, s.cluster)
+ defer pgproxy.Close()
+
+ cmd := exec.Command("bash", "login_ldap_docker_test.sh")
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ cmd.Env = append(os.Environ(), "config_method=pam", "pgport="+pgproxy.Port())
+ err := cmd.Run()
+ c.Check(err, check.IsNil)
+}
+
+func (s *LDAPSuite) TestLoginLDAPBuiltin(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker test in short mode")
+ }
+ if !haveDocker() {
+ c.Skip("skipping docker test because docker is not available")
+ }
+ pgproxy := newPgProxy(c, s.cluster)
+ defer pgproxy.Close()
+
+ cmd := exec.Command("bash", "login_ldap_docker_test.sh")
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ cmd.Env = append(os.Environ(), "config_method=ldap", "pgport="+pgproxy.Port())
+ err := cmd.Run()
+ c.Check(err, check.IsNil)
+}
#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
# This script demonstrates using LDAP for Arvados user authentication.
#
-# It configures pam_ldap(5) and arvados controller in a docker
-# container, with pam_ldap configured to authenticate against an
-# OpenLDAP server in a second docker container.
+# It configures arvados controller in a docker container, optionally
+# with pam_ldap(5) configured to authenticate against an OpenLDAP
+# server in a second docker container.
#
# After adding a "foo" user entry, it uses curl to check that the
# Arvados controller's login endpoint accepts the "foo" account
set -x
fi
+case "${config_method}" in
+ pam | ldap)
+ ;;
+ *)
+ echo >&2 "\$config_method env var must be 'pam' or 'ldap'"
+ exit 1
+ ;;
+esac
+
hostname="$(hostname)"
tmpdir="$(mktemp -d)"
cleanup() {
Connection:
client_encoding: utf8
host: ${hostname}
+ port: "${pgport}"
dbname: arvados_test
user: arvados
password: insecure_arvados_test
ExternalURL: http://0.0.0.0:9999/
InternalURLs:
"http://0.0.0.0:9999/": {}
- Login:
- PAM: true
- # Without this magic PAMDefaultEmailDomain, inserted users would
- # prevent subsequent database/reset from working (see
- # database_controller.rb).
- PAMDefaultEmailDomain: example.com
SystemLogs:
LogLevel: debug
EOF
+case "${config_method}" in
+ pam)
+ setup_pam_ldap="apt update && DEBIAN_FRONTEND=noninteractive apt install -y ldap-utils libpam-ldap && pam-auth-update --package /usr/share/pam-configs/ldap"
+ cat >>"${tmpdir}/zzzzz.yml" <<EOF
+ Login:
+ PAM:
+ Enable: true
+ # Without this specific DefaultEmailDomain, inserted users
+ # would prevent subsequent database/reset from working (see
+ # database_controller.rb).
+ DefaultEmailDomain: example.com
+EOF
+ ;;
+ ldap)
+ setup_pam_ldap=""
+ cat >>"${tmpdir}/zzzzz.yml" <<EOF
+ Login:
+ LDAP:
+ Enable: true
+ URL: ${ldapurl}
+ StartTLS: false
+ SearchBase: dc=example,dc=org
+ SearchBindUser: cn=admin,dc=example,dc=org
+ SearchBindPassword: admin
+EOF
+ ;;
+esac
+
+cat >&2 "${tmpdir}/zzzzz.yml"
cat >"${tmpdir}/pam_ldap.conf" <<EOF
base dc=example,dc=org
gidNumber: 11111
description: "Example group 'bar'"
-dn: uid=foo,dc=example,dc=org
-uid: foo
-cn: foo
+dn: uid=foo-bar,dc=example,dc=org
+uid: foo-bar
+cn: "Foo Bar"
givenName: Foo
sn: Bar
-mail: foobar@example.org
+mail: foo-bar-baz@example.com
objectClass: inetOrgPerson
objectClass: posixAccount
objectClass: top
loginShell: /bin/bash
uidNumber: 11111
gidNumber: 11111
-homeDirectory: /home/foo
+homeDirectory: /home/foo-bar
userPassword: ${passwordhash}
EOF
-echo >&2 "Adding example user entry user=foo pass=secret (retrying until server comes up)"
+echo >&2 "Adding example user entry user=foo-bar pass=secret (retrying until server comes up)"
docker run --rm --entrypoint= \
-v "${tmpdir}/add_example_user.ldif":/add_example_user.ldif:ro \
osixia/openldap:1.3.0 \
-v "${tmpdir}/zzzzz.yml":/etc/arvados/config.yml:ro \
-v $(realpath "${PWD}/../../.."):/arvados:ro \
debian:10 \
- bash -c "apt update && DEBIAN_FRONTEND=noninteractive apt install -y ldap-utils libpam-ldap && pam-auth-update --package /usr/share/pam-configs/ldap && arvados-server controller"
+ bash -c "${setup_pam_ldap:-true} && arvados-server controller"
docker logs --follow ${ctrlctr} 2>$debug >$debug &
ctrlhostport=$(docker port ${ctrlctr} 9999/tcp)
fi
}
+set +x
+
echo >&2 "Testing authentication failure"
-resp="$(curl -s --include -d username=foo -d password=nosecret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
+resp="$(set -x; curl -s --include -d username=foo-bar -d password=nosecret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
check_contains "${resp}" "HTTP/1.1 401"
-check_contains "${resp}" '{"errors":["PAM: Authentication failure (with username \"foo\" and password)"]}'
+if [[ "${config_method}" = ldap ]]; then
+ check_contains "${resp}" '{"errors":["LDAP: Authentication failure (with username \"foo-bar\" and password)"]}'
+else
+ check_contains "${resp}" '{"errors":["PAM: Authentication failure (with username \"foo-bar\" and password)"]}'
+fi
echo >&2 "Testing authentication success"
-resp="$(curl -s --include -d username=foo -d password=secret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
+resp="$(set -x; curl -s --include -d username=foo-bar -d password=secret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
check_contains "${resp}" "HTTP/1.1 200"
check_contains "${resp}" '"api_token":"'
check_contains "${resp}" '"scopes":["all"]'
check_contains "${resp}" '"uuid":"zzzzz-gj3su-'
+secret="${resp##*api_token\":\"}"
+secret="${secret%%\"*}"
+uuid="${resp##*uuid\":\"}"
+uuid="${uuid%%\"*}"
+token="v2/$uuid/$secret"
+echo >&2 "New token is ${token}"
+
+resp="$(set -x; curl -s --include -H "Authorization: Bearer ${token}" "http://${ctrlhostport}/arvados/v1/users/current" | tee $debug)"
+check_contains "${resp}" "HTTP/1.1 200"
+if [[ "${config_method}" = ldap ]]; then
+ # user fields come from LDAP attributes
+ check_contains "${resp}" '"first_name":"Foo"'
+ check_contains "${resp}" '"last_name":"Bar"'
+ check_contains "${resp}" '"username":"foobar"' # "-" removed by rails api
+ check_contains "${resp}" '"email":"foo-bar-baz@example.com"'
+else
+ # PAMDefaultEmailDomain
+ check_contains "${resp}" '"email":"foo-bar@example.com"'
+fi
+
cleanup
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "net/http"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/railsproxy"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/bradleypeabody/godap"
+ "github.com/jmoiron/sqlx"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&LDAPSuite{})
+
+type LDAPSuite struct {
+ cluster *arvados.Cluster
+ ctrl *ldapLoginController
+ ldap *godap.LDAPServer // fake ldap server that accepts auth goodusername/goodpassword
+ db *sqlx.DB
+
+ // transaction context
+ ctx context.Context
+ rollback func() error
+}
+
+func (s *LDAPSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *LDAPSuite) SetUpSuite(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ s.ldap = &godap.LDAPServer{
+ Listener: ln,
+ Handlers: []godap.LDAPRequestHandler{
+ &godap.LDAPBindFuncHandler{
+ LDAPBindFunc: func(binddn string, bindpw []byte) bool {
+ return binddn == "cn=goodusername,dc=example,dc=com" && string(bindpw) == "goodpassword"
+ },
+ },
+ &godap.LDAPSimpleSearchFuncHandler{
+ LDAPSimpleSearchFunc: func(req *godap.LDAPSimpleSearchRequest) []*godap.LDAPSimpleSearchResultEntry {
+ if req.FilterAttr != "uid" || req.BaseDN != "dc=example,dc=com" {
+ return []*godap.LDAPSimpleSearchResultEntry{}
+ }
+ return []*godap.LDAPSimpleSearchResultEntry{
+ &godap.LDAPSimpleSearchResultEntry{
+ DN: "cn=" + req.FilterValue + "," + req.BaseDN,
+ Attrs: map[string]interface{}{
+ "SN": req.FilterValue,
+ "CN": req.FilterValue,
+ "uid": req.FilterValue,
+ "mail": req.FilterValue + "@example.com",
+ },
+ },
+ }
+ },
+ },
+ },
+ }
+ go func() {
+ ctxlog.TestLogger(c).Print(s.ldap.Serve())
+ }()
+
+ s.cluster.Login.LDAP.Enable = true
+ err = json.Unmarshal([]byte(`"ldap://`+ln.Addr().String()+`"`), &s.cluster.Login.LDAP.URL)
+ s.cluster.Login.LDAP.StartTLS = false
+ s.cluster.Login.LDAP.SearchBindUser = "cn=goodusername,dc=example,dc=com"
+ s.cluster.Login.LDAP.SearchBindPassword = "goodpassword"
+ s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
+ c.Assert(err, check.IsNil)
+ s.ctrl = &ldapLoginController{
+ Cluster: s.cluster,
+ RailsProxy: railsproxy.NewConn(s.cluster),
+ }
+ s.db = arvadostest.DB(c, s.cluster)
+}
+
+func (s *LDAPSuite) SetUpTest(c *check.C) {
+ tx, err := s.db.Beginx()
+ c.Assert(err, check.IsNil)
+ s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx)
+ s.rollback = tx.Rollback
+}
+
+func (s *LDAPSuite) TearDownTest(c *check.C) {
+ if s.rollback != nil {
+ s.rollback()
+ }
+}
+
+func (s *LDAPSuite) TestLoginSuccess(c *check.C) {
+ conn := NewConn(s.cluster)
+ conn.loginController = s.ctrl
+ resp, err := conn.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ Username: "goodusername",
+ Password: "goodpassword",
+ })
+ c.Check(err, check.IsNil)
+ c.Check(resp.APIToken, check.Not(check.Equals), "")
+ c.Check(resp.UUID, check.Matches, `zzzzz-gj3su-.*`)
+ c.Check(resp.Scopes, check.DeepEquals, []string{"all"})
+
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{"v2/" + resp.UUID + "/" + resp.APIToken}})
+ user, err := railsproxy.NewConn(s.cluster).UserGetCurrent(ctx, arvados.GetOptions{})
+ c.Check(err, check.IsNil)
+ c.Check(user.Email, check.Equals, "goodusername@example.com")
+ c.Check(user.Username, check.Equals, "goodusername")
+}
+
+func (s *LDAPSuite) TestLoginFailure(c *check.C) {
+ // search returns no results
+ s.cluster.Login.LDAP.SearchBase = "dc=example,dc=invalid"
+ resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ Username: "goodusername",
+ Password: "goodpassword",
+ })
+ c.Check(err, check.ErrorMatches, `LDAP: Authentication failure \(with username "goodusername" and password\)`)
+ hs, ok := err.(interface{ HTTPStatus() int })
+ if c.Check(ok, check.Equals, true) {
+ c.Check(hs.HTTPStatus(), check.Equals, http.StatusUnauthorized)
+ }
+ c.Check(resp.APIToken, check.Equals, "")
+
+ // search returns result, but auth fails
+ s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
+ resp, err = s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ Username: "badusername",
+ Password: "badpassword",
+ })
+ c.Check(err, check.ErrorMatches, `LDAP: Authentication failure \(with username "badusername" and password\)`)
+ hs, ok = err.(interface{ HTTPStatus() int })
+ if c.Check(ok, check.Equals, true) {
+ c.Check(hs.HTTPStatus(), check.Equals, http.StatusUnauthorized)
+ }
+ c.Check(resp.APIToken, check.Equals, "")
+}
"google.golang.org/api/people/v1"
)
-type googleLoginController struct {
- Cluster *arvados.Cluster
- RailsProxy *railsProxy
+type oidcLoginController struct {
+ Cluster *arvados.Cluster
+ RailsProxy *railsProxy
+ Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
+ ClientID string
+ ClientSecret string
+ UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
+ EmailClaim string // OpenID claim to use as email address; typically "email"
+ EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
+ UsernameClaim string // If non-empty, use as preferred username
- issuer string // override OIDC issuer URL (normally https://accounts.google.com) for testing
- peopleAPIBasePath string // override Google People API base URL (normally set by google pkg to https://people.googleapis.com/)
- provider *oidc.Provider
- mu sync.Mutex
+ // override Google People API base URL for testing purposes
+ // (normally empty, set by google pkg to
+ // https://people.googleapis.com/)
+ peopleAPIBasePath string
+
+ provider *oidc.Provider // initialized by setup()
+ oauth2conf *oauth2.Config // initialized by setup()
+ verifier *oidc.IDTokenVerifier // initialized by setup()
+ mu sync.Mutex // protects setup()
}
-func (ctrl *googleLoginController) getProvider() (*oidc.Provider, error) {
+// Initialize ctrl.provider and ctrl.oauth2conf.
+func (ctrl *oidcLoginController) setup() error {
ctrl.mu.Lock()
defer ctrl.mu.Unlock()
- if ctrl.provider == nil {
- issuer := ctrl.issuer
- if issuer == "" {
- issuer = "https://accounts.google.com"
- }
- provider, err := oidc.NewProvider(context.Background(), issuer)
- if err != nil {
- return nil, err
- }
- ctrl.provider = provider
+ if ctrl.provider != nil {
+ // already set up
+ return nil
}
- return ctrl.provider, nil
-}
-
-func (ctrl *googleLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- return noopLogout(ctrl.Cluster, opts)
-}
-
-func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
- provider, err := ctrl.getProvider()
+ redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/" + arvados.EndpointLogin.Path)
if err != nil {
- return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+ return fmt.Errorf("error making redirect URL: %s", err)
}
- redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/login")
+ provider, err := oidc.NewProvider(context.Background(), ctrl.Issuer)
if err != nil {
- return loginError(fmt.Errorf("error making redirect URL: %s", err))
+ return err
}
- conf := &oauth2.Config{
- ClientID: ctrl.Cluster.Login.GoogleClientID,
- ClientSecret: ctrl.Cluster.Login.GoogleClientSecret,
+ ctrl.oauth2conf = &oauth2.Config{
+ ClientID: ctrl.ClientID,
+ ClientSecret: ctrl.ClientSecret,
Endpoint: provider.Endpoint(),
Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
RedirectURL: redirURL.String(),
}
- verifier := provider.Verifier(&oidc.Config{
- ClientID: conf.ClientID,
+ ctrl.verifier = provider.Verifier(&oidc.Config{
+ ClientID: ctrl.ClientID,
})
+ ctrl.provider = provider
+ return nil
+}
+
+func (ctrl *oidcLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+ return noopLogout(ctrl.Cluster, opts)
+}
+
+func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+ err := ctrl.setup()
+ if err != nil {
+ return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+ }
if opts.State == "" {
- // Initiate Google sign-in.
+ // Initiate OIDC sign-in.
if opts.ReturnTo == "" {
return loginError(errors.New("missing return_to parameter"))
}
- me := url.URL(ctrl.Cluster.Services.Controller.ExternalURL)
- callback, err := me.Parse("/" + arvados.EndpointLogin.Path)
- if err != nil {
- return loginError(err)
- }
- conf.RedirectURL = callback.String()
state := ctrl.newOAuth2State([]byte(ctrl.Cluster.SystemRootToken), opts.Remote, opts.ReturnTo)
return arvados.LoginResponse{
- RedirectLocation: conf.AuthCodeURL(state.String(),
+ RedirectLocation: ctrl.oauth2conf.AuthCodeURL(state.String(),
// prompt=select_account tells Google
// to show the "choose which Google
// account" page, even if the client
oauth2.SetAuthURLParam("prompt", "select_account")),
}, nil
} else {
- // Callback after Google sign-in.
+ // Callback after OIDC sign-in.
state := ctrl.parseOAuth2State(opts.State)
if !state.verify([]byte(ctrl.Cluster.SystemRootToken)) {
return loginError(errors.New("invalid OAuth2 state"))
}
- oauth2Token, err := conf.Exchange(ctx, opts.Code)
+ oauth2Token, err := ctrl.oauth2conf.Exchange(ctx, opts.Code)
if err != nil {
return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err))
}
if !ok {
return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
}
- idToken, err := verifier.Verify(ctx, rawIDToken)
+ idToken, err := ctrl.verifier.Verify(ctx, rawIDToken)
if err != nil {
return loginError(fmt.Errorf("error verifying ID token: %s", err))
}
- authinfo, err := ctrl.getAuthInfo(ctx, ctrl.Cluster, conf, oauth2Token, idToken)
+ authinfo, err := ctrl.getAuthInfo(ctx, oauth2Token, idToken)
if err != nil {
return loginError(err)
}
}
}
-func (ctrl *googleLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+func (ctrl *oidcLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest)
}
// primary address at index 0. The provided defaultAddr is always
// included in the returned slice, and is used as the primary if the
// Google API does not indicate one.
-func (ctrl *googleLoginController) getAuthInfo(ctx context.Context, cluster *arvados.Cluster, conf *oauth2.Config, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
+func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
var ret rpc.UserSessionAuthInfo
defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned")
- var claims struct {
- Name string `json:"name"`
- Email string `json:"email"`
- Verified bool `json:"email_verified"`
- }
+ var claims map[string]interface{}
if err := idToken.Claims(&claims); err != nil {
return nil, fmt.Errorf("error extracting claims from ID token: %s", err)
- } else if claims.Verified {
+ } else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" {
// Fall back to this info if the People API call
// (below) doesn't return a primary && verified email.
- if names := strings.Fields(strings.TrimSpace(claims.Name)); len(names) > 1 {
+ name, _ := claims["name"].(string)
+ if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
ret.FirstName = strings.Join(names[0:len(names)-1], " ")
ret.LastName = names[len(names)-1]
} else {
ret.FirstName = names[0]
}
- ret.Email = claims.Email
+ ret.Email, _ = claims[ctrl.EmailClaim].(string)
+ }
+
+ if ctrl.UsernameClaim != "" {
+ ret.Username, _ = claims[ctrl.UsernameClaim].(string)
}
- if !ctrl.Cluster.Login.GoogleAlternateEmailAddresses {
+ if !ctrl.UseGooglePeopleAPI {
if ret.Email == "" {
- return nil, fmt.Errorf("cannot log in with unverified email address %q", claims.Email)
+ return nil, fmt.Errorf("cannot log in with unverified email address %q", claims[ctrl.EmailClaim])
}
return &ret, nil
}
- svc, err := people.NewService(ctx, option.WithTokenSource(conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
+ svc, err := people.NewService(ctx, option.WithTokenSource(ctrl.oauth2conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
if err != nil {
return nil, fmt.Errorf("error setting up People API: %s", err)
}
return nil, errors.New("cannot log in without a verified email address")
}
for ae := range altEmails {
- if ae != ret.Email {
- ret.AlternateEmails = append(ret.AlternateEmails, ae)
- if i := strings.Index(ae, "@"); i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
+ if ae == ret.Email {
+ continue
+ }
+ ret.AlternateEmails = append(ret.AlternateEmails, ae)
+ if ret.Username == "" {
+ i := strings.Index(ae, "@")
+ if i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
ret.Username = strings.SplitN(ae[:i], "+", 2)[0]
}
}
return
}
-func (ctrl *googleLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
+func (ctrl *oidcLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
s := oauth2State{
Time: time.Now().Unix(),
Remote: remote,
ReturnTo string // redirect target
}
-func (ctrl *googleLoginController) parseOAuth2State(encoded string) (s oauth2State) {
+func (ctrl *oidcLoginController) parseOAuth2State(encoded string) (s oauth2State) {
// Errors are not checked. If decoding/parsing fails, the
// token will be rejected by verify().
decoded, _ := base64.RawURLEncoding.DecodeString(encoded)
"context"
"crypto/rand"
"crypto/rsa"
+ "encoding/base64"
"encoding/json"
"fmt"
"net/http"
check.TestingT(t)
}
-var _ = check.Suite(&LoginSuite{})
+var _ = check.Suite(&OIDCLoginSuite{})
-type LoginSuite struct {
+type OIDCLoginSuite struct {
cluster *arvados.Cluster
- ctx context.Context
localdb *Conn
railsSpy *arvadostest.Proxy
fakeIssuer *httptest.Server
issuerKey *rsa.PrivateKey
// expected token request
- validCode string
+ validCode string
+ validClientID string
+ validClientSecret string
// desired response from token endpoint
authEmail string
authEmailVerified bool
authName string
}
-func (s *LoginSuite) TearDownSuite(c *check.C) {
+func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
// Undo any changes/additions to the user database so they
// don't affect subsequent tests.
arvadostest.ResetEnv()
c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
}
-func (s *LoginSuite) SetUpTest(c *check.C) {
+func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
var err error
s.issuerKey, err = rsa.GenerateKey(rand.Reader, 2048)
c.Assert(err, check.IsNil)
"userinfo_endpoint": s.fakeIssuer.URL + "/userinfo",
})
case "/token":
+ var clientID, clientSecret string
+ auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
+ authsplit := strings.Split(string(auth), ":")
+ if len(authsplit) == 2 {
+ clientID, _ = url.QueryUnescape(authsplit[0])
+ clientSecret, _ = url.QueryUnescape(authsplit[1])
+ }
+ if clientID != s.validClientID || clientSecret != s.validClientSecret {
+ c.Logf("fakeIssuer: expected (%q, %q) got (%q, %q)", s.validClientID, s.validClientSecret, clientID, clientSecret)
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
if req.Form.Get("code") != s.validCode || s.validCode == "" {
w.WriteHeader(http.StatusUnauthorized)
return
}
idToken, _ := json.Marshal(map[string]interface{}{
"iss": s.fakeIssuer.URL,
- "aud": []string{"test%client$id"},
+ "aud": []string{clientID},
"sub": "fake-user-id",
- "exp": time.Now().UTC().Add(time.Minute).UnixNano(),
- "iat": time.Now().UTC().UnixNano(),
+ "exp": time.Now().UTC().Add(time.Minute).Unix(),
+ "iat": time.Now().UTC().Unix(),
"nonce": "fake-nonce",
"email": s.authEmail,
"email_verified": s.authEmailVerified,
"name": s.authName,
+ "alt_verified": true, // for custom claim tests
+ "alt_email": "alt_email@example.com", // for custom claim tests
+ "alt_username": "desired-username", // for custom claim tests
})
json.NewEncoder(w).Encode(struct {
AccessToken string `json:"access_token"`
s.fakePeopleAPIResponse = map[string]interface{}{}
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
- s.cluster.Login.ProviderAppID = ""
- s.cluster.Login.ProviderAppSecret = ""
- s.cluster.Login.GoogleClientID = "test%client$id"
- s.cluster.Login.GoogleClientSecret = "test#client/secret"
- s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
c.Assert(err, check.IsNil)
+ s.cluster.Login.SSO.Enable = false
+ s.cluster.Login.Google.Enable = true
+ s.cluster.Login.Google.ClientID = "test%client$id"
+ s.cluster.Login.Google.ClientSecret = "test#client/secret"
+ s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
+ s.validClientID = "test%client$id"
+ s.validClientSecret = "test#client/secret"
s.localdb = NewConn(s.cluster)
- s.localdb.loginController.(*googleLoginController).issuer = s.fakeIssuer.URL
- s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
+ s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeIssuer.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
-func (s *LoginSuite) TearDownTest(c *check.C) {
+func (s *OIDCLoginSuite) TearDownTest(c *check.C) {
s.railsSpy.Close()
}
-func (s *LoginSuite) TestGoogleLogout(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
resp, err := s.localdb.Logout(context.Background(), arvados.LogoutOptions{ReturnTo: "https://foo.example.com/bar"})
c.Check(err, check.IsNil)
c.Check(resp.RedirectLocation, check.Equals, "https://foo.example.com/bar")
}
-func (s *LoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{})
c.Check(err, check.IsNil)
c.Check(resp.RedirectLocation, check.Equals, "")
c.Check(resp.HTML.String(), check.Matches, `.*missing return_to parameter.*`)
}
-func (s *LoginSuite) TestGoogleLogin_Start(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start(c *check.C) {
for _, remote := range []string{"", "zzzzz"} {
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{Remote: remote, ReturnTo: "https://app.example.com/foo?bar"})
c.Check(err, check.IsNil)
c.Check(target.Host, check.Equals, issuerURL.Host)
q := target.Query()
c.Check(q.Get("client_id"), check.Equals, "test%client$id")
- state := s.localdb.loginController.(*googleLoginController).parseOAuth2State(q.Get("state"))
+ state := s.localdb.loginController.(*oidcLoginController).parseOAuth2State(q.Get("state"))
c.Check(state.verify([]byte(s.cluster.SystemRootToken)), check.Equals, true)
c.Check(state.Time, check.Not(check.Equals), 0)
c.Check(state.Remote, check.Equals, remote)
}
}
-func (s *LoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: "first-try-a-bogus-code",
c.Check(resp.HTML.String(), check.Matches, `(?ms).*error in OAuth2 exchange.*cannot fetch token.*`)
}
-func (s *LoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: s.validCode,
c.Check(resp.HTML.String(), check.Matches, `(?ms).*invalid OAuth2 state.*`)
}
-func (s *LoginSuite) setupPeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) {
s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprintln(w, `Error 403: accessNotConfigured`)
}))
- s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
}
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
- s.cluster.Login.GoogleAlternateEmailAddresses = false
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
+ s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false
s.authEmail = "joe.smith@primary.example.com"
s.setupPeopleAPIError(c)
state := s.startLogin(c)
c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
}
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) TestConfig(c *check.C) {
+ s.cluster.Login.Google.Enable = false
+ s.cluster.Login.OpenIDConnect.Enable = true
+ s.cluster.Login.OpenIDConnect.Issuer = "https://accounts.example.com/"
+ s.cluster.Login.OpenIDConnect.ClientID = "oidc-client-id"
+ s.cluster.Login.OpenIDConnect.ClientSecret = "oidc-client-secret"
+ localdb := NewConn(s.cluster)
+ ctrl := localdb.loginController.(*oidcLoginController)
+ c.Check(ctrl.Issuer, check.Equals, "https://accounts.example.com/")
+ c.Check(ctrl.ClientID, check.Equals, "oidc-client-id")
+ c.Check(ctrl.ClientSecret, check.Equals, "oidc-client-secret")
+ c.Check(ctrl.UseGooglePeopleAPI, check.Equals, false)
+
+ for _, enableAltEmails := range []bool{false, true} {
+ s.cluster.Login.OpenIDConnect.Enable = false
+ s.cluster.Login.Google.Enable = true
+ s.cluster.Login.Google.ClientID = "google-client-id"
+ s.cluster.Login.Google.ClientSecret = "google-client-secret"
+ s.cluster.Login.Google.AlternateEmailAddresses = enableAltEmails
+ localdb = NewConn(s.cluster)
+ ctrl = localdb.loginController.(*oidcLoginController)
+ c.Check(ctrl.Issuer, check.Equals, "https://accounts.google.com")
+ c.Check(ctrl.ClientID, check.Equals, "google-client-id")
+ c.Check(ctrl.ClientSecret, check.Equals, "google-client-secret")
+ c.Check(ctrl.UseGooglePeopleAPI, check.Equals, enableAltEmails)
+ }
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
s.setupPeopleAPIError(c)
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
c.Check(resp.RedirectLocation, check.Equals, "")
}
-func (s *LoginSuite) TestGoogleLogin_Success(c *check.C) {
+func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
+ s.cluster.Login.Google.Enable = false
+ s.cluster.Login.OpenIDConnect.Enable = true
+ json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
+ s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
+ s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+ s.validClientID = "oidc#client#id"
+ s.validClientSecret = "oidc#client#secret"
+ for _, trial := range []struct {
+ expectEmail string // "" if failure expected
+ setup func()
+ }{
+ {
+ expectEmail: "user@oidc.example.com",
+ setup: func() {
+ c.Log("=== succeed because email_verified is false but not required")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = ""
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "",
+ setup: func() {
+ c.Log("=== fail because email_verified is false and required")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "user@oidc.example.com",
+ setup: func() {
+ c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "alt_email@example.com",
+ setup: func() {
+ c.Log("=== succeed with custom 'email' and 'email_verified' claims")
+ s.authEmail = "bad@wrong.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username"
+ },
+ },
+ } {
+ trial.setup()
+ if s.railsSpy != nil {
+ s.railsSpy.Close()
+ }
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ s.localdb = NewConn(s.cluster)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+
+ state := s.startLogin(c)
+ resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
+ Code: s.validCode,
+ State: state,
+ })
+ c.Assert(err, check.IsNil)
+ if trial.expectEmail == "" {
+ c.Check(resp.HTML.String(), check.Matches, `(?ms).*Login error.*`)
+ c.Check(resp.RedirectLocation, check.Equals, "")
+ continue
+ }
+ c.Check(resp.HTML.String(), check.Equals, "")
+ target, err := url.Parse(resp.RedirectLocation)
+ c.Assert(err, check.IsNil)
+ token := target.Query().Get("api_token")
+ c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
+ authinfo := getCallbackAuthInfo(c, s.railsSpy)
+ c.Check(authinfo.Email, check.Equals, trial.expectEmail)
+
+ switch s.cluster.Login.OpenIDConnect.UsernameClaim {
+ case "alt_username":
+ c.Check(authinfo.Username, check.Equals, "desired-username")
+ case "":
+ c.Check(authinfo.Username, check.Equals, "")
+ default:
+ c.Fail() // bad test case
+ }
+ }
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: s.validCode,
c.Check(err, check.ErrorMatches, `.*401 Unauthorized: Not logged in.*`)
}
-func (s *LoginSuite) TestGoogleLogin_RealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
s.authEmail = "joe.smith@primary.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"names": []map[string]interface{}{
c.Check(authinfo.LastName, check.Equals, "Psmith")
}
-func (s *LoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
s.authName = "Joe P. Smith"
s.authEmail = "joe.smith@primary.example.com"
state := s.startLogin(c)
}
// People API returns some additional email addresses.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
s.authEmail = "joe.smith@primary.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
}
// Primary address is not the one initially returned by oidc.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
s.authEmail = "joe.smith@alternate.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
c.Check(authinfo.Username, check.Equals, "jsmith")
}
-func (s *LoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
s.authEmail = "joe.smith@unverified.example.com"
s.authEmailVerified = false
s.fakePeopleAPIResponse = map[string]interface{}{
c.Check(authinfo.Username, check.Equals, "")
}
-func (s *LoginSuite) startLogin(c *check.C) (state string) {
+func (s *OIDCLoginSuite) startLogin(c *check.C) (state string) {
// Initiate login, but instead of following the redirect to
// the provider, just grab state from the redirect URL.
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ReturnTo: "https://app.example.com/foo?bar"})
return
}
-func (s *LoginSuite) fakeToken(c *check.C, payload []byte) string {
+func (s *OIDCLoginSuite) fakeToken(c *check.C, payload []byte) string {
signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: s.issuerKey}, nil)
if err != nil {
c.Error(err)
"errors"
"fmt"
"net/http"
- "net/url"
"strings"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/msteinert/pam"
func (ctrl *pamLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
errorMessage := ""
sentPassword := false
- tx, err := pam.StartFunc(ctrl.Cluster.Login.PAMService, opts.Username, func(style pam.Style, message string) (string, error) {
+ tx, err := pam.StartFunc(ctrl.Cluster.Login.PAM.Service, opts.Username, func(style pam.Style, message string) (string, error) {
ctxlog.FromContext(ctx).Debugf("pam conversation: style=%v message=%q", style, message)
switch style {
case pam.ErrorMsg:
return arvados.APIClientAuthorization{}, err
}
email := user
- if domain := ctrl.Cluster.Login.PAMDefaultEmailDomain; domain != "" && !strings.Contains(email, "@") {
+ if domain := ctrl.Cluster.Login.PAM.DefaultEmailDomain; domain != "" && !strings.Contains(email, "@") {
email = email + "@" + domain
}
- ctxlog.FromContext(ctx).WithFields(logrus.Fields{"user": user, "email": email}).Debug("pam authentication succeeded")
- ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{ctrl.Cluster.SystemRootToken}})
- resp, err := ctrl.RailsProxy.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{
- // Send a fake ReturnTo value instead of the caller's
- // opts.ReturnTo. We won't follow the resulting
- // redirect target anyway.
- ReturnTo: ",https://none.invalid",
- AuthInfo: rpc.UserSessionAuthInfo{
- Username: user,
- Email: email,
- },
+ ctxlog.FromContext(ctx).WithFields(logrus.Fields{
+ "user": user,
+ "email": email,
+ }).Debug("pam authentication succeeded")
+ return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{
+ Username: user,
+ Email: email,
})
- if err != nil {
- return arvados.APIClientAuthorization{}, err
- }
- target, err := url.Parse(resp.RedirectLocation)
- if err != nil {
- return arvados.APIClientAuthorization{}, err
- }
- token := target.Query().Get("api_token")
- return ctrl.RailsProxy.APIClientAuthorizationCurrent(auth.NewContext(ctx, auth.NewCredentials(token)), arvados.GetOptions{})
}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-// Skip this slow test unless invoked as "go test -tags docker".
-// +build docker
-
-package localdb
-
-import (
- "os"
- "os/exec"
-
- check "gopkg.in/check.v1"
-)
-
-func (s *PamSuite) TestLoginLDAPViaPAM(c *check.C) {
- cmd := exec.Command("bash", "login_pam_docker_test.sh")
- cmd.Stdout = os.Stderr
- cmd.Stderr = os.Stderr
- err := cmd.Run()
- c.Check(err, check.IsNil)
-}
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.cluster.Login.PAM = true
- s.cluster.Login.PAMDefaultEmailDomain = "example.com"
+ s.cluster.Login.PAM.Enable = true
+ s.cluster.Login.PAM.DefaultEmailDomain = "example.com"
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
s.ctrl = &pamLoginController{
Cluster: s.cluster,
c.Check(resp.Scopes, check.DeepEquals, []string{"all"})
authinfo := getCallbackAuthInfo(c, s.railsSpy)
- c.Check(authinfo.Email, check.Equals, u+"@"+s.cluster.Login.PAMDefaultEmailDomain)
+ c.Check(authinfo.Email, check.Equals, u+"@"+s.cluster.Login.PAM.DefaultEmailDomain)
c.Check(authinfo.AlternateEmails, check.DeepEquals, []string(nil))
}
Header: hdrOut,
Body: reqIn.Body,
}).WithContext(reqIn.Context())
-
- resp, err := client.Do(reqOut)
- return resp, err
+ return client.Do(reqOut)
}
// Copy a response (or error) to the downstream client
"net/http"
"strings"
+ "git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
)
type router struct {
- mux *mux.Router
- fed arvados.API
+ mux *mux.Router
+ backend arvados.API
+ wrapCalls func(api.RoutableFunc) api.RoutableFunc
}
-func New(fed arvados.API) *router {
+// New returns a new router (which implements the http.Handler
+// interface) that serves requests by calling Arvados API methods on
+// the given backend.
+//
+// If wrapCalls is not nil, it is called once for each API method, and
+// the returned method is used in its place. This can be used to
+// install hooks before and after each API call and alter responses;
+// see localdb.WrapCallsInTransaction for an example.
+func New(backend arvados.API, wrapCalls func(api.RoutableFunc) api.RoutableFunc) *router {
rtr := &router{
- mux: mux.NewRouter(),
- fed: fed,
+ mux: mux.NewRouter(),
+ backend: backend,
+ wrapCalls: wrapCalls,
}
rtr.addRoutes()
return rtr
}
-type routableFunc func(ctx context.Context, opts interface{}) (interface{}, error)
-
func (rtr *router) addRoutes() {
for _, route := range []struct {
endpoint arvados.APIEndpoint
defaultOpts func() interface{}
- exec routableFunc
+ exec api.RoutableFunc
}{
{
arvados.EndpointConfigGet,
func() interface{} { return &struct{}{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ConfigGet(ctx)
+ return rtr.backend.ConfigGet(ctx)
},
},
{
arvados.EndpointLogin,
func() interface{} { return &arvados.LoginOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.Login(ctx, *opts.(*arvados.LoginOptions))
+ return rtr.backend.Login(ctx, *opts.(*arvados.LoginOptions))
},
},
{
arvados.EndpointLogout,
func() interface{} { return &arvados.LogoutOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.Logout(ctx, *opts.(*arvados.LogoutOptions))
+ return rtr.backend.Logout(ctx, *opts.(*arvados.LogoutOptions))
},
},
{
arvados.EndpointCollectionCreate,
func() interface{} { return &arvados.CreateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionCreate(ctx, *opts.(*arvados.CreateOptions))
+ return rtr.backend.CollectionCreate(ctx, *opts.(*arvados.CreateOptions))
},
},
{
arvados.EndpointCollectionUpdate,
func() interface{} { return &arvados.UpdateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ return rtr.backend.CollectionUpdate(ctx, *opts.(*arvados.UpdateOptions))
},
},
{
arvados.EndpointCollectionGet,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionGet(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.CollectionGet(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointCollectionList,
func() interface{} { return &arvados.ListOptions{Limit: -1} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionList(ctx, *opts.(*arvados.ListOptions))
+ return rtr.backend.CollectionList(ctx, *opts.(*arvados.ListOptions))
},
},
{
arvados.EndpointCollectionProvenance,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionProvenance(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.CollectionProvenance(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointCollectionUsedBy,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionUsedBy(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.CollectionUsedBy(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointCollectionDelete,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionDelete(ctx, *opts.(*arvados.DeleteOptions))
+ return rtr.backend.CollectionDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
{
arvados.EndpointCollectionTrash,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionTrash(ctx, *opts.(*arvados.DeleteOptions))
+ return rtr.backend.CollectionTrash(ctx, *opts.(*arvados.DeleteOptions))
},
},
{
arvados.EndpointCollectionUntrash,
func() interface{} { return &arvados.UntrashOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.CollectionUntrash(ctx, *opts.(*arvados.UntrashOptions))
+ return rtr.backend.CollectionUntrash(ctx, *opts.(*arvados.UntrashOptions))
},
},
{
arvados.EndpointContainerCreate,
func() interface{} { return &arvados.CreateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerCreate(ctx, *opts.(*arvados.CreateOptions))
+ return rtr.backend.ContainerCreate(ctx, *opts.(*arvados.CreateOptions))
},
},
{
arvados.EndpointContainerUpdate,
func() interface{} { return &arvados.UpdateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ return rtr.backend.ContainerUpdate(ctx, *opts.(*arvados.UpdateOptions))
},
},
{
arvados.EndpointContainerGet,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerGet(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.ContainerGet(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointContainerList,
func() interface{} { return &arvados.ListOptions{Limit: -1} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerList(ctx, *opts.(*arvados.ListOptions))
+ return rtr.backend.ContainerList(ctx, *opts.(*arvados.ListOptions))
},
},
{
arvados.EndpointContainerDelete,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerDelete(ctx, *opts.(*arvados.DeleteOptions))
+ return rtr.backend.ContainerDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
{
return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
},
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerLock(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.ContainerLock(ctx, *opts.(*arvados.GetOptions))
},
},
{
return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
},
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointSpecimenCreate,
func() interface{} { return &arvados.CreateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.SpecimenCreate(ctx, *opts.(*arvados.CreateOptions))
+ return rtr.backend.SpecimenCreate(ctx, *opts.(*arvados.CreateOptions))
},
},
{
arvados.EndpointSpecimenUpdate,
func() interface{} { return &arvados.UpdateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.SpecimenUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ return rtr.backend.SpecimenUpdate(ctx, *opts.(*arvados.UpdateOptions))
},
},
{
arvados.EndpointSpecimenGet,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.SpecimenGet(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.SpecimenGet(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointSpecimenList,
func() interface{} { return &arvados.ListOptions{Limit: -1} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.SpecimenList(ctx, *opts.(*arvados.ListOptions))
+ return rtr.backend.SpecimenList(ctx, *opts.(*arvados.ListOptions))
},
},
{
arvados.EndpointSpecimenDelete,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.SpecimenDelete(ctx, *opts.(*arvados.DeleteOptions))
+ return rtr.backend.SpecimenDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
{
arvados.EndpointUserCreate,
func() interface{} { return &arvados.CreateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserCreate(ctx, *opts.(*arvados.CreateOptions))
+ return rtr.backend.UserCreate(ctx, *opts.(*arvados.CreateOptions))
},
},
{
arvados.EndpointUserMerge,
func() interface{} { return &arvados.UserMergeOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserMerge(ctx, *opts.(*arvados.UserMergeOptions))
+ return rtr.backend.UserMerge(ctx, *opts.(*arvados.UserMergeOptions))
},
},
{
arvados.EndpointUserActivate,
func() interface{} { return &arvados.UserActivateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserActivate(ctx, *opts.(*arvados.UserActivateOptions))
+ return rtr.backend.UserActivate(ctx, *opts.(*arvados.UserActivateOptions))
},
},
{
arvados.EndpointUserSetup,
func() interface{} { return &arvados.UserSetupOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserSetup(ctx, *opts.(*arvados.UserSetupOptions))
+ return rtr.backend.UserSetup(ctx, *opts.(*arvados.UserSetupOptions))
},
},
{
arvados.EndpointUserUnsetup,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserUnsetup(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.UserUnsetup(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointUserGetCurrent,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserGetCurrent(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.UserGetCurrent(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointUserGetSystem,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserGetSystem(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.UserGetSystem(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointUserGet,
func() interface{} { return &arvados.GetOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserGet(ctx, *opts.(*arvados.GetOptions))
+ return rtr.backend.UserGet(ctx, *opts.(*arvados.GetOptions))
},
},
{
arvados.EndpointUserUpdateUUID,
func() interface{} { return &arvados.UpdateUUIDOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserUpdateUUID(ctx, *opts.(*arvados.UpdateUUIDOptions))
+ return rtr.backend.UserUpdateUUID(ctx, *opts.(*arvados.UpdateUUIDOptions))
},
},
{
arvados.EndpointUserUpdate,
func() interface{} { return &arvados.UpdateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserUpdate(ctx, *opts.(*arvados.UpdateOptions))
+ return rtr.backend.UserUpdate(ctx, *opts.(*arvados.UpdateOptions))
},
},
{
arvados.EndpointUserList,
func() interface{} { return &arvados.ListOptions{Limit: -1} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserList(ctx, *opts.(*arvados.ListOptions))
+ return rtr.backend.UserList(ctx, *opts.(*arvados.ListOptions))
},
},
{
arvados.EndpointUserBatchUpdate,
func() interface{} { return &arvados.UserBatchUpdateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserBatchUpdate(ctx, *opts.(*arvados.UserBatchUpdateOptions))
+ return rtr.backend.UserBatchUpdate(ctx, *opts.(*arvados.UserBatchUpdateOptions))
},
},
{
arvados.EndpointUserDelete,
func() interface{} { return &arvados.DeleteOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserDelete(ctx, *opts.(*arvados.DeleteOptions))
+ return rtr.backend.UserDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
{
arvados.EndpointUserAuthenticate,
func() interface{} { return &arvados.UserAuthenticateOptions{} },
func(ctx context.Context, opts interface{}) (interface{}, error) {
- return rtr.fed.UserAuthenticate(ctx, *opts.(*arvados.UserAuthenticateOptions))
+ return rtr.backend.UserAuthenticate(ctx, *opts.(*arvados.UserAuthenticateOptions))
},
},
} {
- rtr.addRoute(route.endpoint, route.defaultOpts, route.exec)
+ exec := route.exec
+ if rtr.wrapCalls != nil {
+ exec = rtr.wrapCalls(exec)
+ }
+ rtr.addRoute(route.endpoint, route.defaultOpts, exec)
}
rtr.mux.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
httpserver.Errors(w, []string{"API endpoint not found"}, http.StatusNotFound)
"GET": "HEAD", // Accept HEAD at any GET route
}
-func (rtr *router) addRoute(endpoint arvados.APIEndpoint, defaultOpts func() interface{}, exec routableFunc) {
+func (rtr *router) addRoute(endpoint arvados.APIEndpoint, defaultOpts func() interface{}, exec api.RoutableFunc) {
methods := []string{endpoint.Method}
if alt, ok := altMethod[endpoint.Method]; ok {
methods = append(methods, alt)
func (s *RouterSuite) SetUpTest(c *check.C) {
s.stub = arvadostest.APIStub{}
s.rtr = &router{
- mux: mux.NewRouter(),
- fed: &s.stub,
+ mux: mux.NewRouter(),
+ backend: &s.stub,
}
s.rtr.addRoutes()
}
cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
url, _ := url.Parse("https://" + os.Getenv("ARVADOS_TEST_API_HOST"))
- s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider))
+ s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider), nil)
}
func (s *RouterIntegrationSuite) TearDownSuite(c *check.C) {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package ctrlctx
+
+import (
+ "context"
+ "errors"
+ "sync"
+
+ "git.arvados.org/arvados.git/lib/controller/api"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/jmoiron/sqlx"
+ _ "github.com/lib/pq"
+)
+
+var (
+ ErrNoTransaction = errors.New("bug: there is no transaction in this context")
+ ErrContextFinished = errors.New("refusing to start a transaction after wrapped function already returned")
+)
+
+// WrapCallsInTransactions returns a call wrapper (suitable for
+// assigning to router.router.WrapCalls) that starts a new transaction
+// for each API call, and commits only if the call succeeds.
+//
+// The wrapper calls getdb() to get a database handle before each API
+// call.
+func WrapCallsInTransactions(getdb func(context.Context) (*sqlx.DB, error)) func(api.RoutableFunc) api.RoutableFunc {
+ return func(origFunc api.RoutableFunc) api.RoutableFunc {
+ return func(ctx context.Context, opts interface{}) (_ interface{}, err error) {
+ ctx, finishtx := New(ctx, getdb)
+ defer finishtx(&err)
+ return origFunc(ctx, opts)
+ }
+ }
+}
+
+// NewWithTransaction returns a child context in which the given
+// transaction will be used by any localdb API call that needs one.
+// The caller is responsible for calling Commit or Rollback on tx.
+func NewWithTransaction(ctx context.Context, tx *sqlx.Tx) context.Context {
+ txn := &transaction{tx: tx}
+ txn.setup.Do(func() {})
+ return context.WithValue(ctx, contextKeyTransaction, txn)
+}
+
+type contextKeyT string
+
+var contextKeyTransaction = contextKeyT("transaction")
+
+type transaction struct {
+ tx *sqlx.Tx
+ err error
+ getdb func(context.Context) (*sqlx.DB, error)
+ setup sync.Once
+}
+
+type finishFunc func(*error)
+
+// New returns a new child context that can be used with
+// CurrentTx(). It does not open a database transaction until the
+// first call to CurrentTx().
+//
+// The caller must eventually call the returned finishtx() func to
+// commit or rollback the transaction, if any.
+//
+// func example(ctx context.Context) (err error) {
+// ctx, finishtx := New(ctx, dber)
+// defer finishtx(&err)
+// // ...
+// tx, err := CurrentTx(ctx)
+// if err != nil {
+// return fmt.Errorf("example: %s", err)
+// }
+// return tx.ExecContext(...)
+// }
+//
+// If *err is nil, finishtx() commits the transaction and assigns any
+// resulting error to *err.
+//
+// If *err is non-nil, finishtx() rolls back the transaction, and
+// does not modify *err.
+func New(ctx context.Context, getdb func(context.Context) (*sqlx.DB, error)) (context.Context, finishFunc) {
+ txn := &transaction{getdb: getdb}
+ return context.WithValue(ctx, contextKeyTransaction, txn), func(err *error) {
+ txn.setup.Do(func() {
+ // Using (*sync.Once)Do() prevents a future
+ // call to CurrentTx() from opening a
+ // transaction which would never get committed
+ // or rolled back. If CurrentTx() hasn't been
+ // called before now, future calls will return
+ // this error.
+ txn.err = ErrContextFinished
+ })
+ if txn.tx == nil {
+ // we never [successfully] started a transaction
+ return
+ }
+ if *err != nil {
+ ctxlog.FromContext(ctx).Debug("rollback")
+ txn.tx.Rollback()
+ return
+ }
+ *err = txn.tx.Commit()
+ }
+}
+
+func CurrentTx(ctx context.Context) (*sqlx.Tx, error) {
+ txn, ok := ctx.Value(contextKeyTransaction).(*transaction)
+ if !ok {
+ return nil, ErrNoTransaction
+ }
+ txn.setup.Do(func() {
+ if db, err := txn.getdb(ctx); err != nil {
+ txn.err = err
+ } else {
+ txn.tx, txn.err = db.Beginx()
+ }
+ })
+ return txn.tx, txn.err
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package ctrlctx
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "testing"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/jmoiron/sqlx"
+ _ "github.com/lib/pq"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&DatabaseSuite{})
+
+type DatabaseSuite struct{}
+
+func (*DatabaseSuite) TestTransactionContext(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+
+ var getterCalled int64
+ getter := func(context.Context) (*sqlx.DB, error) {
+ atomic.AddInt64(&getterCalled, 1)
+ db, err := sqlx.Open("postgres", cluster.PostgreSQL.Connection.String())
+ c.Assert(err, check.IsNil)
+ return db, nil
+ }
+ wrapper := WrapCallsInTransactions(getter)
+ wrappedFunc := wrapper(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ txes := make([]*sqlx.Tx, 20)
+ var wg sync.WaitGroup
+ for i := range txes {
+ i := i
+ wg.Add(1)
+ go func() {
+ // Concurrent calls to CurrentTx(),
+ // with different children of the same
+ // parent context, will all return the
+ // same transaction.
+ defer wg.Done()
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ tx, err := CurrentTx(ctx)
+ c.Check(err, check.IsNil)
+ txes[i] = tx
+ }()
+ }
+ wg.Wait()
+ for i := range txes[1:] {
+ c.Check(txes[i], check.Equals, txes[i+1])
+ }
+ return true, nil
+ })
+
+ ok, err := wrappedFunc(context.Background(), "blah")
+ c.Check(ok, check.Equals, true)
+ c.Check(err, check.IsNil)
+ c.Check(getterCalled, check.Equals, int64(1))
+
+ // When a wrapped func returns without calling CurrentTx(),
+ // calling CurrentTx() later shouldn't start a new
+ // transaction.
+ var savedctx context.Context
+ ok, err = wrapper(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ savedctx = ctx
+ return true, nil
+ })(context.Background(), "blah")
+ c.Check(ok, check.Equals, true)
+ c.Check(err, check.IsNil)
+ tx, err := CurrentTx(savedctx)
+ c.Check(tx, check.IsNil)
+ c.Check(err, check.NotNil)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package deduplicationreport
+
+import (
+ "io"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+type NoPrefixFormatter struct{}
+
+func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ return []byte(entry.Message), nil
+}
+
+// RunCommand implements the subcommand "deduplication-report <collection> <collection> ..."
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Error("fatal")
+ }
+ }()
+
+ logger.SetFormatter(new(NoPrefixFormatter))
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ exitcode := report(prog, args, loader, logger, stdout, stderr)
+
+ return exitcode
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package deduplicationreport
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/manifest"
+
+ "github.com/dustin/go-humanize"
+ "github.com/sirupsen/logrus"
+)
+
+func deDuplicate(inputs []string) (trimmed []string) {
+ seen := make(map[string]bool)
+ for _, uuid := range inputs {
+ if !seen[uuid] {
+ seen[uuid] = true
+ trimmed = append(trimmed, uuid)
+ }
+ }
+ return
+}
+
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitcode int, inputs []string) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `
+Usage:
+ %s [options ...] <collection-uuid> <collection-uuid> ...
+
+ %s [options ...] <collection-pdh>,<collection_uuid> \
+ <collection-pdh>,<collection_uuid> ...
+
+ This program analyzes the overlap in blocks used by 2 or more collections. It
+ prints a deduplication report that shows the nominal space used by the
+ collections, as well as the actual size and the amount of space that is saved
+ by Keep's deduplication.
+
+ The list of collections may be provided in two ways. A list of collection
+ uuids is sufficient. Alternatively, the PDH for each collection may also be
+ provided. This is will greatly speed up operation when the list contains
+ multiple collections with the same PDH.
+
+ Exit status will be zero if there were no errors generating the report.
+
+Example:
+
+ Use the 'arv' and 'jq' commands to get the list of the 100
+ largest collections and generate the deduplication report:
+
+ arv collection list --order 'file_size_total desc' --limit 100 | \
+ jq -r '.items[] | [.portable_data_hash,.uuid] |@csv' | \
+ tail -n+2 |sed -e 's/"//g'|tr '\n' ' ' | \
+ xargs %s
+
+Options:
+`, prog, prog, prog)
+ flags.PrintDefaults()
+ }
+ loader.SetupFlags(flags)
+ loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+ err := flags.Parse(args)
+ if err == flag.ErrHelp {
+ return 0, inputs
+ } else if err != nil {
+ return 2, inputs
+ }
+
+ inputs = flags.Args()
+
+ inputs = deDuplicate(inputs)
+
+ if len(inputs) < 1 {
+ logger.Errorf("Error: no collections provided")
+ flags.Usage()
+ return 2, inputs
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ return 2, inputs
+ }
+ logger.SetLevel(lvl)
+ return
+}
+
+func blockList(collection arvados.Collection) (blocks map[string]int) {
+ blocks = make(map[string]int)
+ m := manifest.Manifest{Text: collection.ManifestText}
+ blockChannel := m.BlockIterWithDuplicates()
+ for b := range blockChannel {
+ blocks[b.Digest.String()] = b.Size
+ }
+ return
+}
+
+func report(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
+
+ var inputs []string
+ exitcode, inputs = parseFlags(prog, args, loader, logger, stderr)
+ if exitcode != 0 {
+ return
+ }
+
+ // Arvados Client setup
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ logger.Errorf("Error creating Arvados object: %s", err)
+ exitcode = 1
+ return
+ }
+
+ type Col struct {
+ FileSizeTotal int64
+ FileCount int64
+ }
+
+ blocks := make(map[string]map[string]int)
+ pdhs := make(map[string]Col)
+ var nominalSize int64
+
+ for _, input := range inputs {
+ var uuid string
+ var pdh string
+ if strings.Contains(input, ",") {
+ // The input is in the format pdh,uuid. This will allow us to save time on duplicate pdh's
+ tmp := strings.Split(input, ",")
+ pdh = tmp[0]
+ uuid = tmp[1]
+ } else {
+ // The input must be a plain uuid
+ uuid = input
+ }
+ if !strings.Contains(uuid, "-4zz18-") {
+ logger.Errorf("Error: uuid must refer to collection object")
+ exitcode = 1
+ return
+ }
+ if _, ok := pdhs[pdh]; ok {
+ // We've processed a collection with this pdh already. Simply add its
+ // size to the totals and move on to the next one.
+ // Note that we simply trust the PDH matches the collection UUID here,
+ // in other words, we use it over the UUID. If they don't match, the report
+ // will be wrong.
+ nominalSize += pdhs[pdh].FileSizeTotal
+ } else {
+ var collection arvados.Collection
+ err = arv.Get("collections", uuid, nil, &collection)
+ if err != nil {
+ logger.Errorf("Error: unable to retrieve collection: %s", err)
+ exitcode = 1
+ return
+ }
+ blocks[uuid] = make(map[string]int)
+ blocks[uuid] = blockList(collection)
+ if pdh != "" && collection.PortableDataHash != pdh {
+ logger.Errorf("Error: the collection with UUID %s has PDH %s, but a different PDH was provided in the arguments: %s", uuid, collection.PortableDataHash, pdh)
+ exitcode = 1
+ return
+ }
+ if pdh == "" {
+ pdh = collection.PortableDataHash
+ }
+
+ col := Col{}
+ if collection.FileSizeTotal != 0 || collection.FileCount != 0 {
+ nominalSize += collection.FileSizeTotal
+ col.FileSizeTotal = collection.FileSizeTotal
+ col.FileCount = int64(collection.FileCount)
+ } else {
+ // Collections created with old Arvados versions do not always have the total file size and count cached in the collections object
+ var collSize int64
+ for _, size := range blocks[uuid] {
+ collSize += int64(size)
+ }
+ nominalSize += collSize
+ col.FileSizeTotal = collSize
+ }
+ pdhs[pdh] = col
+ }
+
+ if pdhs[pdh].FileCount != 0 {
+ fmt.Fprintf(stdout, "Collection %s: pdh %s; nominal size %d (%s); file count %d\n", uuid, pdh, pdhs[pdh].FileSizeTotal, humanize.IBytes(uint64(pdhs[pdh].FileSizeTotal)), pdhs[pdh].FileCount)
+ } else {
+ fmt.Fprintf(stdout, "Collection %s: pdh %s; nominal size %d (%s)\n", uuid, pdh, pdhs[pdh].FileSizeTotal, humanize.IBytes(uint64(pdhs[pdh].FileSizeTotal)))
+ }
+ }
+
+ var totalSize int64
+ seen := make(map[string]bool)
+ for _, v := range blocks {
+ for pdh, size := range v {
+ if !seen[pdh] {
+ seen[pdh] = true
+ totalSize += int64(size)
+ }
+ }
+ }
+ fmt.Fprintln(stdout)
+ fmt.Fprintf(stdout, "Collections: %15d\n", len(inputs))
+ fmt.Fprintf(stdout, "Nominal size of stored data: %15d bytes (%s)\n", nominalSize, humanize.IBytes(uint64(nominalSize)))
+ fmt.Fprintf(stdout, "Actual size of stored data: %15d bytes (%s)\n", totalSize, humanize.IBytes(uint64(totalSize)))
+ fmt.Fprintf(stdout, "Saved by Keep deduplication: %15d bytes (%s)\n", nominalSize-totalSize, humanize.IBytes(uint64(nominalSize-totalSize)))
+
+ return exitcode
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package deduplicationreport
+
+import (
+ "bytes"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the database so they don't affect subsequent tests.
+ arvadostest.ResetEnv()
+}
+
+func (*Suite) TestUsage(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{"-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestTwoIdenticalUUIDs(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run dedupreport with 2 identical uuids
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, "(?ms).*Collections:[[:space:]]+1.*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+3 bytes \\(3 B\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+3 bytes \\(3 B\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+0 bytes \\(0 B\\).*")
+ c.Log(stderr.String())
+}
+
+func (*Suite) TestTwoUUIDsInvalidPDH(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run dedupreport with pdh,uuid where pdh does not match
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooAndBarFilesInDirPDH + "," + arvadostest.FooCollection, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*Error: the collection with UUID zzzzz-4zz18-fy296fx3hot09f7 has PDH 1f4b0bc7583c2a7f9102c395f4ffc5e3\+45, but a different PDH was provided in the arguments: 870369fc72738603c2fad16664e50e2d\+58.*`)
+}
+
+func (*Suite) TestNonExistentCollection(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run dedupreport with many UUIDs
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.NonexistentCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "Collection zzzzz-4zz18-fy296fx3hot09f7: pdh 1f4b0bc7583c2a7f9102c395f4ffc5e3+45; nominal size 3 (3 B)\n")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*Error: unable to retrieve collection:.*404 Not Found.*`)
+}
+
+func (*Suite) TestManyUUIDsNoOverlap(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run dedupreport with 5 UUIDs
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.HelloWorldCollection, arvadostest.FooBarDirCollection, arvadostest.WazVersion1Collection, arvadostest.UserAgreementCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, "(?ms).*Collections:[[:space:]]+5.*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+249049 bytes \\(243 KiB\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+249049 bytes \\(243 KiB\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+0 bytes \\(0 B\\).*")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Equals, "")
+}
+
+func (*Suite) TestTwoOverlappingCollections(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Create two collections
+ arv := arvados.NewClientFromEnv()
+
+ var c1 arvados.Collection
+ err := arv.RequestAndDecode(&c1, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{"manifest_text": ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:foo\n"}})
+ c.Assert(err, check.Equals, nil)
+
+ var c2 arvados.Collection
+ err = arv.RequestAndDecode(&c2, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{"manifest_text": ". c157a79031e1c40f85931829bc5fc552+4 d3b07384d113edec49eaa6238ad5ff00+4 0:4:bar 4:4:foo\n"}})
+ c.Assert(err, check.Equals, nil)
+
+ for _, trial := range []struct {
+ field1 string
+ field2 string
+ }{
+ {
+ // Run dedupreport with 2 arguments: uuid uuid
+ field1: c1.UUID,
+ field2: c2.UUID,
+ },
+ {
+ // Run dedupreport with 2 arguments: pdh,uuid uuid
+ field1: c1.PortableDataHash + "," + c1.UUID,
+ field2: c2.UUID,
+ },
+ } {
+ exitcode := Command.RunCommand("deduplicationreport.test", []string{trial.field1, trial.field2}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+12 bytes \\(12 B\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+8 bytes \\(8 B\\).*")
+ c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+4 bytes \\(4 B\\).*")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Equals, "")
+ }
+}
}
defer func() {
cmd.Process.Signal(syscall.SIGTERM)
- logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+ logger.Info("sent SIGTERM; waiting for postgres to shut down")
cmd.Wait()
}()
- for deadline := time.Now().Add(10 * time.Second); ; {
- output, err2 := exec.Command("pg_isready").CombinedOutput()
- if err2 == nil {
- break
- } else if time.Now().After(deadline) {
- err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
- return 1
- } else {
- time.Sleep(time.Second)
- }
+ err = waitPostgreSQLReady()
+ if err != nil {
+ return 1
}
}
// might never have been run.
}
+ var needcoll []string
+ // If the en_US.UTF-8 locale wasn't installed when
+ // postgresql initdb ran, it needs to be added
+ // explicitly before we can use it in our test suite.
+ for _, collname := range []string{"en_US", "en_US.UTF-8"} {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_collation WHERE collname='"+collname+"' AND collcollate IN ('en_US.UTF-8', 'en_US.utf8')")
+ cmd.Dir = "/"
+ out, err2 := cmd.CombinedOutput()
+ if err != nil {
+ err = fmt.Errorf("error while checking postgresql collations: %s", err2)
+ return 1
+ }
+ if strings.Contains(string(out), "1") {
+ logger.Infof("postgresql supports collation %s", collname)
+ } else {
+ needcoll = append(needcoll, collname)
+ }
+ }
+ if len(needcoll) > 0 && os.Getpid() != 1 {
+ // In order for the CREATE COLLATION statement
+ // below to work, the locale must have existed
+ // when PostgreSQL started up. If we're
+ // running as init, we must have started
+ // PostgreSQL ourselves after installing the
+ // locales. Otherwise, it might need a
+ // restart, so we attempt to restart it with
+ // systemd.
+ if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+ logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
+ } else if err = waitPostgreSQLReady(); err != nil {
+ return 1
+ }
+ }
+ for _, collname := range needcoll {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE COLLATION \""+collname+"\" (LOCALE = \"en_US.UTF-8\")")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ cmd.Dir = "/"
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("error adding postgresql collation %s: %s", collname, err)
+ return 1
+ }
+ }
+
withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff)
cmd.Dir = "/"
return osv, nil
}
+func waitPostgreSQLReady() error {
+ for deadline := time.Now().Add(10 * time.Second); ; {
+ output, err := exec.Command("pg_isready").CombinedOutput()
+ if err == nil {
+ return nil
+ } else if time.Now().After(deadline) {
+ return fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+ } else {
+ time.Sleep(time.Second)
+ }
+ }
+}
+
func runBash(script string, stdout, stderr io.Writer) error {
cmd := exec.Command("bash", "-")
cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
--- /dev/null
+pam_arvados.h
+pam_arvados.so
--- /dev/null
+For configuration advice, please refer to https://doc.arvados.org/install/install-webshell.html
+
+Usage (in pam config):
+
+ pam_arvados.so arvados_api_host my_vm_hostname ["insecure"] ["debug"]
+
+pam_arvados.so passes authentication if (according to
+arvados_api_host) the supplied PAM token belongs to an Arvados user
+who is allowed to log in to my_vm_host_name with the supplied PAM
+username.
+
+If my_vm_hostname is omitted or "-", the current hostname is used.
+
+"insecure" -- continue even if the TLS certificate presented by
+arvados_api_host fails verification.
+
+"debug" -- enable debug-level log messages in syslog and (when not in
+"silent" mode) on the calling application's stderr.
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "crypto/tls"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "gopkg.in/check.v1"
+)
+
+type DockerSuite struct {
+ tmpdir string
+ hostip string
+ proxyln net.Listener
+ proxysrv *http.Server
+}
+
+var _ = check.Suite(&DockerSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *DockerSuite) SetUpSuite(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+
+ s.tmpdir = c.MkDir()
+
+ // The integration-testing controller listens on the loopback
+ // interface, so it won't be reachable directly from the
+ // docker container -- so here we run a proxy on 0.0.0.0 for
+ // the duration of the test.
+ hostips, err := exec.Command("hostname", "-I").Output()
+ c.Assert(err, check.IsNil)
+ s.hostip = strings.Split(strings.Trim(string(hostips), "\n"), " ")[0]
+ ln, err := net.Listen("tcp", s.hostip+":0")
+ c.Assert(err, check.IsNil)
+ s.proxyln = ln
+ proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")})
+ proxy.Transport = &http.Transport{
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: true,
+ },
+ }
+ s.proxysrv = &http.Server{Handler: proxy}
+ go s.proxysrv.ServeTLS(ln, "../../services/api/tmp/self-signed.pem", "../../services/api/tmp/self-signed.key")
+
+ // Build a pam module to install & configure in the docker
+ // container.
+ cmd := exec.Command("go", "build", "-buildmode=c-shared", "-o", s.tmpdir+"/pam_arvados.so")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ // Build the testclient program that will (from inside the
+ // docker container) configure the system to use the above PAM
+ // config, and then try authentication.
+ cmd = exec.Command("go", "build", "-o", s.tmpdir+"/testclient", "./testclient.go")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+}
+
+func (s *DockerSuite) TearDownSuite(c *check.C) {
+ if s.proxysrv != nil {
+ s.proxysrv.Close()
+ }
+ if s.proxyln != nil {
+ s.proxyln.Close()
+ }
+}
+
+func (s *DockerSuite) SetUpTest(c *check.C) {
+ // Write a PAM config file that uses our proxy as
+ // ARVADOS_API_HOST.
+ proxyhost := s.proxyln.Addr().String()
+ confdata := fmt.Sprintf(`Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s testvm2.shell insecure
+Auth-Initial:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s testvm2.shell insecure
+`, proxyhost, proxyhost)
+ err := ioutil.WriteFile(s.tmpdir+"/conffile", []byte(confdata), 0755)
+ c.Assert(err, check.IsNil)
+}
+
+func (s *DockerSuite) runTestClient(c *check.C, args ...string) (stdout, stderr *bytes.Buffer, err error) {
+
+ cmd := exec.Command("docker", append([]string{
+ "run", "--rm",
+ "--hostname", "testvm2.shell",
+ "--add-host", "zzzzz.arvadosapi.com:" + s.hostip,
+ "-v", s.tmpdir + "/pam_arvados.so:/usr/lib/pam_arvados.so:ro",
+ "-v", s.tmpdir + "/conffile:/usr/share/pam-configs/arvados:ro",
+ "-v", s.tmpdir + "/testclient:/testclient:ro",
+ "debian:buster",
+ "/testclient"}, args...)...)
+ stdout = &bytes.Buffer{}
+ stderr = &bytes.Buffer{}
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ return
+}
+
+func (s *DockerSuite) TestSuccess(c *check.C) {
+ stdout, stderr, err := s.runTestClient(c, "try", "active", arvadostest.ActiveTokenV2)
+ c.Check(err, check.IsNil)
+ c.Logf("%s", stderr.String())
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*authentication succeeded.*`)
+}
+
+func (s *DockerSuite) TestFailure(c *check.C) {
+ for _, trial := range []struct {
+ label string
+ username string
+ token string
+ }{
+ {"bad token", "active", arvadostest.ActiveTokenV2 + "badtoken"},
+ {"empty token", "active", ""},
+ {"empty username", "", arvadostest.ActiveTokenV2},
+ {"wrong username", "wrongusername", arvadostest.ActiveTokenV2},
+ } {
+ c.Logf("trial: %s", trial.label)
+ stdout, stderr, err := s.runTestClient(c, "try", trial.username, trial.token)
+ c.Logf("%s", stderr.String())
+ c.Check(err, check.NotNil)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*authentication failed.*`)
+ }
+}
+
+func (s *DockerSuite) TestDefaultHostname(c *check.C) {
+ confdata := fmt.Sprintf(`Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s - insecure debug
+Auth-Initial:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s - insecure debug
+`, s.proxyln.Addr().String(), s.proxyln.Addr().String())
+ err := ioutil.WriteFile(s.tmpdir+"/conffile", []byte(confdata), 0755)
+ c.Assert(err, check.IsNil)
+
+ stdout, stderr, err := s.runTestClient(c, "try", "active", arvadostest.ActiveTokenV2)
+ c.Check(err, check.IsNil)
+ c.Logf("%s", stderr.String())
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*authentication succeeded.*`)
+}
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+fpm_depends+=(ca-certificates)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This file is packaged as /usr/share/pam-configs/arvados-go; see build/run-library.sh
+
+# 1. Run `pam-auth-update` and choose Arvados authentication
+# 2. In /etc/pam.d/common-auth, change "api.example" to your ARVADOS_API_HOST
+# 3. In /etc/pam.d/common-auth, change "shell.example" to this host's hostname
+# (as it appears in the Arvados virtual_machines list)
+
+Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+ [success=end default=ignore] /usr/lib/pam_arvados.so api.example shell.example
+Auth-Initial:
+ [success=end default=ignore] /usr/lib/pam_arvados.so api.example shell.example
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// To enable, add an entry in /etc/pam.d/common-auth where pam_unix.so
+// would normally be. Examples:
+//
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example insecure debug
+//
+// Replace zzzzz.arvadosapi.com with your controller host or
+// host:port.
+//
+// Replace vmhostname.example with the VM's name as it appears in the
+// Arvados virtual_machine object.
+//
+// Use "insecure" if your API server certificate does not pass name
+// verification.
+//
+// Use "debug" to enable debug log messages.
+
+package main
+
+import (
+ "io/ioutil"
+ "log/syslog"
+ "os"
+
+ "context"
+ "errors"
+ "fmt"
+ "runtime"
+ "syscall"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+ lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+ "golang.org/x/sys/unix"
+)
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i);
+const char *get_user(pam_handle_t *pamh);
+const char *get_authtoken(pam_handle_t *pamh);
+*/
+import "C"
+
+func main() {}
+
+func init() {
+ if err := unix.Prctl(syscall.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil {
+ newLogger(false).WithError(err).Warn("unable to disable ptrace")
+ }
+}
+
+//export pam_sm_setcred
+func pam_sm_setcred(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+ return C.PAM_IGNORE
+}
+
+//export pam_sm_authenticate
+func pam_sm_authenticate(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+ runtime.GOMAXPROCS(1)
+ logger := newLogger(flags&C.PAM_SILENT == 0)
+ cUsername := C.get_user(pamh)
+ if cUsername == nil {
+ return C.PAM_USER_UNKNOWN
+ }
+
+ cToken := C.get_authtoken(pamh)
+ if cToken == nil {
+ return C.PAM_AUTH_ERR
+ }
+
+ argv := make([]string, cArgc)
+ for i := 0; i < int(cArgc); i++ {
+ argv[i] = C.GoString(C.stringindex(cArgv, C.int(i)))
+ }
+
+ err := authenticate(logger, C.GoString(cUsername), C.GoString(cToken), argv)
+ if err != nil {
+ logger.WithError(err).Error("authentication failed")
+ return C.PAM_AUTH_ERR
+ }
+ return C.PAM_SUCCESS
+}
+
+func authenticate(logger *logrus.Logger, username, token string, argv []string) error {
+ hostname := ""
+ apiHost := ""
+ insecure := false
+ for idx, arg := range argv {
+ if idx == 0 {
+ apiHost = arg
+ } else if idx == 1 {
+ hostname = arg
+ } else if arg == "insecure" {
+ insecure = true
+ } else if arg == "debug" {
+ logger.SetLevel(logrus.DebugLevel)
+ } else {
+ logger.Warnf("unkown option: %s\n", arg)
+ }
+ }
+ if hostname == "" || hostname == "-" {
+ h, err := os.Hostname()
+ if err != nil {
+ logger.WithError(err).Warnf("cannot get hostname -- try using an explicit hostname in pam config")
+ return fmt.Errorf("cannot get hostname: %w", err)
+ }
+ hostname = h
+ }
+ logger.Debugf("username=%q arvados_api_host=%q hostname=%q insecure=%t", username, apiHost, hostname, insecure)
+ if apiHost == "" {
+ logger.Warnf("cannot authenticate: config error: arvados_api_host and hostname must be non-empty")
+ return errors.New("config error")
+ }
+ arv := &arvados.Client{
+ Scheme: "https",
+ APIHost: apiHost,
+ AuthToken: token,
+ Insecure: insecure,
+ }
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+ defer cancel()
+ var vms arvados.VirtualMachineList
+ err := arv.RequestAndDecodeContext(ctx, &vms, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{
+ Limit: 2,
+ Filters: []arvados.Filter{
+ {"hostname", "=", hostname},
+ },
+ })
+ if err != nil {
+ return err
+ }
+ if len(vms.Items) == 0 {
+ // It's possible there is no VM entry for the
+ // configured hostname, but typically this just means
+ // the user does not have permission to see (let alone
+ // log in to) this VM.
+ return errors.New("permission denied")
+ } else if len(vms.Items) > 1 {
+ return fmt.Errorf("multiple results for hostname %q", hostname)
+ } else if vms.Items[0].Hostname != hostname {
+ return fmt.Errorf("looked up hostname %q but controller returned record with hostname %q", hostname, vms.Items[0].Hostname)
+ }
+ var user arvados.User
+ err = arv.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return err
+ }
+ var links arvados.LinkList
+ err = arv.RequestAndDecodeContext(ctx, &links, "GET", "arvados/v1/links", nil, arvados.ListOptions{
+ Limit: 1,
+ Filters: []arvados.Filter{
+ {"link_class", "=", "permission"},
+ {"name", "=", "can_login"},
+ {"tail_uuid", "=", user.UUID},
+ {"head_uuid", "=", vms.Items[0].UUID},
+ {"properties.username", "=", username},
+ },
+ })
+ if err != nil {
+ return err
+ }
+ if len(links.Items) < 1 || links.Items[0].Properties["username"] != username {
+ return errors.New("permission denied")
+ }
+ logger.Debugf("permission granted based on link with UUID %s", links.Items[0].UUID)
+ return nil
+}
+
+func newLogger(stderr bool) *logrus.Logger {
+ logger := logrus.New()
+ if !stderr {
+ logger.Out = ioutil.Discard
+ }
+ if hook, err := lSyslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_AUTH|syslog.LOG_INFO, "pam_arvados"); err != nil {
+ logger.Hooks.Add(hook)
+ }
+ return logger
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i) { return a[i]; }
+const char *get_user(pam_handle_t *pamh) {
+ const char *user;
+ if (pam_get_item(pamh, PAM_USER, (const void**)&user) != PAM_SUCCESS)
+ return NULL;
+ return user;
+}
+const char *get_authtoken(pam_handle_t *pamh) {
+ const char *token;
+ if (pam_get_authtok(pamh, PAM_AUTHTOK, &token, NULL) != PAM_SUCCESS)
+ return NULL;
+ return token;
+}
+*/
+import "C"
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// +build never
+
+// This file is compiled by docker_test.go to build a test client.
+// It's not part of the pam module itself.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+
+ "github.com/msteinert/pam"
+ "github.com/sirupsen/logrus"
+)
+
+func main() {
+ if len(os.Args) != 4 || os.Args[1] != "try" {
+ logrus.Print("usage: testclient try 'username' 'password'")
+ os.Exit(1)
+ }
+ username := os.Args[2]
+ password := os.Args[3]
+
+ // Configure PAM to use arvados token auth by default.
+ cmd := exec.Command("pam-auth-update", "--force", "arvados", "--remove", "unix")
+ cmd.Env = append([]string{"DEBIAN_FRONTEND=noninteractive"}, os.Environ()...)
+ cmd.Stdin = nil
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ logrus.WithError(err).Error("pam-auth-update failed")
+ os.Exit(1)
+ }
+
+ // Check that pam-auth-update actually added arvados config.
+ cmd = exec.Command("grep", "-Hn", "arvados", "/etc/pam.d/common-auth")
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ if err != nil {
+ panic(err)
+ }
+
+ logrus.Debugf("starting pam: username=%q password=%q", username, password)
+
+ sentPassword := false
+ errorMessage := ""
+ tx, err := pam.StartFunc("default", username, func(style pam.Style, message string) (string, error) {
+ logrus.Debugf("pam conversation: style=%v message=%q", style, message)
+ switch style {
+ case pam.ErrorMsg:
+ logrus.WithField("Message", message).Info("pam.ErrorMsg")
+ errorMessage = message
+ return "", nil
+ case pam.TextInfo:
+ logrus.WithField("Message", message).Info("pam.TextInfo")
+ errorMessage = message
+ return "", nil
+ case pam.PromptEchoOn, pam.PromptEchoOff:
+ sentPassword = true
+ return password, nil
+ default:
+ return "", fmt.Errorf("unrecognized message style %d", style)
+ }
+ })
+ if err != nil {
+ logrus.WithError(err).Print("StartFunc failed")
+ os.Exit(1)
+ }
+ err = tx.Authenticate(pam.DisallowNullAuthtok)
+ if err != nil {
+ err = fmt.Errorf("PAM: %s (message = %q)", err, errorMessage)
+ logrus.WithError(err).Print("authentication failed")
+ os.Exit(1)
+ }
+ logrus.Print("authentication succeeded")
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package recovercollection
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "strings"
+ "sync"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Error("fatal")
+ }
+ logger.Info("exiting")
+ }()
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `Usage:
+ %s [options ...] { /path/to/manifest.txt | log-or-collection-uuid } [...]
+
+ This program recovers deleted collections. Recovery is
+ possible when the collection's manifest is still available and
+ all of its data blocks are still available or recoverable
+ (e.g., garbage collection is not enabled, the blocks are too
+ new for garbage collection, the blocks are referenced by other
+ collections, or the blocks have been trashed but not yet
+ deleted).
+
+ There are multiple ways to specify a collection to recover:
+
+ * Path to a local file containing a manifest with the desired
+ data
+
+ * UUID of an Arvados log entry, typically a "delete" or
+ "update" event, whose "old attributes" have a manifest with
+ the desired data
+
+ * UUID of an Arvados collection whose most recent log entry,
+ typically a "delete" or "update" event, has the desired
+ data in its "old attributes"
+
+ For each provided collection manifest, once all data blocks
+ are recovered/protected from garbage collection, a new
+ collection is saved and its UUID is printed on stdout.
+
+ Restored collections will belong to the system (root) user.
+
+ Exit status will be zero if recovery is successful, i.e., a
+ collection is saved for each provided manifest.
+Options:
+`, prog)
+ flags.PrintDefaults()
+ }
+ loader.SetupFlags(flags)
+ loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ }
+
+ if len(flags.Args()) == 0 {
+ flags.Usage()
+ return 2
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ return 2
+ }
+ logger.SetLevel(lvl)
+
+ cfg, err := loader.Load()
+ if err != nil {
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return 1
+ }
+ client.AuthToken = cluster.SystemRootToken
+ rcvr := recoverer{
+ client: client,
+ cluster: cluster,
+ logger: logger,
+ }
+
+ exitcode := 0
+ for _, src := range flags.Args() {
+ logger := logger.WithField("src", src)
+ var mtxt string
+ if !strings.Contains(src, "/") && len(src) == 27 && src[5] == '-' && src[11] == '-' {
+ var filters []arvados.Filter
+ if src[5:12] == "-57u5n-" {
+ filters = []arvados.Filter{{"uuid", "=", src}}
+ } else if src[5:12] == "-4zz18-" {
+ filters = []arvados.Filter{{"object_uuid", "=", src}}
+ } else {
+ logger.Error("looks like a UUID but not a log or collection UUID (if it's really a file, prepend './')")
+ exitcode = 1
+ continue
+ }
+ var resp struct {
+ Items []struct {
+ UUID string `json:"uuid"`
+ EventType string `json:"event_type"`
+ EventAt time.Time `json:"event_at"`
+ ObjectUUID string `json:"object_uuid"`
+ Properties struct {
+ OldAttributes struct {
+ ManifestText string `json:"manifest_text"`
+ } `json:"old_attributes"`
+ } `json:"properties"`
+ }
+ }
+ err = client.RequestAndDecode(&resp, "GET", "arvados/v1/logs", nil, arvados.ListOptions{
+ Limit: 1,
+ Order: []string{"event_at desc"},
+ Filters: filters,
+ })
+ if err != nil {
+ logger.WithError(err).Error("error looking up log entry")
+ exitcode = 1
+ continue
+ } else if len(resp.Items) == 0 {
+ logger.Error("log entry not found")
+ exitcode = 1
+ continue
+ }
+ logent := resp.Items[0]
+ logger.WithFields(logrus.Fields{
+ "uuid": logent.UUID,
+ "old_collection_uuid": logent.ObjectUUID,
+ "logged_event_type": logent.EventType,
+ "logged_event_time": logent.EventAt,
+ "logged_object_uuid": logent.ObjectUUID,
+ }).Info("loaded log entry")
+ mtxt = logent.Properties.OldAttributes.ManifestText
+ if mtxt == "" {
+ logger.Error("log entry properties.old_attributes.manifest_text missing or empty")
+ exitcode = 1
+ continue
+ }
+ } else {
+ buf, err := ioutil.ReadFile(src)
+ if err != nil {
+ logger.WithError(err).Error("failed to load manifest data from file")
+ exitcode = 1
+ continue
+ }
+ mtxt = string(buf)
+ }
+ uuid, err := rcvr.RecoverManifest(string(mtxt))
+ if err != nil {
+ logger.WithError(err).Error("recovery failed")
+ exitcode = 1
+ continue
+ }
+ logger.WithField("UUID", uuid).Info("recovery succeeded")
+ fmt.Fprintln(stdout, uuid)
+ }
+ return exitcode
+}
+
+type recoverer struct {
+ client *arvados.Client
+ cluster *arvados.Cluster
+ logger logrus.FieldLogger
+}
+
+var errNotFound = errors.New("not found")
+
+// Finds the timestamp of the newest copy of blk on svc. Returns
+// errNotFound if blk is not on svc at all.
+func (rcvr recoverer) newestMtime(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+ found, err := svc.Index(ctx, rcvr.client, blk)
+ if err != nil {
+ logger.WithError(err).Warn("error getting index")
+ return time.Time{}, err
+ } else if len(found) == 0 {
+ return time.Time{}, errNotFound
+ }
+ var latest time.Time
+ for _, ent := range found {
+ t := time.Unix(0, ent.Mtime)
+ if t.After(latest) {
+ latest = t
+ }
+ }
+ logger.WithField("latest", latest).Debug("found")
+ return latest, nil
+}
+
+var errTouchIneffective = errors.New("(BUG?) touch succeeded but had no effect -- reported timestamp is still too old")
+
+// Ensures the given block exists on the given server and won't be
+// eligible for trashing until after our chosen deadline (blobsigexp).
+// Returns an error if the block doesn't exist on the given server, or
+// has an old timestamp and can't be updated.
+//
+// After we decide a block is "safe" (whether or not we had to untrash
+// it), keep-balance might notice that it's currently unreferenced and
+// decide to trash it, all before our recovered collection gets
+// saved. But if the block's timestamp is more recent than blobsigttl,
+// keepstore will refuse to trash it even if told to by keep-balance.
+func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
+ if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err != nil {
+ return err
+ } else if latest.Add(blobsigttl).After(blobsigexp) {
+ return nil
+ }
+ if err := svc.Touch(ctx, rcvr.client, blk); err != nil {
+ return fmt.Errorf("error updating timestamp: %s", err)
+ }
+ logger.Debug("updated timestamp")
+ if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err == errNotFound {
+ return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
+ } else if err != nil {
+ return err
+ } else if latest.Add(blobsigttl).After(blobsigexp) {
+ return nil
+ } else {
+ return errTouchIneffective
+ }
+}
+
+// Untrash and update GC timestamps (as needed) on blocks referenced
+// by the given manifest, save a new collection and return the new
+// collection's UUID.
+func (rcvr recoverer) RecoverManifest(mtxt string) (string, error) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ coll := arvados.Collection{ManifestText: mtxt}
+ blks, err := coll.SizedDigests()
+ if err != nil {
+ return "", err
+ }
+ todo := make(chan int, len(blks))
+ for idx := range blks {
+ todo <- idx
+ }
+ go close(todo)
+
+ var services []arvados.KeepService
+ err = rcvr.client.EachKeepService(func(svc arvados.KeepService) error {
+ if svc.ServiceType == "proxy" {
+ rcvr.logger.WithField("service", svc).Debug("ignore proxy service")
+ } else {
+ services = append(services, svc)
+ }
+ return nil
+ })
+ if err != nil {
+ return "", fmt.Errorf("error getting list of keep services: %s", err)
+ }
+ rcvr.logger.WithField("services", services).Debug("got list of services")
+
+ // blobsigexp is our deadline for saving the rescued
+ // collection. This must be less than BlobSigningTTL
+ // (otherwise our rescued blocks could be garbage collected
+ // again before we protect them by saving the collection) but
+ // the exact value is somewhat arbitrary. If it's too soon, it
+ // will arrive before we're ready to save, and save will
+ // fail. If it's too late, we'll needlessly update timestamps
+ // on some blocks that were recently written/touched (e.g., by
+ // a previous attempt to rescue this same collection) and
+ // would have lived long enough anyway if left alone.
+ // BlobSigningTTL/2 (typically around 1 week) is much longer
+ // than than we need to recover even a very large collection.
+ blobsigttl := rcvr.cluster.Collections.BlobSigningTTL.Duration()
+ blobsigexp := time.Now().Add(blobsigttl / 2)
+ rcvr.logger.WithField("blobsigexp", blobsigexp).Debug("chose save deadline")
+
+ // We'll start a number of threads, each working on
+ // checking/recovering one block at a time. The threads
+ // themselves don't need much CPU/memory, but to avoid hitting
+ // limits on keepstore connections, backend storage bandwidth,
+ // etc., we limit concurrency to 2 per keepstore node.
+ workerThreads := 2 * len(services)
+
+ blkFound := make([]bool, len(blks))
+ var wg sync.WaitGroup
+ for i := 0; i < workerThreads; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ nextblk:
+ for idx := range todo {
+ blk := strings.SplitN(string(blks[idx]), "+", 2)[0]
+ logger := rcvr.logger.WithField("block", blk)
+ for _, untrashing := range []bool{false, true} {
+ for _, svc := range services {
+ logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
+ if untrashing {
+ if err := svc.Untrash(ctx, rcvr.client, blk); err != nil {
+ logger.WithError(err).Debug("untrash failed")
+ continue
+ }
+ logger.Info("untrashed")
+ }
+ err := rcvr.ensureSafe(ctx, logger, blk, svc, blobsigttl, blobsigexp)
+ if err == errNotFound {
+ logger.Debug(err)
+ } else if err != nil {
+ logger.Error(err)
+ } else {
+ blkFound[idx] = true
+ continue nextblk
+ }
+ }
+ }
+ logger.Debug("unrecoverable")
+ }
+ }()
+ }
+ wg.Wait()
+
+ var have, havenot int
+ for _, ok := range blkFound {
+ if ok {
+ have++
+ } else {
+ havenot++
+ }
+ }
+ if havenot > 0 {
+ if have > 0 {
+ rcvr.logger.Warn("partial recovery is not implemented")
+ }
+ return "", fmt.Errorf("unable to recover %d of %d blocks", havenot, have+havenot)
+ }
+
+ if rcvr.cluster.Collections.BlobSigning {
+ key := []byte(rcvr.cluster.Collections.BlobSigningKey)
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, rcvr.client.AuthToken, blobsigexp, blobsigttl, key)
+ }
+ rcvr.logger.WithField("manifest", coll.ManifestText).Debug("updated blob signatures in manifest")
+ err = rcvr.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": coll.ManifestText,
+ },
+ })
+ if err != nil {
+ return "", fmt.Errorf("error saving new collection: %s", err)
+ }
+ rcvr.logger.WithField("UUID", coll.UUID).Debug("created new collection")
+ return coll.UUID, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package recovercollection
+
+import (
+ "bytes"
+ "encoding/json"
+ "io/ioutil"
+ "os"
+ "testing"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (*Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+}
+
+func (*Suite) TestUnrecoverableBlock(c *check.C) {
+ tmp := c.MkDir()
+ mfile := tmp + "/manifest"
+ ioutil.WriteFile(mfile, []byte(". aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+410 0:410:Gone\n"), 0777)
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="not found" block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="untrash failed" block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg=unrecoverable block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="recovery failed".*`)
+}
+
+func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
+ tmp := c.MkDir()
+ mfile := tmp + "/manifest"
+ ioutil.WriteFile(mfile, []byte(". dcd0348cb2532ee90c99f1b846efaee7+13 0:13:test.txt\n"), 0777)
+
+ logger := ctxlog.TestLogger(c)
+ loader := config.NewLoader(&bytes.Buffer{}, logger)
+ cfg, err := loader.Load()
+ c.Assert(err, check.IsNil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ var datadirs []string
+ for _, v := range cluster.Volumes {
+ var params struct {
+ Root string
+ }
+ err := json.Unmarshal(v.DriverParameters, ¶ms)
+ c.Assert(err, check.IsNil)
+ if params.Root != "" {
+ datadirs = append(datadirs, params.Root)
+ err := os.Remove(params.Root + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if err != nil && !os.IsNotExist(err) {
+ c.Error(err)
+ }
+ }
+ }
+ c.Logf("keepstore datadirs are %q", datadirs)
+
+ // Currently StartKeep(2, true) uses dirs called "keep0" and
+ // "keep1" so we could just put our fake trashed file in keep0
+ // ... but we don't want to rely on arvadostest's
+ // implementation details, so we put a trashed file in every
+ // dir that keepstore might be using.
+ for _, datadir := range datadirs {
+ if fi, err := os.Stat(datadir); err != nil || !fi.IsDir() {
+ continue
+ }
+ c.Logf("placing backdated trashed block in datadir %q", datadir)
+ trashfile := datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7.trash.999999999"
+ os.Mkdir(datadir+"/dcd", 0777)
+ err = ioutil.WriteFile(trashfile, []byte("undelete test"), 0777)
+ c.Assert(err, check.IsNil)
+ t := time.Now().Add(-time.Hour * 24 * 365)
+ err = os.Chtimes(trashfile, t, t)
+ }
+
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, `zzzzz-4zz18-.{15}\n`)
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg=untrashed block=dcd0348.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="updated timestamp" block=dcd0348.*`)
+
+ found := false
+ for _, datadir := range datadirs {
+ buf, err := ioutil.ReadFile(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if err == nil {
+ found = true
+ c.Check(buf, check.DeepEquals, []byte("undelete test"))
+ fi, err := os.Stat(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if c.Check(err, check.IsNil) {
+ c.Logf("recovered block's modtime is %s", fi.ModTime())
+ c.Check(time.Now().Sub(fi.ModTime()) < time.Hour, check.Equals, true)
+ }
+ }
+ }
+ c.Check(found, check.Equals, true)
+}
+
+func (*Suite) TestUnusableManifestSourceArg(c *check.C) {
+ for _, trial := range []struct {
+ srcArg string
+ errRegexp string
+ }{
+ {"zzzzz-4zz18-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found".*`},
+ {"zzzzz-57u5n-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found.*`},
+ {"zzzzz-57u5n-containerlog006", `(?ms).*msg="log entry properties\.old_attributes\.manifest_text missing or empty".*`},
+ {"zzzzz-j7d0g-aaaaaaaaaaaaaaa", `(?ms).*msg="looks like a UUID but not a log or collection UUID.*`},
+ } {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", trial.srcArg}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, trial.errRegexp)
+ }
+}
} else if url, err := url.Parse(want); err != nil {
return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
} else {
+ if url.Path == "" {
+ url.Path = "/"
+ }
return arvados.URL(*url), nil
}
if (!requireNamespace("markdown")) {
install.packages("markdown")
}
+if (!requireNamespace("XML")) {
+ # XML 3.99-0.4 depends on R >= 4.0.0, but we run tests on debian
+ # stable (10) with R 3.5.2 so we install an older version from
+ # source.
+ install.packages("https://cran.r-project.org/src/contrib/Archive/XML/XML_3.99-0.3.tar.gz", repos=NULL, type="source")
+}
devtools::install_dev_deps()
import cwltool.process
import cwltool.argparser
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing
import arvados
import arvados.config
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
- customschema10 = res10.read()
- customschema11 = res11.read()
+ customschema10 = res10.read().decode('utf-8')
+ customschema11 = res11.read().decode('utf-8')
use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class
from cwltool.command_line_tool import compute_checksums
from cwltool.load_tool import load_tool
import arvados.collection
from .util import collectionUUID
import ruamel.yaml as yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
#
# Found a file, check for secondaryFiles
#
- primary["secondaryFiles"] = []
+ specs = []
+ primary["secondaryFiles"] = secondaryspec
for i, sf in enumerate(aslist(secondaryspec)):
pattern = builder.do_eval(sf["pattern"], context=primary)
if pattern is None:
continue
+ if isinstance(pattern, list):
+ specs.extend(pattern)
+ elif isinstance(pattern, dict):
+ specs.append(pattern)
+ elif isinstance(pattern, str):
+ specs.append({"pattern": pattern})
+ else:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Expression must return list, object, string or null")
+
+ found = []
+ for i, sf in enumerate(specs):
+ if isinstance(sf, dict):
+ if sf.get("class") == "File":
+ pattern = sf["basename"]
+ else:
+ pattern = sf["pattern"]
+ required = sf.get("required")
+ elif isinstance(sf, str):
+ pattern = sf
+ required = True
+ else:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Expression must return list, object, string or null")
+
sfpath = substitute(primary["location"], pattern)
- required = builder.do_eval(sf.get("required"), context=primary)
+ required = builder.do_eval(required, context=primary)
if fsaccess.exists(sfpath):
- primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+ found.append({"location": sfpath, "class": "File"})
elif required:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Required secondary file '%s' does not exist" % sfpath)
- primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+ primary["secondaryFiles"] = cmap(found)
if discovered is not None:
discovered[primary["location"]] = primary["secondaryFiles"]
elif inputschema["type"] not in primitive_types_set:
discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
if "$schemas" in workflowobj:
- sch = []
+ sch = CommentedSeq()
for s in workflowobj["$schemas"]:
sch.append(mapper.mapper(s).resolved)
workflowobj["$schemas"] = sch
def visit(v, cur_id):
if isinstance(v, dict):
if v.get("class") in ("CommandLineTool", "Workflow"):
- if "id" not in v:
- raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
- cur_id = rewrite_to_orig.get(v["id"], v["id"])
+ if tool.metadata["cwlVersion"] == "v1.0" and "id" not in v:
+ raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field, add an 'id' or use to cwlVersion: v1.1")
+ if "id" in v:
+ cur_id = rewrite_to_orig.get(v["id"], v["id"])
+ if "path" in v and "location" not in v:
+ v["location"] = v["path"]
+ del v["path"]
if "location" in v and not v["location"].startswith("keep:"):
v["location"] = merged_map[cur_id].resolved[v["location"]]
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
debian8)
fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
;;
+ debian9 | ubuntu1604)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
debian* | ubuntu*)
- fpm_depends+=(libcurl3-gnutls libpython2.7)
+ fpm_depends+=(libcurl3-gnutls python3-distutils)
;;
esac
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200317203547',
- 'schema-salad==5.0.20200302192450',
+ 'cwltool==3.0.20200530110633',
+ 'schema-salad==6.0.20200601095207',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
set -x
-if [ \$PYCMD = "python3" ]; then
+if [ "\$PYCMD" = "python3" ]; then
pip3 install cwltest
else
pip install cwltest
git clone https://github.com/common-workflow-language/cwl-\${version}.git
fi
cd cwl-\${version}
+elif [[ "$suite" != "integration" ]] ; then
+ echo "ERROR: unknown suite '$suite'"
+ exit 1
fi
if [[ "$suite" != "integration" ]] ; then
elif test "$tag" = "latest" ; then
arv-keepdocker --pull arvados/jobs $tag
else
- jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
- arv-keepdocker --pull arvados/jobs \$jobsimg
- docker tag arvados/jobs:\$jobsimg arvados/jobs:latest
+ set +u
+ export WORKSPACE=/usr/src/arvados
+ . /usr/src/arvados/build/run-library.sh
+ TMPHERE=\$(pwd)
+ cd /usr/src/arvados
+ calculate_python_sdk_cwl_package_versions
+ cd \$TMPHERE
+ set -u
+
+ arv-keepdocker --pull arvados/jobs \$cwl_runner_version
+ docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
arv-keepdocker arvados/jobs latest
fi
EXTRA=--compute-checksum
-if [[ $devcwl == 1 ]] ; then
+if [[ $devcwl -eq 1 ]] ; then
EXTRA="\$EXTRA --enable-dev"
fi
output:
out: null
tool: wf-defaults/wf4.cwl
- doc: default in embedded subworkflow missing 'id' field
+ doc: default in embedded subworkflow missing 'id' field, v1.0
should_fail: true
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf8.cwl
+ doc: default in embedded subworkflow missing 'id' field, v1.1
+ should_fail: false
+
- job: null
output:
out: null
logincluster:
type: boolean
default: false
+ arvbox_mode:
+ type: string?
+ default: "dev"
outputs:
arvados_api_token:
type: string
arvbox_data: mkdir/arvbox_data
arvbox_bin: arvbox
branch: branch
+ arvbox_mode: arvbox_mode
out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
branch:
type: string
default: master
+ arvbox_mode:
+ type: string?
+ default: "dev"
outputs:
cluster_id:
type: string
- shellQuote: false
valueFrom: |
set -ex
- mkdir -p $ARVBOX_DATA
- if ! test -d $ARVBOX_DATA/arvados ; then
- cd $ARVBOX_DATA
- git clone https://git.arvados.org/arvados.git
+ if test $(inputs.arvbox_mode) = dev ; then
+ mkdir -p $ARVBOX_DATA
+ if ! test -d $ARVBOX_DATA/arvados ; then
+ cd $ARVBOX_DATA
+ git clone https://git.arvados.org/arvados.git
+ fi
+ cd $ARVBOX_DATA/arvados
+ gitver=`git rev-parse HEAD`
+ git fetch
+ git checkout -f $(inputs.branch)
+ git pull
+ pulled=`git rev-parse HEAD`
+ git --no-pager log -n1 $pulled
+ else
+ export ARVBOX_BASE=$(runtime.tmpdir)
+ unset ARVBOX_DATA
fi
- cd $ARVBOX_DATA/arvados
- gitver=`git rev-parse HEAD`
- git fetch
- git checkout -f $(inputs.branch)
- git pull
- pulled=`git rev-parse HEAD`
- git --no-pager log -n1 $pulled
-
cd $(runtime.outdir)
if test "$gitver" = "$pulled" ; then
- $(inputs.arvbox_bin.path) start dev
+ $(inputs.arvbox_bin.path) start $(inputs.arvbox_mode)
else
- $(inputs.arvbox_bin.path) restart dev
+ $(inputs.arvbox_bin.path) restart $(inputs.arvbox_mode)
fi
$(inputs.arvbox_bin.path) status > status.txt
$(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
class: Directory
location: inp1
outputs: []
- arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
+ arguments: [echo, $(inputs.inp2)]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run:
+ class: CommandLineTool
+ inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ outputs: []
+ arguments: [echo, $(inputs.inp2)]
step1:
in: []
out: []
- run: default-dir4.cwl
\ No newline at end of file
+ run: default-dir4.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir8.cwl
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Generate and verify permission signatures for Keep locators.
+//
+// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
+
+package arvados
+
+import (
+ "crypto/hmac"
+ "crypto/sha1"
+ "errors"
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+var (
+ // ErrSignatureExpired - a signature was rejected because the
+ // expiry time has passed.
+ ErrSignatureExpired = errors.New("Signature expired")
+ // ErrSignatureInvalid - a signature was rejected because it
+ // was badly formatted or did not match the given secret key.
+ ErrSignatureInvalid = errors.New("Invalid signature")
+ // ErrSignatureMissing - the given locator does not have a
+ // signature hint.
+ ErrSignatureMissing = errors.New("Missing signature")
+)
+
+// makePermSignature generates a SHA-1 HMAC digest for the given blob,
+// token, expiry, and site secret.
+func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
+ hmac := hmac.New(sha1.New, permissionSecret)
+ hmac.Write([]byte(blobHash))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(apiToken))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(expiry))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(blobSignatureTTL))
+ digest := hmac.Sum(nil)
+ return fmt.Sprintf("%x", digest)
+}
+
+var (
+ mBlkRe = regexp.MustCompile(`^[0-9a-f]{32}.*`)
+ mPermHintRe = regexp.MustCompile(`\+A[^+]*`)
+)
+
+// SignManifest signs all locators in the given manifest, discarding
+// any existing signatures.
+func SignManifest(manifest string, apiToken string, expiry time.Time, ttl time.Duration, permissionSecret []byte) string {
+ return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
+ if mBlkRe.MatchString(tok) {
+ return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
+ } else {
+ return tok
+ }
+ })
+}
+
+// SignLocator returns blobLocator with a permission signature
+// added. If either permissionSecret or apiToken is empty, blobLocator
+// is returned untouched.
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
+ if len(permissionSecret) == 0 || apiToken == "" {
+ return blobLocator
+ }
+ // Strip off all hints: only the hash is used to sign.
+ blobHash := strings.Split(blobLocator, "+")[0]
+ timestampHex := fmt.Sprintf("%08x", expiry.Unix())
+ blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+ return blobLocator +
+ "+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
+ "@" + timestampHex
+}
+
+var SignedLocatorRe = regexp.MustCompile(
+ //1 2 34 5 6 7 89
+ `^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
+
+// VerifySignature returns nil if the signature on the signedLocator
+// can be verified using the given apiToken. Otherwise it returns
+// ErrSignatureExpired (if the signature's expiry time has passed,
+// which is something the client could have figured out
+// independently), ErrSignatureMissing (if there is no signature hint
+// at all), or ErrSignatureInvalid (if the signature is present but
+// badly formatted or incorrect).
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
+ matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
+ if matches == nil {
+ return ErrSignatureMissing
+ }
+ blobHash := matches[1]
+ signatureHex := matches[6]
+ expiryHex := matches[7]
+ if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
+ return ErrSignatureInvalid
+ } else if expiryTime.Before(time.Now()) {
+ return ErrSignatureExpired
+ }
+ blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+ if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
+ return ErrSignatureInvalid
+ }
+ return nil
+}
+
+func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
+ if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
+ ts = time.Unix(tsInt, 0)
+ } else {
+ err = e
+ }
+ return ts, err
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "time"
+
+ check "gopkg.in/check.v1"
+)
+
+const (
+ knownHash = "acbd18db4cc2f85cedef654fccc4a4d8"
+ knownLocator = knownHash + "+3"
+ knownToken = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
+ knownKey = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
+ "p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
+ "ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
+ "jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
+ "gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
+ "vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
+ "786u5rw2a9gx743dj3fgq2irk"
+ knownSignature = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
+ knownTimestamp = "7fffffff"
+ knownSigHint = "+A" + knownSignature + "@" + knownTimestamp
+ knownSignedLocator = knownLocator + knownSigHint
+ blobSignatureTTL = 1209600 * time.Second
+)
+
+var _ = check.Suite(&BlobSignatureSuite{})
+
+type BlobSignatureSuite struct{}
+
+func (s *BlobSignatureSuite) TestSignLocator(c *check.C) {
+ ts, err := parseHexTimestamp(knownTimestamp)
+ c.Check(err, check.IsNil)
+ c.Check(SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)), check.Equals, knownSignedLocator)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignature(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExtraHints(c *check.C) {
+ // handle hint before permission signature
+ c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle hint after permission signature
+ c.Check(VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle hints around permission signature
+ c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+// The size hint on the locator string should not affect signature
+// validation.
+func (s *BlobSignatureSuite) TestVerifySignatureWrongSize(c *check.C) {
+ // handle incorrect size hint
+ c.Check(VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle missing size hint
+ c.Check(VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSig(c *check.C) {
+ badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
+ c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadTimestamp(c *check.C) {
+ badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
+ c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSecret(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadToken(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExpired(c *check.C) {
+ yesterday := time.Now().AddDate(0, 0, -1)
+ expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
+ c.Check(VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureExpired)
+}
// HTTP headers to add/override in outgoing requests.
SendHeader http.Header
+ // Timeout for requests. NewClientFromConfig and
+ // NewClientFromEnv return a Client with a default 5 minute
+ // timeout. To disable this timeout and rely on each
+ // http.Request's context deadline instead, set Timeout to
+ // zero.
+ Timeout time.Duration
+
dd *DiscoveryDocument
- ctx context.Context
+ defaultRequestID string
}
// The default http.Client used by a Client with Insecure==true and
var InsecureHTTPClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true}},
- Timeout: 5 * time.Minute}
+ InsecureSkipVerify: true}}}
// The default http.Client used by a Client otherwise.
-var DefaultSecureClient = &http.Client{
- Timeout: 5 * time.Minute}
+var DefaultSecureClient = &http.Client{}
// NewClientFromConfig creates a new Client that uses the endpoints in
// the given cluster.
Scheme: ctrlURL.Scheme,
APIHost: ctrlURL.Host,
Insecure: cluster.TLS.Insecure,
+ Timeout: 5 * time.Minute,
}, nil
}
AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
Insecure: insecure,
KeepServiceURIs: svcs,
+ Timeout: 5 * time.Minute,
}
}
}
if req.Header.Get("X-Request-Id") == "" {
- reqid, _ := req.Context().Value(contextKeyRequestID{}).(string)
- if reqid == "" {
- reqid, _ = c.context().Value(contextKeyRequestID{}).(string)
- }
- if reqid == "" {
+ var reqid string
+ if ctxreqid, _ := req.Context().Value(contextKeyRequestID{}).(string); ctxreqid != "" {
+ reqid = ctxreqid
+ } else if c.defaultRequestID != "" {
+ reqid = c.defaultRequestID
+ } else {
reqid = reqIDGen.Next()
}
if req.Header == nil {
req.Header.Set("X-Request-Id", reqid)
}
}
- return c.httpClient().Do(req)
+ var cancel context.CancelFunc
+ if c.Timeout > 0 {
+ ctx := req.Context()
+ ctx, cancel = context.WithDeadline(ctx, time.Now().Add(c.Timeout))
+ req = req.WithContext(ctx)
+ }
+ resp, err := c.httpClient().Do(req)
+ if err == nil && cancel != nil {
+ // We need to call cancel() eventually, but we can't
+ // use "defer cancel()" because the context has to
+ // stay alive until the caller has finished reading
+ // the response body.
+ resp.Body = cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
+ } else if cancel != nil {
+ cancel()
+ }
+ return resp, err
+}
+
+// cancelOnClose calls a provided CancelFunc when its wrapped
+// ReadCloser's Close() method is called.
+type cancelOnClose struct {
+ io.ReadCloser
+ cancel context.CancelFunc
+}
+
+func (coc cancelOnClose) Close() error {
+ err := coc.ReadCloser.Close()
+ coc.cancel()
+ return err
}
func isRedirectStatus(code int) bool {
//
// path must not contain a query string.
func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
- return c.RequestAndDecodeContext(c.context(), dst, method, path, body, params)
+ return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params)
}
func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error {
// header.
func (c *Client) WithRequestID(reqid string) *Client {
cc := *c
- cc.ctx = ContextWithRequestID(cc.context(), reqid)
+ cc.defaultRequestID = reqid
return &cc
}
-func (c *Client) context() context.Context {
- if c.ctx == nil {
- return context.Background()
- }
- return c.ctx
-}
-
func (c *Client) httpClient() *http.Client {
switch {
case c.Client != nil:
}()
type Config struct {
- Clusters map[string]Cluster
+ Clusters map[string]Cluster
+ AutoReloadConfig bool
}
// GetConfig returns the current system config, loading it from
MaxPermissionEntries int
MaxUUIDEntries int
}
+
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
BalancePeriod Duration
BalanceCollectionBatch int
BalanceCollectionBuffers int
+ BalanceTimeout Duration
WebDAVCache WebDAVCacheConfig
}
Repositories string
}
Login struct {
- GoogleClientID string
- GoogleClientSecret string
- GoogleAlternateEmailAddresses bool
- PAM bool
- PAMService string
- PAMDefaultEmailDomain string
- ProviderAppID string
- ProviderAppSecret string
- LoginCluster string
- RemoteTokenRefresh Duration
+ LDAP struct {
+ Enable bool
+ URL URL
+ StartTLS bool
+ InsecureTLS bool
+ StripDomain string
+ AppendDomain string
+ SearchAttribute string
+ SearchBindUser string
+ SearchBindPassword string
+ SearchBase string
+ SearchFilters string
+ EmailAttribute string
+ UsernameAttribute string
+ }
+ Google struct {
+ Enable bool
+ ClientID string
+ ClientSecret string
+ AlternateEmailAddresses bool
+ }
+ OpenIDConnect struct {
+ Enable bool
+ Issuer string
+ ClientID string
+ ClientSecret string
+ EmailClaim string
+ EmailVerifiedClaim string
+ UsernameClaim string
+ }
+ PAM struct {
+ Enable bool
+ Service string
+ DefaultEmailDomain string
+ }
+ SSO struct {
+ Enable bool
+ ProviderAppID string
+ ProviderAppSecret string
+ }
+ LoginCluster string
+ RemoteTokenRefresh Duration
}
Mail struct {
MailchimpAPIKey string
}
type S3VolumeDriverParameters struct {
+ IAMRole string
AccessKey string
SecretKey string
Endpoint string
Region string
Bucket string
LocationConstraint bool
+ V2Signature bool
IndexPageSize int
ConnectTimeout Duration
ReadTimeout Duration
u, err := url.Parse(string(text))
if err == nil {
*su = URL(*u)
+ if su.Path == "" && su.Host != "" {
+ // http://example really means http://example/
+ su.Path = "/"
+ }
}
return err
}
package arvados
import (
+ "encoding/json"
+
"github.com/ghodss/yaml"
check "gopkg.in/check.v1"
)
c.Check(itm["foo8"].IncludedScratch, check.Equals, ByteSize(0))
}
}
+
+func (s *ConfigSuite) TestURLTrailingSlash(c *check.C) {
+ var a, b map[URL]bool
+ json.Unmarshal([]byte(`{"https://foo.example": true}`), &a)
+ json.Unmarshal([]byte(`{"https://foo.example/": true}`), &b)
+ c.Check(a, check.DeepEquals, b)
+}
SchedulingParameters SchedulingParameters `json:"scheduling_parameters"`
ExitCode int `json:"exit_code"`
RuntimeStatus map[string]interface{} `json:"runtime_status"`
+ StartedAt *time.Time `json:"started_at"` // nil if not yet started
+ FinishedAt *time.Time `json:"finished_at"` // nil if not yet finished
}
// Container is an arvados#container resource.
Limit int `json:"limit"`
}
+// ContainerRequestList is an arvados#containerRequestList resource.
+type ContainerRequestList struct {
+ Items []ContainerRequest `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
+
// ContainerState is a string corresponding to a valid Container state.
type ContainerState string
import (
"bufio"
+ "context"
"fmt"
+ "io/ioutil"
"net/http"
"strconv"
"strings"
return mounts, nil
}
+// Touch updates the timestamp on the given block.
+func (s *KeepService) Touch(ctx context.Context, c *Client, blk string) error {
+ req, err := http.NewRequest("TOUCH", s.url(blk), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := c.Do(req.WithContext(ctx))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+ }
+ return nil
+}
+
+// Untrash moves/copies the given block out of trash.
+func (s *KeepService) Untrash(ctx context.Context, c *Client, blk string) error {
+ req, err := http.NewRequest("PUT", s.url("untrash/"+blk), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := c.Do(req.WithContext(ctx))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+ }
+ return nil
+}
+
// Index returns an unsorted list of blocks at the given mount point.
-func (s *KeepService) IndexMount(c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
- return s.index(c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
+func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
+ return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
}
// Index returns an unsorted list of blocks that can be retrieved from
// this server.
-func (s *KeepService) Index(c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
- return s.index(c, s.url("index/"+prefix))
+func (s *KeepService) Index(ctx context.Context, c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
+ return s.index(ctx, c, s.url("index/"+prefix))
}
-func (s *KeepService) index(c *Client, url string) ([]KeepServiceIndexEntry, error) {
- req, err := http.NewRequest("GET", url, nil)
+func (s *KeepService) index(ctx context.Context, c *Client, url string) ([]KeepServiceIndexEntry, error) {
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
- return nil, fmt.Errorf("NewRequest(%v): %v", url, err)
+ return nil, fmt.Errorf("NewRequestWithContext(%v): %v", url, err)
}
resp, err := c.Do(req)
if err != nil {
package arvados
import (
+ "context"
"net/http"
check "gopkg.in/check.v1"
APIHost: "zzzzz.arvadosapi.com",
AuthToken: "xyzzy",
}
- _, err := (&KeepService{}).IndexMount(client, "fake", "")
+ _, err := (&KeepService{}).IndexMount(context.Background(), client, "fake", "")
c.Check(err, check.ErrorMatches, `.*timeout.*`)
}
// Link is an arvados#link record
type Link struct {
- UUID string `json:"uuid,omiempty"`
- OwnerUUID string `json:"owner_uuid"`
- Name string `json:"name"`
- LinkClass string `json:"link_class"`
- HeadUUID string `json:"head_uuid"`
- HeadKind string `json:"head_kind"`
- TailUUID string `json:"tail_uuid"`
- TailKind string `json:"tail_kind"`
+ UUID string `json:"uuid,omiempty"`
+ OwnerUUID string `json:"owner_uuid"`
+ Name string `json:"name"`
+ LinkClass string `json:"link_class"`
+ HeadUUID string `json:"head_uuid"`
+ HeadKind string `json:"head_kind"`
+ TailUUID string `json:"tail_uuid"`
+ TailKind string `json:"tail_kind"`
+ Properties map[string]interface{} `json:"properties"`
}
// UserList is an arvados#userList resource.
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "time"
+
+// VirtualMachine is an arvados#virtualMachine resource.
+type VirtualMachine struct {
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ Hostname string `json:"hostname"`
+ CreatedAt *time.Time `json:"created_at"`
+ ModifiedAt *time.Time `json:"modified_at"`
+ ModifiedByUserUUID string `json:"modified_by_user_uuid"`
+}
+
+// VirtualMachineList is an arvados#virtualMachineList resource.
+type VirtualMachineList struct {
+ Items []VirtualMachine `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
import (
"sync"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
)
// A ClientPool is a pool of ArvadosClients. This is useful for
// credentials. See arvados-git-httpd for an example, and sync.Pool
// for more information about garbage collection.
type ClientPool struct {
- // Initialize new clients by coping this one.
+ // Initialize new clients by copying this one.
Prototype *ArvadosClient
pool *sync.Pool
// MakeClientPool returns a new empty ClientPool, using environment
// variables to initialize the prototype.
func MakeClientPool() *ClientPool {
- proto, err := MakeArvadosClient()
+ return MakeClientPoolWith(nil)
+}
+
+// MakeClientPoolWith returns a new empty ClientPool with a previously
+// initialized arvados.Client.
+func MakeClientPoolWith(client *arvados.Client) *ClientPool {
+ var err error
+ var proto *ArvadosClient
+
+ if client == nil {
+ proto, err = MakeArvadosClient()
+ } else {
+ proto, err = New(client)
+ }
return &ClientPool{
Prototype: proto,
lastErr: err,
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvadostest
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/ctrlctx"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/jmoiron/sqlx"
+ _ "github.com/lib/pq"
+ "gopkg.in/check.v1"
+)
+
+// DB returns a DB connection for the given cluster config.
+func DB(c *check.C, cluster *arvados.Cluster) *sqlx.DB {
+ db, err := sqlx.Open("postgres", cluster.PostgreSQL.Connection.String())
+ c.Assert(err, check.IsNil)
+ return db
+}
+
+// TransactionContext returns a context suitable for running a test
+// case in a new transaction, and a rollback func which the caller
+// should call after the test.
+func TransactionContext(c *check.C, db *sqlx.DB) (ctx context.Context, rollback func()) {
+ tx, err := db.Beginx()
+ c.Assert(err, check.IsNil)
+ return ctrlctx.NewWithTransaction(context.Background(), tx), func() {
+ c.Check(tx.Rollback(), check.IsNil)
+ }
+}
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
)
-// ClearCache clears the Keep service discovery cache.
+// RefreshServiceDiscovery clears the Keep service discovery cache.
func RefreshServiceDiscovery() {
var wg sync.WaitGroup
defer wg.Wait()
}
}
-// ClearCacheOnSIGHUP installs a signal handler that calls
-// ClearCache when SIGHUP is received.
+// RefreshServiceDiscoveryOnSIGHUP installs a signal handler that calls
+// RefreshServiceDiscovery when SIGHUP is received.
func RefreshServiceDiscoveryOnSIGHUP() {
svcListCacheMtx.Lock()
defer svcListCacheMtx.Unlock()
//
// SPDX-License-Identifier: Apache-2.0
-// Generate and verify permission signatures for Keep locators.
-//
-// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
-
package keepclient
-import (
- "crypto/hmac"
- "crypto/sha1"
- "errors"
- "fmt"
- "regexp"
- "strconv"
- "strings"
- "time"
-)
+import "git.arvados.org/arvados.git/sdk/go/arvados"
var (
- // ErrSignatureExpired - a signature was rejected because the
- // expiry time has passed.
- ErrSignatureExpired = errors.New("Signature expired")
- // ErrSignatureInvalid - a signature was rejected because it
- // was badly formatted or did not match the given secret key.
- ErrSignatureInvalid = errors.New("Invalid signature")
- // ErrSignatureMissing - the given locator does not have a
- // signature hint.
- ErrSignatureMissing = errors.New("Missing signature")
+ ErrSignatureExpired = arvados.ErrSignatureExpired
+ ErrSignatureInvalid = arvados.ErrSignatureInvalid
+ ErrSignatureMissing = arvados.ErrSignatureMissing
+ SignLocator = arvados.SignLocator
+ SignedLocatorRe = arvados.SignedLocatorRe
+ VerifySignature = arvados.VerifySignature
)
-
-// makePermSignature generates a SHA-1 HMAC digest for the given blob,
-// token, expiry, and site secret.
-func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
- hmac := hmac.New(sha1.New, permissionSecret)
- hmac.Write([]byte(blobHash))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(apiToken))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(expiry))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(blobSignatureTTL))
- digest := hmac.Sum(nil)
- return fmt.Sprintf("%x", digest)
-}
-
-// SignLocator returns blobLocator with a permission signature
-// added. If either permissionSecret or apiToken is empty, blobLocator
-// is returned untouched.
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
- if len(permissionSecret) == 0 || apiToken == "" {
- return blobLocator
- }
- // Strip off all hints: only the hash is used to sign.
- blobHash := strings.Split(blobLocator, "+")[0]
- timestampHex := fmt.Sprintf("%08x", expiry.Unix())
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- return blobLocator +
- "+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
- "@" + timestampHex
-}
-
-var SignedLocatorRe = regexp.MustCompile(
- //1 2 34 5 6 7 89
- `^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
-
-// VerifySignature returns nil if the signature on the signedLocator
-// can be verified using the given apiToken. Otherwise it returns
-// ErrSignatureExpired (if the signature's expiry time has passed,
-// which is something the client could have figured out
-// independently), ErrSignatureMissing (if there is no signature hint
-// at all), or ErrSignatureInvalid (if the signature is present but
-// badly formatted or incorrect).
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
- matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
- if matches == nil {
- return ErrSignatureMissing
- }
- blobHash := matches[1]
- signatureHex := matches[6]
- expiryHex := matches[7]
- if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
- return ErrSignatureInvalid
- } else if expiryTime.Before(time.Now()) {
- return ErrSignatureExpired
- }
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
- return ErrSignatureInvalid
- }
- return nil
-}
-
-func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
- if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
- ts = time.Unix(tsInt, 0)
- } else {
- err = e
- }
- return ts, err
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package keepclient
-
-import (
- "testing"
- "time"
-)
-
-const (
- knownHash = "acbd18db4cc2f85cedef654fccc4a4d8"
- knownLocator = knownHash + "+3"
- knownToken = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
- knownKey = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
- "p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
- "ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
- "jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
- "gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
- "vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
- "786u5rw2a9gx743dj3fgq2irk"
- knownSignature = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
- knownTimestamp = "7fffffff"
- knownSigHint = "+A" + knownSignature + "@" + knownTimestamp
- knownSignedLocator = knownLocator + knownSigHint
- blobSignatureTTL = 1209600 * time.Second
-)
-
-func TestSignLocator(t *testing.T) {
- if ts, err := parseHexTimestamp(knownTimestamp); err != nil {
- t.Errorf("bad knownTimestamp %s", knownTimestamp)
- } else {
- if knownSignedLocator != SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)) {
- t.Fail()
- }
- }
-}
-
-func TestVerifySignature(t *testing.T) {
- if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fail()
- }
-}
-
-func TestVerifySignatureExtraHints(t *testing.T) {
- if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hint before permission signature")
- }
-
- if VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hint after permission signature")
- }
-
- if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hints around permission signature")
- }
-}
-
-// The size hint on the locator string should not affect signature validation.
-func TestVerifySignatureWrongSize(t *testing.T) {
- if VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle incorrect size hint")
- }
-
- if VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle missing size hint")
- }
-}
-
-func TestVerifySignatureBadSig(t *testing.T) {
- badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
- if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadTimestamp(t *testing.T) {
- badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
- if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadSecret(t *testing.T) {
- if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")) != ErrSignatureInvalid {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadToken(t *testing.T) {
- if VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)) != ErrSignatureInvalid {
- t.Fail()
- }
-}
-
-func TestVerifySignatureExpired(t *testing.T) {
- yesterday := time.Now().AddDate(0, 0, -1)
- expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
- if VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureExpired {
- t.Fail()
- }
-}
import urllib.parse
import os
import hashlib
+import re
from arvados._version import __version__
EMAIL=0
def update_username(args, email, user_uuid, username, migratecluster, migratearv):
print("(%s) Updating username of %s to '%s' on %s" % (email, user_uuid, username, migratecluster))
- if not args.dry_run:
- try:
- conflicts = migratearv.users().list(filters=[["username", "=", username]], bypass_federation=True).execute()
- if conflicts["items"]:
- # There's already a user with the username, move the old user out of the way
- migratearv.users().update(uuid=conflicts["items"][0]["uuid"],
- bypass_federation=True,
- body={"user": {"username": username+"migrate"}}).execute()
- migratearv.users().update(uuid=user_uuid,
- bypass_federation=True,
- body={"user": {"username": username}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, user_uuid, username, migratecluster, e))
+ if args.dry_run:
+ return
+ try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]], bypass_federation=True).execute()
+ if conflicts["items"]:
+ # There's already a user with the username, move the old user out of the way
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"],
+ bypass_federation=True,
+ body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=user_uuid,
+ bypass_federation=True,
+ body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, user_uuid, username, migratecluster, e))
def choose_new_user(args, by_email, email, userhome, username, old_user_uuid, clusters):
conflicts = homearv.users().list(filters=[["username", "=", username]],
bypass_federation=True).execute()
if conflicts["items"]:
- homearv.users().update(uuid=conflicts["items"][0]["uuid"],
- bypass_federation=True,
- body={"user": {"username": username+"migrate"}}).execute()
- user = homearv.users().create(body={"user": {"email": email, "username": username,
- "is_active": olduser["is_active"]}}).execute()
+ homearv.users().update(
+ uuid=conflicts["items"][0]["uuid"],
+ bypass_federation=True,
+ body={"user": {"username": username+"migrate"}}).execute()
+ user = homearv.users().create(
+ body={"user": {
+ "email": email,
+ "first_name": olduser["first_name"],
+ "last_name": olduser["last_name"],
+ "username": username,
+ "is_active": olduser["is_active"]}}).execute()
except arvados.errors.ApiError as e:
print("(%s) Could not create user: %s" % (email, str(e)))
return None
newuser = arvados.api(host=ru.netloc, token=salted,
insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
else:
- newuser = {"is_active": True, "username": username}
+ newuser = {"is_active": True, "username": email.split('@')[0], "is_admin": False}
except arvados.errors.ApiError as e:
print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
return None
return None
if olduser["is_admin"] and not newuser["is_admin"]:
- print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+ print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s. Please ensure the user admin status is the same on both clusters. Note that a federated admin account has admin privileges on the entire federation." % (email, old_user_uuid, new_user_uuid, migratecluster))
return None
return newuser
def migrate_user(args, migratearv, email, new_user_uuid, old_user_uuid):
+ if args.dry_run:
+ return
try:
- if not args.dry_run:
+ new_owner_uuid = new_user_uuid
+ if args.data_into_subproject:
grp = migratearv.groups().create(body={
"owner_uuid": new_user_uuid,
"name": "Migrated from %s (%s)" % (email, old_user_uuid),
"group_class": "project"
}, ensure_unique_name=True).execute()
- migratearv.users().merge(old_user_uuid=old_user_uuid,
- new_user_uuid=new_user_uuid,
- new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
+ new_owner_uuid = grp["uuid"]
+ migratearv.users().merge(old_user_uuid=old_user_uuid,
+ new_user_uuid=new_user_uuid,
+ new_owner_uuid=new_owner_uuid,
+ redirect_to_new_user=True).execute()
except arvados.errors.ApiError as e:
- print("(%s) Error migrating user: %s" % (email, e))
+ name_collision = re.search(r'Key \(owner_uuid, name\)=\((.*?), (.*?)\) already exists\.\n.*UPDATE "(.*?)"', e._get_reason())
+ if name_collision:
+ target_owner, rsc_name, rsc_type = name_collision.groups()
+ print("(%s) Cannot migrate to %s because both origin and target users have a %s named '%s'. Please rename the conflicting items or use --data-into-subproject to migrate all users' data into a special subproject." % (email, target_owner, rsc_type[:-1], rsc_name))
+ else:
+ print("(%s) Skipping user migration because of error: %s" % (email, e))
def main():
-
parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
parser.add_argument(
'--version', action='version', version="%s %s" % (sys.argv[0], __version__),
help='Print version and exit.')
- parser.add_argument('--tokens', type=str, required=False)
+ parser.add_argument('--tokens', type=str, metavar='FILE', required=False, help="Read tokens from FILE. Not needed when using LoginCluster.")
+ parser.add_argument('--data-into-subproject', action="store_true", help="Migrate user's data into a separate subproject. This can be used to avoid name collisions from within an account.")
group = parser.add_mutually_exclusive_group(required=True)
- group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
- group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
- group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
- group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
+ group.add_argument('--report', type=str, metavar='FILE', help="Generate report .csv file listing users by email address and their associated Arvados accounts.")
+ group.add_argument('--migrate', type=str, metavar='FILE', help="Consume report .csv and migrate users to designated Arvados accounts.")
+ group.add_argument('--dry-run', type=str, metavar='FILE', help="Consume report .csv and report how user would be migrated to designated Arvados accounts.")
+ group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected.")
args = parser.parse_args()
clusters, errors, loginCluster = connect_clusters(args)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if self.insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ else:
+ curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
if method == "HEAD":
curl.setopt(pycurl.NOBODY, True)
self._setcurltimeouts(curl, timeout, method=="HEAD")
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if self.insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ else:
+ curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
self._setcurltimeouts(curl, timeout)
try:
curl.perform()
it returns the value of `fallback` (httplib2's CA certs by default).
"""
for ca_certs_path in [
+ # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+ # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+ os.environ.get('SSL_CERT_FILE'),
# Arvados specific:
'/etc/arvados/ca-certificates.crt',
# Debian:
# Red Hat:
'/etc/pki/tls/certs/ca-bundle.crt',
]:
- if os.path.exists(ca_certs_path):
+ if ca_certs_path and os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
'ruamel.yaml >=0.15.54, <=0.16.5',
'setuptools',
'ws4py >=0.4.2',
+ 'rsa < 4.1'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
branch:
type: string
default: master
+ arvbox_mode:
+ type: string?
outputs:
arvados_api_hosts:
type: string[]
in:
arvbox_base: arvbox_base
branch: branch
+ arvbox_mode: arvbox_mode
logincluster:
default: true
out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers, arvbox_bin]
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import json
import sys
for i in range(1, 10):
found = False
for u in users["items"]:
- if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i):
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser":
found = True
by_username[u["username"]] = u["uuid"]
assert found
found = False
for u in users["items"]:
if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+ u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
found = True
assert found, "Not found case%i" % i
found = False
for u in users["items"]:
if (u["username"] == "case9" and u["email"] == "case9@test" and
+ u["first_name"] == "Case9" and u["last_name"] == "Testuser" and
u["uuid"] == by_username[u["username"]] and u["is_active"] is False):
found = True
assert found
found = False
for u in users["items"]:
if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+ u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
found = True
assert found
found = False
for u in users["items"]:
if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+ u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
found = True
assert not found
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import json
import sys
def maketoken(newtok):
return 'v2/' + newtok["uuid"] + '/' + newtok["api_token"]
+def get_user_data(case_nr, is_active=True):
+ return {
+ "email": "case{}@test".format(case_nr),
+ "first_name": "Case{}".format(case_nr),
+ "last_name": "Testuser",
+ "is_active": is_active
+ }
+
# case 1
# user only exists on cluster A
-apiA.users().create(body={"user": {"email": "case1@test", "is_active": True}}).execute()
+apiA.users().create(body={"user": get_user_data(case_nr=1)}).execute()
# case 2
# user exists on cluster A and has remotes on B and C
-case2 = apiA.users().create(body={"user": {"email": "case2@test", "is_active": True}}).execute()
+case2 = apiA.users().create(body={"user": get_user_data(case_nr=2)}).execute()
newtok = apiA.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case2["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtok), insecure=True).users().current().execute()
# case 3
# user only exists on cluster B
-case3 = apiB.users().create(body={"user": {"email": "case3@test", "is_active": True}}).execute()
+case3 = apiB.users().create(body={"user": get_user_data(case_nr=3)}).execute()
# case 4
# user only exists on cluster B and has remotes on A and C
-case4 = apiB.users().create(body={"user": {"email": "case4@test", "is_active": True}}).execute()
+case4 = apiB.users().create(body={"user": get_user_data(case_nr=4)}).execute()
newtok = apiB.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case4["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtok), insecure=True).users().current().execute()
# case 5
# user exists on both cluster A and B
-case5 = apiA.users().create(body={"user": {"email": "case5@test", "is_active": True}}).execute()
-case5 = apiB.users().create(body={"user": {"email": "case5@test", "is_active": True}}).execute()
+case5 = apiA.users().create(body={"user": get_user_data(case_nr=5)}).execute()
+case5 = apiB.users().create(body={"user": get_user_data(case_nr=5)}).execute()
# case 6
# user exists on both cluster A and B, with remotes on A, B and C
-case6_A = apiA.users().create(body={"user": {"email": "case6@test", "is_active": True}}).execute()
+case6_A = apiA.users().create(body={"user": get_user_data(case_nr=6)}).execute()
newtokA = apiA.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case6_A["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
-case6_B = apiB.users().create(body={"user": {"email": "case6@test", "is_active": True}}).execute()
+case6_B = apiB.users().create(body={"user": get_user_data(case_nr=6)}).execute()
newtokB = apiB.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case6_B["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
# case 7
# user exists on both cluster B and A, with remotes on A, B and C
-case7_B = apiB.users().create(body={"user": {"email": "case7@test", "is_active": True}}).execute()
+case7_B = apiB.users().create(body={"user": get_user_data(case_nr=7)}).execute()
newtokB = apiB.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case7_B["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
-case7_A = apiA.users().create(body={"user": {"email": "case7@test", "is_active": True}}).execute()
+case7_A = apiA.users().create(body={"user": get_user_data(case_nr=7)}).execute()
newtokA = apiA.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case7_A["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
# case 8
# user exists on both cluster B and C, with remotes on A, B and C
-case8_B = apiB.users().create(body={"user": {"email": "case8@test", "is_active": True}}).execute()
+case8_B = apiB.users().create(body={"user": get_user_data(case_nr=8)}).execute()
newtokB = apiB.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case8_B["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
-case8_C = apiC.users().create(body={"user": {"email": "case8@test", "is_active": True}}).execute()
+case8_C = apiC.users().create(body={"user": get_user_data(case_nr=8)}).execute()
newtokC = apiC.api_client_authorizations().create(body={
"api_client_authorization": {'owner_uuid': case8_C["uuid"]}}).execute()
arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokC), insecure=True).users().current().execute()
# case 9
# user only exists on cluster B, but is inactive
-case9 = apiB.users().create(body={"user": {"email": "case9@test", "is_active": False}}).execute()
+case9 = apiB.users().create(body={"user": get_user_data(case_nr=9, is_active=False)}).execute()
--- /dev/null
+#!/bin/bash
+
+if test -z "$WORKSPACE" ; then
+ echo "WORKSPACE unset"
+ exit 1
+fi
+
+docker stop fedbox1 fedbox2 fedbox3
+docker rm fedbox1 fedbox2 fedbox3
+docker rm fedbox1-data fedbox2-data fedbox3-data
+
+set -ex
+
+mkdir -p $WORKSPACE/tmp
+cd $WORKSPACE/tmp
+virtualenv --python python3 venv3
+. venv3/bin/activate
+
+cd $WORKSPACE/sdk/python
+pip install -e .
+
+cd $WORKSPACE/sdk/cwl
+pip install -e .
+
+export PATH=$PATH:$WORKSPACE/tools/arvbox/bin
+
+mkdir -p $WORKSPACE/tmp/arvbox
+cd $WORKSPACE/sdk/python/tests/fed-migrate
+cwltool arvbox-make-federation.cwl \
+ --arvbox_base $WORKSPACE/tmp/arvbox \
+ --branch $(git rev-parse HEAD) \
+ --arvbox_mode localdemo > fed.json
+
+cwltool fed-migrate.cwl fed.json
proxy_request_buffering off;
}
}
+ upstream health {
+ server {{LISTENHOST}}:{{HEALTHPORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
+ server_name health;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://health;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
+ }
+ }
server {
listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
server_name keep-web-dl ~.*;
nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
+ nginxconf['HEALTHPORT'] = internal_port_from_config("Health")
+ nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")
nginxconf['WSPORT'] = internal_port_from_config("Websocket")
nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
workbench1_external_port = find_available_port()
git_httpd_port = find_available_port()
git_httpd_external_port = find_available_port()
+ health_httpd_port = find_available_port()
+ health_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
keepproxy_external_port = find_available_port()
keepstore_ports = sorted([str(find_available_port()) for _ in xrange(0,4)])
"http://%s:%s"%(localhost, git_httpd_port): {}
},
},
+ "Health": {
+ "ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, health_httpd_port): {}
+ },
+ },
"Keepstore": {
"InternalURLs": {
"http://%s:%s"%(localhost, port): {} for port in keepstore_ports
"RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
},
"Login": {
- "ProviderAppID": "arvados-server",
- "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+ "SSO": {
+ "ProviderAppID": "arvados-server",
+ "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+ },
},
"SystemLogs": {
"LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
pg (1.1.4)
power_assert (1.1.4)
public_suffix (4.0.3)
- rack (2.0.7)
+ rack (2.2.3)
rack-test (0.6.3)
rack (>= 1.0)
rails (5.0.7.2)
json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
PLATFORMS
ruby
uglifier (~> 2.0)
BUNDLED WITH
- 1.11
+ 1.16.6
@object = current_user
end
if not @object.is_active
- if not (current_user.is_admin or @object.is_invited)
+ if @object.uuid[0..4] == Rails.configuration.Login.LoginCluster &&
+ @object.uuid[0..4] != Rails.configuration.ClusterID
+ logger.warn "Local user #{@object.uuid} called users#activate but only LoginCluster can do that"
+ raise ArgumentError.new "cannot activate user #{@object.uuid} here, only the #{@object.uuid[0..4]} cluster can do that"
+ elsif not (current_user.is_admin or @object.is_invited)
logger.warn "User #{@object.uuid} called users.activate " +
"but is not invited"
raise ArgumentError.new "Cannot activate without being invited."
raise
end
- require 'refresh_permission_view'
+ require 'update_permissions'
- refresh_permission_view
+ refresh_permissions
+ refresh_trashed
# Done.
send_json success: true
flash[:notice] = 'You have logged off'
return_to = params[:return_to] || root_url
- redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}/users/sign_out?redirect_uri=#{CGI.escape return_to}"
+ redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}users/sign_out?redirect_uri=#{CGI.escape return_to}"
end
# login - Just bounce to /auth/joshid. The only purpose of this function is
(secret == auth.api_token ||
secret == OpenSSL::HMAC.hexdigest('sha1', auth.api_token, remote))
# found it
+ if token_uuid[0..4] != Rails.configuration.ClusterID
+ Rails.logger.debug "found cached remote token #{token_uuid} with secret #{secret} in local db"
+ end
return auth
end
api_token: secret,
api_client_id: 0,
expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
+ Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
end
return auth
else
sql_conds = nil
user_uuids = users_list.map { |u| u.uuid }
+ # For details on how the trashed_groups table is constructed, see
+ # see db/migrate/20200501150153_permission_table.rb
+
exclude_trashed_records = ""
if !include_trash and (sql_table == "groups" or sql_table == "collections") then
- # Only include records that are not explicitly trashed
- exclude_trashed_records = "AND #{sql_table}.is_trashed = false"
+ # Only include records that are not trashed
+ exclude_trashed_records = "AND (#{sql_table}.trash_at is NULL or #{sql_table}.trash_at > statement_timestamp())"
end
if users_list.select { |u| u.is_admin }.any?
if !include_trash
if sql_table != "api_client_authorizations"
# Only include records where the owner is not trashed
- sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
- "WHERE trashed = 1) #{exclude_trashed_records}"
+ sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} "+
+ "where trash_at <= statement_timestamp()) #{exclude_trashed_records}"
end
end
else
trashed_check = ""
if !include_trash then
- trashed_check = "AND trashed = 0"
+ trashed_check = "AND target_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} where trash_at <= statement_timestamp())"
end
+ # The core of the permission check is a join against the
+ # materialized_permissions table to determine if the user has at
+ # least read permission to either the object itself or its
+ # direct owner (if traverse_owned is true). See
+ # db/migrate/20200501150153_permission_table.rb for details on
+ # how the permissions are computed.
+
+ # A user can have can_manage access to another user, this grants
+ # full access to all that user's stuff. To implement that we
+ # need to include those other users in the permission query.
+ user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: ":user_uuids", perm_level: 1}
+
# Note: it is possible to combine the direct_check and
# owner_check into a single EXISTS() clause, however it turns
# out query optimizer doesn't like it and forces a sequential
# Match a direct read permission link from the user to the record uuid
direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check})"
+ "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check})"
- # Match a read permission link from the user to the record's owner_uuid
+ # Match a read permission for the user to the record's
+ # owner_uuid. This is so we can have a permissions table that
+ # mostly consists of users and groups (projects are a type of
+ # group) and not have to compute and list user permission to
+ # every single object in the system.
+ #
+ # Don't do this for API keys (special behavior) or groups
+ # (already covered by direct_check).
+ #
+ # The traverse_owned flag indicates whether the permission to
+ # read an object also implies transitive permission to read
+ # things the object owns. The situation where this is important
+ # are determining if we can read an object owned by another
+ # user. This makes it possible to have permission to read the
+ # user record without granting permission to read things the
+ # other user owns.
owner_check = ""
if sql_table != "api_client_authorizations" and sql_table != "groups" then
owner_check = "OR #{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND target_owner_uuid IS NOT NULL) "
+ "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check} AND traverse_owned) "
end
links_cond = ""
# users some permission _or_ gives anyone else permission to
# view one of the authorized users.
links_cond = "OR (#{sql_table}.link_class IN (:permission_link_classes) AND "+
- "(#{sql_table}.head_uuid IN (:user_uuids) OR #{sql_table}.tail_uuid IN (:user_uuids)))"
+ "(#{sql_table}.head_uuid IN (#{user_uuids_subquery}) OR #{sql_table}.tail_uuid IN (#{user_uuids_subquery})))"
end
sql_conds = "(#{direct_check} #{owner_check} #{links_cond}) #{exclude_trashed_records}"
logger.warn "User #{current_user.uuid} tried to set ownership of #{self.class.to_s} #{self.uuid} but does not have permission to write #{which} owner_uuid #{check_uuid}"
errors.add :owner_uuid, "cannot be set or changed without write permission on #{which} owner"
raise PermissionDeniedError
+ elsif rsc_class == Group && Group.find_by_uuid(owner_uuid).group_class != "project"
+ errors.add :owner_uuid, "must be a project"
+ raise PermissionDeniedError
end
end
else
# itself.
if !current_user.can?(write: self.uuid)
logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} without write permission"
- errors.add :uuid, "is not writable"
+ errors.add :uuid, " #{uuid} is not writable by #{current_user.uuid}"
raise PermissionDeniedError
end
end
return errors.add :auth_uuid, 'is readonly'
end
if not [Locked, Running].include? self.state
- # don't need one
- self.auth.andand.update_attributes(expires_at: db_current_time)
+ # Don't need one. If auth already exists, expire it.
+ #
+ # We use db_transaction_time here (not db_current_time) to
+ # ensure the token doesn't validate later in the same
+ # transaction (e.g., in a test case) by satisfying expires_at >
+ # transaction timestamp.
+ self.auth.andand.update_attributes(expires_at: db_transaction_time)
self.auth = nil
return
elsif self.auth
#
# SPDX-License-Identifier: AGPL-3.0
+require 'update_permissions'
+
class DatabaseSeeds
extend CurrentApiClient
def self.install
anonymous_group_read_permission
anonymous_user
empty_collection
+ refresh_permissions
+ refresh_trashed
end
end
attribute :properties, :jsonbHash, default: {}
validate :ensure_filesystem_compatible_name
- after_create :invalidate_permissions_cache
- after_update :maybe_invalidate_permissions_cache
+ validate :check_group_class
before_create :assign_name
+ after_create :after_ownership_change
+ after_create :update_trash
+
+ before_update :before_ownership_change
+ after_update :after_ownership_change
+
+ after_create :add_role_manage_link
+
+ after_update :update_trash
+ before_destroy :clear_permissions_and_trash
api_accessible :user, extend: :common do |t|
t.add :name
super if group_class == 'project'
end
- def maybe_invalidate_permissions_cache
- if uuid_changed? or owner_uuid_changed? or is_trashed_changed?
- # This can change users' permissions on other groups as well as
- # this one.
- invalidate_permissions_cache
+ def check_group_class
+ if group_class != 'project' && group_class != 'role'
+ errors.add :group_class, "value must be one of 'project' or 'role', was '#{group_class}'"
+ end
+ if group_class_changed? && !group_class_was.nil?
+ errors.add :group_class, "cannot be modified after record is created"
+ end
+ end
+
+ def update_trash
+ if trash_at_changed? or owner_uuid_changed?
+ # The group was added or removed from the trash.
+ #
+ # Strategy:
+ # Compute project subtree, propagating trash_at to subprojects
+ # Remove groups that don't belong from trash
+ # Add/update groups that do belong in the trash
+
+ temptable = "group_subtree_#{rand(2**64).to_s(10)}"
+ ActiveRecord::Base.connection.exec_query %{
+create temporary table #{temptable} on commit drop
+as select * from project_subtree_with_trash_at($1, LEAST($2, $3)::timestamp)
+},
+ 'Group.update_trash.select',
+ [[nil, self.uuid],
+ [nil, TrashedGroup.find_by_group_uuid(self.owner_uuid).andand.trash_at],
+ [nil, self.trash_at]]
+
+ ActiveRecord::Base.connection.exec_delete %{
+delete from trashed_groups where group_uuid in (select target_uuid from #{temptable} where trash_at is NULL);
+},
+ "Group.update_trash.delete"
+
+ ActiveRecord::Base.connection.exec_query %{
+insert into trashed_groups (group_uuid, trash_at)
+ select target_uuid as group_uuid, trash_at from #{temptable} where trash_at is not NULL
+on conflict (group_uuid) do update set trash_at=EXCLUDED.trash_at;
+},
+ "Group.update_trash.insert"
end
end
- def invalidate_permissions_cache
- # Ensure a new group can be accessed by the appropriate users
- # immediately after being created.
- User.invalidate_permissions_cache self.async_permissions_update
+ def before_ownership_change
+ if owner_uuid_changed? and !self.owner_uuid_was.nil?
+ MaterializedPermission.where(user_uuid: owner_uuid_was, target_uuid: uuid).delete_all
+ update_permissions self.owner_uuid_was, self.uuid, REVOKE_PERM
+ end
+ end
+
+ def after_ownership_change
+ if owner_uuid_changed?
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+ end
+ end
+
+ def clear_permissions_and_trash
+ MaterializedPermission.where(target_uuid: uuid).delete_all
+ ActiveRecord::Base.connection.exec_delete %{
+delete from trashed_groups where group_uuid=$1
+}, "Group.clear_permissions_and_trash", [[nil, self.uuid]]
+
end
def assign_name
end
true
end
+
+ def ensure_owner_uuid_is_permitted
+ if group_class == "role"
+ @requested_manager_uuid = nil
+ if new_record?
+ @requested_manager_uuid = owner_uuid
+ self.owner_uuid = system_user_uuid
+ return true
+ end
+ if self.owner_uuid != system_user_uuid
+ raise "Owner uuid for role must be system user"
+ end
+ raise PermissionDeniedError unless current_user.can?(manage: uuid)
+ true
+ else
+ super
+ end
+ end
+
+ def add_role_manage_link
+ if group_class == "role" && @requested_manager_uuid
+ act_as_system_user do
+ Link.create!(tail_uuid: @requested_manager_uuid,
+ head_uuid: self.uuid,
+ link_class: "permission",
+ name: "can_manage")
+ end
+ end
+ end
end
# already know how to properly treat them.
attribute :properties, :jsonbHash, default: {}
- before_create :permission_to_attach_to_objects
- before_update :permission_to_attach_to_objects
- after_update :maybe_invalidate_permissions_cache
- after_create :maybe_invalidate_permissions_cache
- after_destroy :maybe_invalidate_permissions_cache
validate :name_links_are_obsolete
+ validate :permission_to_attach_to_objects
+ before_update :restrict_alter_permissions
+ after_update :call_update_permissions
+ after_create :call_update_permissions
+ before_destroy :clear_permissions
+ after_destroy :check_permissions
api_accessible :user, extend: :common do |t|
t.add :tail_uuid
# All users can write links that don't affect permissions
return true if self.link_class != 'permission'
+ if PERM_LEVEL[self.name].nil?
+ errors.add(:name, "is invalid permission, must be one of 'can_read', 'can_write', 'can_manage', 'can_login'")
+ return false
+ end
+
+ rsc_class = ArvadosModel::resource_class_for_uuid tail_uuid
+ if rsc_class == Group
+ tail_obj = Group.find_by_uuid(tail_uuid)
+ if tail_obj.nil?
+ errors.add(:tail_uuid, "does not exist")
+ return false
+ end
+ if tail_obj.group_class != "role"
+ errors.add(:tail_uuid, "must be a user or role, was group with group_class #{tail_obj.group_class}")
+ return false
+ end
+ elsif rsc_class != User
+ errors.add(:tail_uuid, "must be a user or role")
+ return false
+ end
+
# Administrators can grant permissions
return true if current_user.is_admin
head_obj = ArvadosModel.find_by_uuid(head_uuid)
# No permission links can be pointed to past collection versions
- return false if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+ if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+ errors.add(:head_uuid, "cannot point to a past version of a collection")
+ return false
+ end
# All users can grant permissions on objects they own or can manage
return true if current_user.can?(manage: head_obj)
false
end
- def maybe_invalidate_permissions_cache
+ def restrict_alter_permissions
+ return true if self.link_class != 'permission' && self.link_class_was != 'permission'
+
+ return true if current_user.andand.uuid == system_user.uuid
+
+ if link_class_changed? || tail_uuid_changed? || head_uuid_changed?
+ raise "Can only alter permission link level"
+ end
+ end
+
+ PERM_LEVEL = {
+ 'can_read' => 1,
+ 'can_login' => 1,
+ 'can_write' => 2,
+ 'can_manage' => 3,
+ }
+
+ def call_update_permissions
+ if self.link_class == 'permission'
+ update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
+ end
+ end
+
+ def clear_permissions
+ if self.link_class == 'permission'
+ update_permissions tail_uuid, head_uuid, REVOKE_PERM, self.uuid
+ end
+ end
+
+ def check_permissions
if self.link_class == 'permission'
- # Clearing the entire permissions cache can generate many
- # unnecessary queries if many active users are not affected by
- # this change. In such cases it would be better to search cached
- # permissions for head_uuid and tail_uuid, and invalidate the
- # cache for only those users. (This would require a browseable
- # cache.)
- User.invalidate_permissions_cache
+ check_permissions_against_full_refresh
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class MaterializedPermission < ApplicationRecord
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class TrashedGroup < ApplicationRecord
+end
# SPDX-License-Identifier: AGPL-3.0
require 'can_be_an_owner'
-require 'refresh_permission_view'
class User < ArvadosModel
include HasUuid
user.username.nil? and user.username_changed?
}
before_update :setup_on_activate
+
before_create :check_auto_admin
before_create :set_initial_username, :if => Proc.new { |user|
user.username.nil? and user.email
}
+ after_create :after_ownership_change
after_create :setup_on_activate
after_create :add_system_group_permission_link
- after_create :invalidate_permissions_cache
after_create :auto_setup_new_user, :if => Proc.new { |user|
Rails.configuration.Users.AutoSetupNewUsers and
(user.uuid != system_user_uuid) and
(user.uuid != anonymous_user_uuid)
}
after_create :send_admin_notifications
+
+ before_update :before_ownership_change
+ after_update :after_ownership_change
after_update :send_profile_created_notification
after_update :sync_repository_names, :if => Proc.new { |user|
(user.uuid != system_user_uuid) and
user.username_changed? and
(not user.username_was.nil?)
}
+ before_destroy :clear_permissions
+ after_destroy :remove_self_from_permissions
has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
has_many :repositories, foreign_key: :owner_uuid, primary_key: :uuid
{read: true, write: true},
{read: true, write: true, manage: true}]
+ VAL_FOR_PERM =
+ {:read => 1,
+ :write => 2,
+ :manage => 3}
+
+
def full_name
"#{first_name} #{last_name}".strip
end
end
def groups_i_can(verb)
- my_groups = self.group_permissions.select { |uuid, mask| mask[verb] }.keys
+ my_groups = self.group_permissions(VAL_FOR_PERM[verb]).keys
if verb == :read
my_groups << anonymous_group_uuid
end
end
end
next if target_uuid == self.uuid
- next if (group_permissions[target_uuid] and
- group_permissions[target_uuid][action])
- if target.respond_to? :owner_uuid
- next if target.owner_uuid == self.uuid
- next if (group_permissions[target.owner_uuid] and
- group_permissions[target.owner_uuid][action])
- end
- sufficient_perms = case action
- when :manage
- ['can_manage']
- when :write
- ['can_manage', 'can_write']
- when :read
- ['can_manage', 'can_write', 'can_read']
- else
- # (Skip this kind of permission opportunity
- # if action is an unknown permission type)
- end
- if sufficient_perms
- # Check permission links with head_uuid pointing directly at
- # the target object. If target is a Group, this is redundant
- # and will fail except [a] if permission caching is broken or
- # [b] during a race condition, where a permission link has
- # *just* been added.
- if Link.where(link_class: 'permission',
- name: sufficient_perms,
- tail_uuid: groups_i_can(action) + [self.uuid],
- head_uuid: target_uuid).any?
- next
- end
+
+ target_owner_uuid = target.owner_uuid if target.respond_to? :owner_uuid
+
+ user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$3"}
+
+ unless ActiveRecord::Base.connection.
+ exec_query(%{
+SELECT 1 FROM #{PERMISSION_VIEW}
+ WHERE user_uuid in (#{user_uuids_subquery}) and
+ ((target_uuid = $2 and perm_level >= $3)
+ or (target_uuid = $4 and perm_level >= $3 and traverse_owned))
+},
+ # "name" arg is a query label that appears in logs:
+ "user_can_query",
+ [[nil, self.uuid],
+ [nil, target_uuid],
+ [nil, VAL_FOR_PERM[action]],
+ [nil, target_owner_uuid]]
+ ).any?
+ return false
end
- return false
end
true
end
- def self.invalidate_permissions_cache(async=false)
- refresh_permission_view(async)
+ def before_ownership_change
+ if owner_uuid_changed? and !self.owner_uuid_was.nil?
+ MaterializedPermission.where(user_uuid: owner_uuid_was, target_uuid: uuid).delete_all
+ update_permissions self.owner_uuid_was, self.uuid, REVOKE_PERM
+ end
end
- def invalidate_permissions_cache
- User.invalidate_permissions_cache
+ def after_ownership_change
+ if owner_uuid_changed?
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+ end
+ end
+
+ def clear_permissions
+ MaterializedPermission.where("user_uuid = ? and target_uuid != ?", uuid, uuid).delete_all
+ end
+
+ def remove_self_from_permissions
+ MaterializedPermission.where("target_uuid = ?", uuid).delete_all
+ check_permissions_against_full_refresh
end
# Return a hash of {user_uuid: group_perms}
+ #
+ # note: this does not account for permissions that a user gains by
+ # having can_manage on another user.
def self.all_group_permissions
all_perms = {}
ActiveRecord::Base.connection.
- exec_query("SELECT user_uuid, target_owner_uuid, perm_level, trashed
+ exec_query(%{
+SELECT user_uuid, target_uuid, perm_level
FROM #{PERMISSION_VIEW}
- WHERE target_owner_uuid IS NOT NULL",
+ WHERE traverse_owned
+},
# "name" arg is a query label that appears in logs:
- "all_group_permissions",
- ).rows.each do |user_uuid, group_uuid, max_p_val, trashed|
+ "all_group_permissions").
+ rows.each do |user_uuid, group_uuid, max_p_val|
all_perms[user_uuid] ||= {}
all_perms[user_uuid][group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
end
# Return a hash of {group_uuid: perm_hash} where perm_hash[:read]
# and perm_hash[:write] are true if this user can read and write
# objects owned by group_uuid.
- def group_permissions
- group_perms = {self.uuid => {:read => true, :write => true, :manage => true}}
+ def group_permissions(level=1)
+ group_perms = {}
+
+ user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$2"}
+
ActiveRecord::Base.connection.
- exec_query("SELECT target_owner_uuid, perm_level, trashed
- FROM #{PERMISSION_VIEW}
- WHERE user_uuid = $1
- AND target_owner_uuid IS NOT NULL",
+ exec_query(%{
+SELECT target_uuid, perm_level
+ FROM #{PERMISSION_VIEW}
+ WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= $2
+},
# "name" arg is a query label that appears in logs:
- "group_permissions for #{uuid}",
+ "User.group_permissions",
# "binds" arg is an array of [col_id, value] for '$1' vars:
- [[nil, uuid]],
- ).rows.each do |group_uuid, max_p_val, trashed|
+ [[nil, uuid],
+ [nil, level]]).
+ rows.each do |group_uuid, max_p_val|
group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
end
group_perms
end
def must_unsetup_to_deactivate
- if self.is_active_changed? &&
- self.is_active_was == true &&
+ if !self.new_record? &&
+ self.uuid[0..4] == Rails.configuration.Login.LoginCluster &&
+ self.uuid[0..4] != Rails.configuration.ClusterID
+ # OK to update our local record to whatever the LoginCluster
+ # reports, because self-activate is not allowed.
+ return
+ elsif self.is_active_changed? &&
+ self.is_active_was &&
!self.is_active
group = Group.where(name: 'All users').select do |g|
self.uuid = new_uuid
save!(validate: false)
change_all_uuid_refs(old_uuid: old_uuid, new_uuid: new_uuid)
+ ActiveRecord::Base.connection.exec_update %{
+update #{PERMISSION_VIEW} set user_uuid=$1 where user_uuid = $2
+},
+ 'User.update_uuid.update_permissions_user_uuid',
+ [[nil, new_uuid],
+ [nil, old_uuid]]
+ ActiveRecord::Base.connection.exec_update %{
+update #{PERMISSION_VIEW} set target_uuid=$1 where target_uuid = $2
+},
+ 'User.update_uuid.update_permissions_target_uuid',
+ [[nil, new_uuid],
+ [nil, old_uuid]]
end
end
raise "user does not exist" if !new_user
raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
+ self.clear_permissions
+ new_user.clear_permissions
+
# If 'self' is a remote user, don't transfer authorizations
# (i.e. ability to access the account) to the new user, because
# that gives the remote site the ability to access the 'new'
if redirect_to_new_user
update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
end
- invalidate_permissions_cache
+ skip_check_permissions_against_full_refresh do
+ update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM
+ update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM
+ update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM
+ end
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
end
end
# Real values will be copied from globals by omniauth_init.rb. For
# now, assign some strings so the generic *.yml config loader
# doesn't overwrite them or complain that they're missing.
- Rails.configuration.Login["ProviderAppID"] = 'xxx'
- Rails.configuration.Login["ProviderAppSecret"] = 'xxx'
+ Rails.configuration.Login["SSO"]["ProviderAppID"] = 'xxx'
+ Rails.configuration.Login["SSO"]["ProviderAppSecret"] = 'xxx'
Rails.configuration.Services["SSO"]["ExternalURL"] = '//xxx'
WARNED_OMNIAUTH_CONFIG = true
end
arvcfg.declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
arvcfg.declare_config "Users.NewUserNotificationRecipients", Hash, :new_user_notification_recipients, ->(cfg, k, v) { arrayToHash cfg, "Users.NewUserNotificationRecipients", v }
arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Hash, :new_inactive_user_notification_recipients, method(:arrayToHash)
-arvcfg.declare_config "Login.ProviderAppSecret", String, :sso_app_secret
-arvcfg.declare_config "Login.ProviderAppID", String, :sso_app_id
+arvcfg.declare_config "Login.SSO.ProviderAppSecret", String, :sso_app_secret
+arvcfg.declare_config "Login.SSO.ProviderAppID", String, :sso_app_id
arvcfg.declare_config "Login.LoginCluster", String
arvcfg.declare_config "Login.RemoteTokenRefresh", ActiveSupport::Duration
arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure
dbcfg.declare_config "PostgreSQL.Connection.dbname", String, :database
dbcfg.declare_config "PostgreSQL.Connection.template", String, :template
dbcfg.declare_config "PostgreSQL.Connection.encoding", String, :encoding
+dbcfg.declare_config "PostgreSQL.Connection.collation", String, :collation
application_config = {}
%w(application.default application).each do |cfgfile|
# Use template0 when creating a new database. Avoids
# character-encoding/collation problems.
$arvados_config["PostgreSQL"]["Connection"]["template"] = "template0"
+ # Some test cases depend on en_US.UTF-8 collation.
+ $arvados_config["PostgreSQL"]["Connection"]["collation"] = "en_US.UTF-8"
end
if $arvados_config["PostgreSQL"]["Connection"]["password"].empty?
"#{dbhost}/#{$arvados_config["PostgreSQL"]["Connection"]["dbname"]}?"+
"template=#{$arvados_config["PostgreSQL"]["Connection"]["template"]}&"+
"encoding=#{$arvados_config["PostgreSQL"]["Connection"]["client_encoding"]}&"+
+ "collation=#{$arvados_config["PostgreSQL"]["Connection"]["collation"]}&"+
"pool=#{$arvados_config["PostgreSQL"]["ConnectionPool"]}"
Server::Application.configure do
if defined? CUSTOM_PROVIDER_URL
Rails.logger.warn "Copying omniauth from globals in legacy config file."
- Rails.configuration.Login["ProviderAppID"] = APP_ID
- Rails.configuration.Login["ProviderAppSecret"] = APP_SECRET
- Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL
+ Rails.configuration.Login["SSO"]["ProviderAppID"] = APP_ID
+ Rails.configuration.Login["SSO"]["ProviderAppSecret"] = APP_SECRET
+ Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL.sub(/\/$/, "") + "/"
else
Rails.application.config.middleware.use OmniAuth::Builder do
provider(:josh_id,
- Rails.configuration.Login["ProviderAppID"],
- Rails.configuration.Login["ProviderAppSecret"],
+ Rails.configuration.Login["SSO"]["ProviderAppID"],
+ Rails.configuration.Login["SSO"]["ProviderAppSecret"],
Rails.configuration.Services["SSO"]["ExternalURL"])
end
OmniAuth.config.on_failure = StaticController.action(:login_failure)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ActiveRecord::Base.connection.class.set_callback :checkout, :after do
+ # If the database connection is in a time zone other than UTC,
+ # "timestamp" values don't behave as desired.
+ #
+ # For example, ['select now() > ?', Time.now] returns true in time
+ # zones +0100 and UTC (which makes sense since Time.now is evaluated
+ # before now()), but false in time zone -0100 (now() returns an
+ # earlier clock time, and its time zone is dropped when comparing to
+ # a "timestamp without time zone").
+ raw_connection.sync_exec("SET TIME ZONE 'UTC'")
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+class PermissionTable < ActiveRecord::Migration[5.0]
+ def up
+ # This is a major migration. We are replacing the
+ # materialized_permission_view, which is fully recomputed any time
+ # a permission changes (and becomes very expensive as the number
+ # of users/groups becomes large), with a new strategy that only
+ # recomputes permissions for the subset of objects that are
+ # potentially affected by the addition or removal of a permission
+ # relationship (i.e. ownership or a permission link).
+ #
+ # This also disentangles the concept of "trashed groups" from the
+ # permissions system. Updating trashed items follows a similar
+ # (but less complicated) strategy to updating permissions, so it
+ # may be helpful to look at that first.
+
+ ActiveRecord::Base.connection.execute "DROP MATERIALIZED VIEW IF EXISTS materialized_permission_view;"
+ drop_table :permission_refresh_lock
+
+ # This table stores the set of trashed groups and their trash_at
+ # time. Used to exclude trashed projects and their contents when
+ # getting object listings.
+ create_table :trashed_groups, :id => false do |t|
+ t.string :group_uuid
+ t.datetime :trash_at
+ end
+ add_index :trashed_groups, :group_uuid, :unique => true
+
+ ActiveRecord::Base.connection.execute %{
+create or replace function project_subtree_with_trash_at (starting_uuid varchar(27), starting_trash_at timestamp)
+returns table (target_uuid varchar(27), trash_at timestamp)
+STABLE
+language SQL
+as $$
+/* Starting from a project, recursively traverse all the projects
+ underneath it and return a set of project uuids and trash_at times
+ (may be null). The initial trash_at can be a timestamp or null.
+ The trash_at time propagates downward to groups it owns, i.e. when a
+ group is trashed, everything underneath it in the ownership
+ hierarchy is also considered trashed. However, this is fact is
+ recorded in the trashed_groups table, not by updating trash_at field
+ in the groups table.
+*/
+WITH RECURSIVE
+ project_subtree(uuid, trash_at) as (
+ values (starting_uuid, starting_trash_at)
+ union
+ select groups.uuid, LEAST(project_subtree.trash_at, groups.trash_at)
+ from groups join project_subtree on (groups.owner_uuid = project_subtree.uuid)
+ )
+ select uuid, trash_at from project_subtree;
+$$;
+}
+
+ # Now populate the table. For a non-test databse this is the only
+ # time this ever happens, after this the trash table is updated
+ # incrementally. See app/models/group.rb#update_trash
+ refresh_trashed
+
+ # The table to store the flattened permissions. This is almost
+ # exactly the same as the old materalized_permission_view except
+ # that the target_owner_uuid colunm in the view is now just a
+ # boolean traverse_owned (the column was only ever tested for null
+ # or non-null).
+ #
+ # For details on how this table is used to apply permissions to
+ # queries, see app/models/arvados_model.rb#readable_by
+ #
+ create_table :materialized_permissions, :id => false do |t|
+ t.string :user_uuid
+ t.string :target_uuid
+ t.integer :perm_level
+ t.boolean :traverse_owned
+ end
+ add_index :materialized_permissions, [:user_uuid, :target_uuid], unique: true, name: 'permission_user_target'
+ add_index :materialized_permissions, [:target_uuid], unique: false, name: 'permission_target'
+
+ ActiveRecord::Base.connection.execute %{
+create or replace function should_traverse_owned (starting_uuid varchar(27),
+ starting_perm integer)
+ returns bool
+IMMUTABLE
+language SQL
+as $$
+/* Helper function. Determines if permission on an object implies
+ transitive permission to things the object owns. This is always
+ true for groups, but only true for users when the permission level
+ is can_manage.
+*/
+select starting_uuid like '_____-j7d0g-_______________' or
+ (starting_uuid like '_____-tpzed-_______________' and starting_perm >= 3);
+$$;
+}
+
+ # Merge all permission relationships into a single view. This
+ # consists of: groups owned by users and projects, users owned
+ # by other users, users have permission on themselves,
+ # and explicit permission links.
+ #
+ # A SQL view gets inlined into the query where it is used as a
+ # subquery. This enables the query planner to inject constraints,
+ # so it only has to look up edges it plans to traverse and avoid a brute
+ # force query of all edges.
+ ActiveRecord::Base.connection.execute %{
+create view permission_graph_edges as
+ select groups.owner_uuid as tail_uuid, groups.uuid as head_uuid,
+ (3) as val, groups.uuid as edge_id from groups
+union all
+ select users.owner_uuid as tail_uuid, users.uuid as head_uuid,
+ (3) as val, users.uuid as edge_id from users
+union all
+ select users.uuid as tail_uuid, users.uuid as head_uuid,
+ (3) as val, '' as edge_id from users
+union all
+ select links.tail_uuid,
+ links.head_uuid,
+ CASE
+ WHEN links.name = 'can_read' THEN 1
+ WHEN links.name = 'can_login' THEN 1
+ WHEN links.name = 'can_write' THEN 2
+ WHEN links.name = 'can_manage' THEN 3
+ ELSE 0
+ END as val,
+ links.uuid as edge_id
+ from links
+ where links.link_class='permission'
+}
+
+ # This is used to ensure that the permission edge passed into
+ # compute_permission_subgraph takes replaces the existing edge in
+ # the "edges" view that is about to be removed.
+ edge_perm = %{
+case (edges.edge_id = perm_edge_id)
+ when true then starting_perm
+ else edges.val
+ end
+}
+
+ # The primary function to compute permissions for a subgraph.
+ # Comments on how it works are inline.
+ #
+ # Due to performance issues due to the query optimizer not
+ # working across function and "with" expression boundaries, I
+ # had to fall back on using string templates for repeated code
+ # in order to inline it.
+
+ ActiveRecord::Base.connection.execute %{
+create or replace function compute_permission_subgraph (perm_origin_uuid varchar(27),
+ starting_uuid varchar(27),
+ starting_perm integer,
+ perm_edge_id varchar(27))
+returns table (user_uuid varchar(27), target_uuid varchar(27), val integer, traverse_owned bool)
+STABLE
+language SQL
+as $$
+
+/* The purpose of this function is to compute the permissions for a
+ subgraph of the database, starting from a given edge. The newly
+ computed permissions are used to add and remove rows from the main
+ permissions table.
+
+ perm_origin_uuid: The object that 'gets' the permission.
+
+ starting_uuid: The starting object the permission applies to.
+
+ starting_perm: The permission that perm_origin_uuid 'has' on
+ starting_uuid One of 1, 2, 3 for can_read,
+ can_write, can_manage respectively, or 0 to revoke
+ permissions.
+
+ perm_edge_id: Identifies the permission edge that is being updated.
+ Changes of ownership, this is starting_uuid.
+ For links, this is the uuid of the link object.
+ This is used to override the edge value in the database
+ with starting_perm. This is necessary when revoking
+ permissions because the update happens before edge is
+ actually removed.
+*/
+with
+ /* Starting from starting_uuid, determine the set of objects that
+ could be affected by this permission change.
+
+ Note: We don't traverse users unless it is an "identity"
+ permission (permission origin is self).
+ */
+ perm_from_start(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ values (perm_origin_uuid, starting_uuid, starting_perm,
+ should_traverse_owned(starting_uuid, starting_perm),
+ (perm_origin_uuid = starting_uuid or starting_uuid not like '_____-tpzed-_______________'))
+},
+:edge_perm => edge_perm
+} }),
+
+ /* Find other inbound edges that grant permissions to 'targets' in
+ perm_from_start, and compute permissions that originate from
+ those.
+
+ This is necessary for two reasons:
+
+ 1) Other users may have access to a subset of the objects
+ through other permission links than the one we started from.
+ If we don't recompute them, their permission will get dropped.
+
+ 2) There may be more than one path through which a user gets
+ permission to an object. For example, a user owns a project
+ and also shares it can_read with a group the user belongs
+ to. adding the can_read link must not overwrite the existing
+ can_manage permission granted by ownership.
+ */
+ additional_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select edges.tail_uuid as origin_uuid, edges.head_uuid as target_uuid, edges.val,
+ should_traverse_owned(edges.head_uuid, edges.val),
+ edges.head_uuid like '_____-j7d0g-_______________'
+ from permission_graph_edges as edges
+ where edges.edge_id != perm_edge_id and
+ edges.tail_uuid not in (select target_uuid from perm_from_start where target_uuid like '_____-j7d0g-_______________') and
+ edges.head_uuid in (select target_uuid from perm_from_start)
+},
+:edge_perm => edge_perm
+} }),
+
+ /* Combine the permissions computed in the first two phases. */
+ all_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+ select * from perm_from_start
+ union all
+ select * from additional_perms
+ )
+
+ /* The actual query that produces rows to be added or removed
+ from the materialized_permissions table. This is the clever
+ bit.
+
+ Key insights:
+
+ * For every group, the materialized_permissions lists all users
+ that can access to that group.
+
+ * The all_perms subquery has computed permissions on on a set of
+ objects for all inbound "origins", which are users or groups.
+
+ * Permissions through groups are transitive.
+
+ We can infer:
+
+ 1) The materialized_permissions table declares that user X has permission N on group Y
+ 2) The all_perms result has determined group Y has permission M on object Z
+ 3) Therefore, user X has permission min(N, M) on object Z
+
+ This allows us to efficiently determine the set of users that
+ have permissions on the subset of objects, without having to
+ follow the chain of permission back up to find those users.
+
+ In addition, because users always have permission on themselves, this
+ query also makes sure those permission rows are always
+ returned.
+ */
+ select v.user_uuid, v.target_uuid, max(v.perm_level), bool_or(v.traverse_owned) from
+ (select m.user_uuid,
+ u.target_uuid,
+ least(u.val, m.perm_level) as perm_level,
+ u.traverse_owned
+ from all_perms as u, materialized_permissions as m
+ where u.perm_origin_uuid = m.target_uuid AND m.traverse_owned
+ AND (m.user_uuid = m.target_uuid or m.target_uuid not like '_____-tpzed-_______________')
+ union all
+ select target_uuid as user_uuid, target_uuid, 3, true
+ from all_perms
+ where all_perms.target_uuid like '_____-tpzed-_______________') as v
+ group by v.user_uuid, v.target_uuid
+$$;
+ }
+
+ #
+ # Populate materialized_permissions by traversing permissions
+ # starting at each user.
+ #
+ refresh_permissions
+ end
+
+ def down
+ drop_table :materialized_permissions
+ drop_table :trashed_groups
+
+ ActiveRecord::Base.connection.execute "DROP function project_subtree_with_trash_at (varchar, timestamp);"
+ ActiveRecord::Base.connection.execute "DROP function compute_permission_subgraph (varchar, varchar, integer, varchar);"
+ ActiveRecord::Base.connection.execute "DROP function should_traverse_owned(varchar, integer);"
+ ActiveRecord::Base.connection.execute "DROP view permission_graph_edges;"
+
+ ActiveRecord::Base.connection.execute(%{
+CREATE MATERIALIZED VIEW materialized_permission_view AS
+ WITH RECURSIVE perm_value(name, val) AS (
+ VALUES ('can_read'::text,(1)::smallint), ('can_login'::text,1), ('can_write'::text,2), ('can_manage'::text,3)
+ ), perm_edges(tail_uuid, head_uuid, val, follow, trashed) AS (
+ SELECT links.tail_uuid,
+ links.head_uuid,
+ pv.val,
+ ((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
+ (0)::smallint AS trashed,
+ (0)::smallint AS followtrash
+ FROM ((public.links
+ LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
+ LEFT JOIN public.groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
+ WHERE ((links.link_class)::text = 'permission'::text)
+ UNION ALL
+ SELECT groups.owner_uuid,
+ groups.uuid,
+ 3,
+ true AS bool,
+ CASE
+ WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
+ ELSE 0
+ END AS "case",
+ 1
+ FROM public.groups
+ ), perm(val, follow, user_uuid, target_uuid, trashed) AS (
+ SELECT (3)::smallint AS val,
+ true AS follow,
+ (users.uuid)::character varying(32) AS user_uuid,
+ (users.uuid)::character varying(32) AS target_uuid,
+ (0)::smallint AS trashed
+ FROM public.users
+ UNION
+ SELECT (LEAST((perm_1.val)::integer, edges.val))::smallint AS val,
+ edges.follow,
+ perm_1.user_uuid,
+ (edges.head_uuid)::character varying(32) AS target_uuid,
+ ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
+ FROM (perm perm_1
+ JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
+ )
+ SELECT perm.user_uuid,
+ perm.target_uuid,
+ max(perm.val) AS perm_level,
+ CASE perm.follow
+ WHEN true THEN perm.target_uuid
+ ELSE NULL::character varying
+ END AS target_owner_uuid,
+ max(perm.trashed) AS trashed
+ FROM perm
+ GROUP BY perm.user_uuid, perm.target_uuid,
+ CASE perm.follow
+ WHEN true THEN perm.target_uuid
+ ELSE NULL::character varying
+ END
+ WITH NO DATA;
+}
+ )
+
+ add_index :materialized_permission_view, [:trashed, :target_uuid], name: 'permission_target_trashed'
+ add_index :materialized_permission_view, [:user_uuid, :trashed, :perm_level], name: 'permission_target_user_trashed_level'
+ create_table :permission_refresh_lock
+
+ ActiveRecord::Base.connection.execute 'REFRESH MATERIALIZED VIEW materialized_permission_view;'
+ end
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'fix_roles_projects'
+
+class FixRolesProjects < ActiveRecord::Migration[5.0]
+ def up
+ # defined in a function for easy testing.
+ fix_roles_projects
+ end
+
+ def down
+ # This migration is not reversible. However, the results are
+ # backwards compatible.
+ end
+end
-- COMMENT ON EXTENSION pg_trgm IS 'text similarity measurement and index searching based on trigrams';
+--
+-- Name: compute_permission_subgraph(character varying, character varying, integer, character varying); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.compute_permission_subgraph(perm_origin_uuid character varying, starting_uuid character varying, starting_perm integer, perm_edge_id character varying) RETURNS TABLE(user_uuid character varying, target_uuid character varying, val integer, traverse_owned boolean)
+ LANGUAGE sql STABLE
+ AS $$
+
+/* The purpose of this function is to compute the permissions for a
+ subgraph of the database, starting from a given edge. The newly
+ computed permissions are used to add and remove rows from the main
+ permissions table.
+
+ perm_origin_uuid: The object that 'gets' the permission.
+
+ starting_uuid: The starting object the permission applies to.
+
+ starting_perm: The permission that perm_origin_uuid 'has' on
+ starting_uuid One of 1, 2, 3 for can_read,
+ can_write, can_manage respectively, or 0 to revoke
+ permissions.
+
+ perm_edge_id: Identifies the permission edge that is being updated.
+ Changes of ownership, this is starting_uuid.
+ For links, this is the uuid of the link object.
+ This is used to override the edge value in the database
+ with starting_perm. This is necessary when revoking
+ permissions because the update happens before edge is
+ actually removed.
+*/
+with
+ /* Starting from starting_uuid, determine the set of objects that
+ could be affected by this permission change.
+
+ Note: We don't traverse users unless it is an "identity"
+ permission (permission origin is self).
+ */
+ perm_from_start(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+
+WITH RECURSIVE
+ traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+
+ values (perm_origin_uuid, starting_uuid, starting_perm,
+ should_traverse_owned(starting_uuid, starting_perm),
+ (perm_origin_uuid = starting_uuid or starting_uuid not like '_____-tpzed-_______________'))
+
+ union
+ (select traverse_graph.origin_uuid,
+ edges.head_uuid,
+ least(
+case (edges.edge_id = perm_edge_id)
+ when true then starting_perm
+ else edges.val
+ end
+,
+ traverse_graph.val),
+ should_traverse_owned(edges.head_uuid, edges.val),
+ false
+ from permission_graph_edges as edges, traverse_graph
+ where traverse_graph.target_uuid = edges.tail_uuid
+ and (edges.tail_uuid like '_____-j7d0g-_______________' or
+ traverse_graph.starting_set)))
+ select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+ group by (traverse_graph.origin_uuid, target_uuid)
+),
+
+ /* Find other inbound edges that grant permissions to 'targets' in
+ perm_from_start, and compute permissions that originate from
+ those.
+
+ This is necessary for two reasons:
+
+ 1) Other users may have access to a subset of the objects
+ through other permission links than the one we started from.
+ If we don't recompute them, their permission will get dropped.
+
+ 2) There may be more than one path through which a user gets
+ permission to an object. For example, a user owns a project
+ and also shares it can_read with a group the user belongs
+ to. adding the can_read link must not overwrite the existing
+ can_manage permission granted by ownership.
+ */
+ additional_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+
+WITH RECURSIVE
+ traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+
+ select edges.tail_uuid as origin_uuid, edges.head_uuid as target_uuid, edges.val,
+ should_traverse_owned(edges.head_uuid, edges.val),
+ edges.head_uuid like '_____-j7d0g-_______________'
+ from permission_graph_edges as edges
+ where edges.edge_id != perm_edge_id and
+ edges.tail_uuid not in (select target_uuid from perm_from_start where target_uuid like '_____-j7d0g-_______________') and
+ edges.head_uuid in (select target_uuid from perm_from_start)
+
+ union
+ (select traverse_graph.origin_uuid,
+ edges.head_uuid,
+ least(
+case (edges.edge_id = perm_edge_id)
+ when true then starting_perm
+ else edges.val
+ end
+,
+ traverse_graph.val),
+ should_traverse_owned(edges.head_uuid, edges.val),
+ false
+ from permission_graph_edges as edges, traverse_graph
+ where traverse_graph.target_uuid = edges.tail_uuid
+ and (edges.tail_uuid like '_____-j7d0g-_______________' or
+ traverse_graph.starting_set)))
+ select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+ group by (traverse_graph.origin_uuid, target_uuid)
+),
+
+ /* Combine the permissions computed in the first two phases. */
+ all_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+ select * from perm_from_start
+ union all
+ select * from additional_perms
+ )
+
+ /* The actual query that produces rows to be added or removed
+ from the materialized_permissions table. This is the clever
+ bit.
+
+ Key insights:
+
+ * For every group, the materialized_permissions lists all users
+ that can access to that group.
+
+ * The all_perms subquery has computed permissions on on a set of
+ objects for all inbound "origins", which are users or groups.
+
+ * Permissions through groups are transitive.
+
+ We can infer:
+
+ 1) The materialized_permissions table declares that user X has permission N on group Y
+ 2) The all_perms result has determined group Y has permission M on object Z
+ 3) Therefore, user X has permission min(N, M) on object Z
+
+ This allows us to efficiently determine the set of users that
+ have permissions on the subset of objects, without having to
+ follow the chain of permission back up to find those users.
+
+ In addition, because users always have permission on themselves, this
+ query also makes sure those permission rows are always
+ returned.
+ */
+ select v.user_uuid, v.target_uuid, max(v.perm_level), bool_or(v.traverse_owned) from
+ (select m.user_uuid,
+ u.target_uuid,
+ least(u.val, m.perm_level) as perm_level,
+ u.traverse_owned
+ from all_perms as u, materialized_permissions as m
+ where u.perm_origin_uuid = m.target_uuid AND m.traverse_owned
+ AND (m.user_uuid = m.target_uuid or m.target_uuid not like '_____-tpzed-_______________')
+ union all
+ select target_uuid as user_uuid, target_uuid, 3, true
+ from all_perms
+ where all_perms.target_uuid like '_____-tpzed-_______________') as v
+ group by v.user_uuid, v.target_uuid
+$$;
+
+
+--
+-- Name: project_subtree_with_trash_at(character varying, timestamp without time zone); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.project_subtree_with_trash_at(starting_uuid character varying, starting_trash_at timestamp without time zone) RETURNS TABLE(target_uuid character varying, trash_at timestamp without time zone)
+ LANGUAGE sql STABLE
+ AS $$
+/* Starting from a project, recursively traverse all the projects
+ underneath it and return a set of project uuids and trash_at times
+ (may be null). The initial trash_at can be a timestamp or null.
+ The trash_at time propagates downward to groups it owns, i.e. when a
+ group is trashed, everything underneath it in the ownership
+ hierarchy is also considered trashed. However, this is fact is
+ recorded in the trashed_groups table, not by updating trash_at field
+ in the groups table.
+*/
+WITH RECURSIVE
+ project_subtree(uuid, trash_at) as (
+ values (starting_uuid, starting_trash_at)
+ union
+ select groups.uuid, LEAST(project_subtree.trash_at, groups.trash_at)
+ from groups join project_subtree on (groups.owner_uuid = project_subtree.uuid)
+ )
+ select uuid, trash_at from project_subtree;
+$$;
+
+
+--
+-- Name: should_traverse_owned(character varying, integer); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.should_traverse_owned(starting_uuid character varying, starting_perm integer) RETURNS boolean
+ LANGUAGE sql IMMUTABLE
+ AS $$
+/* Helper function. Determines if permission on an object implies
+ transitive permission to things the object owns. This is always
+ true for groups, but only true for users when the permission level
+ is can_manage.
+*/
+select starting_uuid like '_____-j7d0g-_______________' or
+ (starting_uuid like '_____-tpzed-_______________' and starting_perm >= 3);
+$$;
+
+
SET default_tablespace = '';
SET default_with_oids = false;
--
--- Name: users; Type: TABLE; Schema: public; Owner: -
+-- Name: materialized_permissions; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE public.users (
- id integer NOT NULL,
- uuid character varying(255),
- owner_uuid character varying(255) NOT NULL,
- created_at timestamp without time zone NOT NULL,
- modified_by_client_uuid character varying(255),
- modified_by_user_uuid character varying(255),
- modified_at timestamp without time zone,
- email character varying(255),
- first_name character varying(255),
- last_name character varying(255),
- identity_url character varying(255),
- is_admin boolean,
- prefs text,
- updated_at timestamp without time zone NOT NULL,
- default_owner_uuid character varying(255),
- is_active boolean DEFAULT false,
- username character varying(255),
- redirect_to_user_uuid character varying
+CREATE TABLE public.materialized_permissions (
+ user_uuid character varying,
+ target_uuid character varying,
+ perm_level integer,
+ traverse_owned boolean
);
---
--- Name: materialized_permission_view; Type: MATERIALIZED VIEW; Schema: public; Owner: -
---
-
-CREATE MATERIALIZED VIEW public.materialized_permission_view AS
- WITH RECURSIVE perm_value(name, val) AS (
- VALUES ('can_read'::text,(1)::smallint), ('can_login'::text,1), ('can_write'::text,2), ('can_manage'::text,3)
- ), perm_edges(tail_uuid, head_uuid, val, follow, trashed) AS (
- SELECT links.tail_uuid,
- links.head_uuid,
- pv.val,
- ((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
- (0)::smallint AS trashed,
- (0)::smallint AS followtrash
- FROM ((public.links
- LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
- LEFT JOIN public.groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
- WHERE ((links.link_class)::text = 'permission'::text)
- UNION ALL
- SELECT groups.owner_uuid,
- groups.uuid,
- 3,
- true AS bool,
- CASE
- WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
- ELSE 0
- END AS "case",
- 1
- FROM public.groups
- ), perm(val, follow, user_uuid, target_uuid, trashed) AS (
- SELECT (3)::smallint AS val,
- true AS follow,
- (users.uuid)::character varying(32) AS user_uuid,
- (users.uuid)::character varying(32) AS target_uuid,
- (0)::smallint AS trashed
- FROM public.users
- UNION
- SELECT (LEAST((perm_1.val)::integer, edges.val))::smallint AS val,
- edges.follow,
- perm_1.user_uuid,
- (edges.head_uuid)::character varying(32) AS target_uuid,
- ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
- FROM (perm perm_1
- JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
- )
- SELECT perm.user_uuid,
- perm.target_uuid,
- max(perm.val) AS perm_level,
- CASE perm.follow
- WHEN true THEN perm.target_uuid
- ELSE NULL::character varying
- END AS target_owner_uuid,
- max(perm.trashed) AS trashed
- FROM perm
- GROUP BY perm.user_uuid, perm.target_uuid,
- CASE perm.follow
- WHEN true THEN perm.target_uuid
- ELSE NULL::character varying
- END
- WITH NO DATA;
-
-
--
-- Name: nodes; Type: TABLE; Schema: public; Owner: -
--
--
--- Name: permission_refresh_lock; Type: TABLE; Schema: public; Owner: -
+-- Name: users; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE public.permission_refresh_lock (
- id integer NOT NULL
+CREATE TABLE public.users (
+ id integer NOT NULL,
+ uuid character varying(255),
+ owner_uuid character varying(255) NOT NULL,
+ created_at timestamp without time zone NOT NULL,
+ modified_by_client_uuid character varying(255),
+ modified_by_user_uuid character varying(255),
+ modified_at timestamp without time zone,
+ email character varying(255),
+ first_name character varying(255),
+ last_name character varying(255),
+ identity_url character varying(255),
+ is_admin boolean,
+ prefs text,
+ updated_at timestamp without time zone NOT NULL,
+ default_owner_uuid character varying(255),
+ is_active boolean DEFAULT false,
+ username character varying(255),
+ redirect_to_user_uuid character varying
);
--
--- Name: permission_refresh_lock_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.permission_refresh_lock_id_seq
- START WITH 1
- INCREMENT BY 1
- NO MINVALUE
- NO MAXVALUE
- CACHE 1;
-
-
---
--- Name: permission_refresh_lock_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.permission_refresh_lock_id_seq OWNED BY public.permission_refresh_lock.id;
+-- Name: permission_graph_edges; Type: VIEW; Schema: public; Owner: -
+--
+
+CREATE VIEW public.permission_graph_edges AS
+ SELECT groups.owner_uuid AS tail_uuid,
+ groups.uuid AS head_uuid,
+ 3 AS val,
+ groups.uuid AS edge_id
+ FROM public.groups
+UNION ALL
+ SELECT users.owner_uuid AS tail_uuid,
+ users.uuid AS head_uuid,
+ 3 AS val,
+ users.uuid AS edge_id
+ FROM public.users
+UNION ALL
+ SELECT users.uuid AS tail_uuid,
+ users.uuid AS head_uuid,
+ 3 AS val,
+ ''::character varying AS edge_id
+ FROM public.users
+UNION ALL
+ SELECT links.tail_uuid,
+ links.head_uuid,
+ CASE
+ WHEN ((links.name)::text = 'can_read'::text) THEN 1
+ WHEN ((links.name)::text = 'can_login'::text) THEN 1
+ WHEN ((links.name)::text = 'can_write'::text) THEN 2
+ WHEN ((links.name)::text = 'can_manage'::text) THEN 3
+ ELSE 0
+ END AS val,
+ links.uuid AS edge_id
+ FROM public.links
+ WHERE ((links.link_class)::text = 'permission'::text);
--
ALTER SEQUENCE public.traits_id_seq OWNED BY public.traits.id;
+--
+-- Name: trashed_groups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.trashed_groups (
+ group_uuid character varying,
+ trash_at timestamp without time zone
+);
+
+
--
-- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
---
--- Name: permission_refresh_lock id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.permission_refresh_lock ALTER COLUMN id SET DEFAULT nextval('public.permission_refresh_lock_id_seq'::regclass);
-
-
--
-- Name: pipeline_instances id; Type: DEFAULT; Schema: public; Owner: -
--
ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
---
--- Name: permission_refresh_lock permission_refresh_lock_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.permission_refresh_lock
- ADD CONSTRAINT permission_refresh_lock_pkey PRIMARY KEY (id);
-
-
--
-- Name: pipeline_instances pipeline_instances_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
CREATE UNIQUE INDEX index_traits_on_uuid ON public.traits USING btree (uuid);
+--
+-- Name: index_trashed_groups_on_group_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_trashed_groups_on_group_uuid ON public.trashed_groups USING btree (group_uuid);
+
+
--
-- Name: index_users_on_created_at; Type: INDEX; Schema: public; Owner: -
--
--
--- Name: permission_target_trashed; Type: INDEX; Schema: public; Owner: -
+-- Name: permission_target; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX permission_target_trashed ON public.materialized_permission_view USING btree (trashed, target_uuid);
+CREATE INDEX permission_target ON public.materialized_permissions USING btree (target_uuid);
--
--- Name: permission_target_user_trashed_level; Type: INDEX; Schema: public; Owner: -
+-- Name: permission_user_target; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX permission_target_user_trashed_level ON public.materialized_permission_view USING btree (user_uuid, trashed, perm_level);
+CREATE UNIQUE INDEX permission_user_target ON public.materialized_permissions USING btree (user_uuid, target_uuid);
--
('20190523180148'),
('20190808145904'),
('20190809135453'),
-('20190905151603');
+('20190905151603'),
+('20200501150153'),
+('20200602141328');
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# These constants are used in both
+# db/migrate/20200501150153_permission_table and update_permissions
+#
+# This file allows them to be easily imported by both to avoid duplication.
+#
+# Don't mess with this! Any changes will affect both the current
+# update_permissions and the past migration. If you are tinkering
+# with the permission system and need to change how
+# PERM_QUERY_TEMPLATE, refresh_trashed or refresh_permissions works,
+# you should make a new file with your modified functions and have
+# update_permissions reference that file instead.
+
+PERMISSION_VIEW = "materialized_permissions"
+
+TRASHED_GROUPS = "trashed_groups"
+
+# We need to use this parameterized query in a few different places,
+# including as a subquery in a larger query.
+#
+# There's basically two options, the way I did this originally was to
+# put this in a postgres function and do a lateral join over it.
+# However, postgres functions impose an optimization barrier, and
+# possibly have other overhead with temporary tables, so I ended up
+# going with the brute force approach of inlining the whole thing.
+#
+# The two substitutions are "base_case" which determines the initial
+# set of permission origins and "edge_perm" which is used to ensure
+# that the new permission takes precedence over the one in the edges
+# table (but some queries don't need that.)
+#
+PERM_QUERY_TEMPLATE = %{
+WITH RECURSIVE
+ traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+ %{base_case}
+ union
+ (select traverse_graph.origin_uuid,
+ edges.head_uuid,
+ least(%{edge_perm},
+ traverse_graph.val),
+ should_traverse_owned(edges.head_uuid, edges.val),
+ false
+ from permission_graph_edges as edges, traverse_graph
+ where traverse_graph.target_uuid = edges.tail_uuid
+ and (edges.tail_uuid like '_____-j7d0g-_______________' or
+ traverse_graph.starting_set)))
+ select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+ group by (traverse_graph.origin_uuid, target_uuid)
+}
+
+def refresh_trashed
+ ActiveRecord::Base.transaction do
+ ActiveRecord::Base.connection.execute("LOCK TABLE #{TRASHED_GROUPS}")
+ ActiveRecord::Base.connection.execute("DELETE FROM #{TRASHED_GROUPS}")
+
+ # Helper populate trashed_groups table. This starts with
+ # each group owned by a user and computes the subtree under that
+ # group to find any groups that are trashed.
+ ActiveRecord::Base.connection.execute(%{
+INSERT INTO #{TRASHED_GROUPS}
+select ps.target_uuid as group_uuid, ps.trash_at from groups,
+ lateral project_subtree_with_trash_at(groups.uuid, groups.trash_at) ps
+ where groups.owner_uuid like '_____-tpzed-_______________'
+})
+ end
+end
+
+def refresh_permissions
+ ActiveRecord::Base.transaction do
+ ActiveRecord::Base.connection.execute("LOCK TABLE #{PERMISSION_VIEW}")
+ ActiveRecord::Base.connection.execute("DELETE FROM #{PERMISSION_VIEW}")
+
+ ActiveRecord::Base.connection.execute %{
+INSERT INTO materialized_permissions
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select uuid, uuid, 3, true, true from users
+},
+:edge_perm => 'edges.val'
+} }
+}, "refresh_permission_view.do"
+ end
+end
ActiveRecord::Base.transaction do
Group.where(uuid: system_group_uuid).
first_or_create!(name: "System group",
- description: "System group") do |g|
+ description: "System group",
+ group_class: "role") do |g|
g.save!
User.all.collect(&:uuid).each do |user_uuid|
Link.create!(link_class: 'permission',
end
end
- def empty_collection_uuid
+ def empty_collection_pdh
'd41d8cd98f00b204e9800998ecf8427e+0'
end
act_as_system_user do
ActiveRecord::Base.transaction do
Collection.
- where(portable_data_hash: empty_collection_uuid).
- first_or_create!(manifest_text: '', owner_uuid: anonymous_group.uuid)
+ where(portable_data_hash: empty_collection_pdh).
+ first_or_create(manifest_text: '', owner_uuid: system_user.uuid, name: "empty collection") do |c|
+ c.save!
+ Link.where(tail_uuid: anonymous_group.uuid,
+ head_uuid: c.uuid,
+ link_class: 'permission',
+ name: 'can_read').
+ first_or_create!
+ c
+ end
end
end
end
# SPDX-License-Identifier: AGPL-3.0
module DbCurrentTime
- CURRENT_TIME_SQL = "SELECT clock_timestamp()"
+ CURRENT_TIME_SQL = "SELECT clock_timestamp() AT TIME ZONE 'UTC'"
def db_current_time
- Time.parse(ActiveRecord::Base.connection.select_value(CURRENT_TIME_SQL)).to_time
+ Time.parse(ActiveRecord::Base.connection.select_value(CURRENT_TIME_SQL) + " +0000")
+ end
+
+ def db_transaction_time
+ Time.parse(ActiveRecord::Base.connection.select_value("SELECT current_timestamp AT TIME ZONE 'UTC'") + " +0000")
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'update_permissions'
+
+include CurrentApiClient
+
+def fix_roles_projects
+ batch_update_permissions do
+ # This migration is not reversible. However, the behavior it
+ # enforces is backwards-compatible, and most of the time there
+ # shouldn't be anything to do at all.
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ Group.where("group_class != 'project' or group_class is null").each do |g|
+ # 1) any group not group_class != project becomes a 'role' (both empty and invalid groups)
+ old_owner = g.owner_uuid
+ g.owner_uuid = system_user_uuid
+ g.group_class = 'role'
+ g.save_with_unique_name!
+
+ if old_owner != system_user_uuid
+ # 2) Ownership of a role becomes a can_manage link
+ Link.new(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: old_owner,
+ head_uuid: g.uuid).
+ save!(validate: false)
+ end
+ end
+
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+ next if [ApiClientAuthorization,
+ AuthorizedKey,
+ Log,
+ Group].include?(klass)
+ next if !klass.columns.collect(&:name).include?('owner_uuid')
+
+ # 3) If a role owns anything, give it to system user and it
+ # becomes a can_manage link
+ klass.joins("join groups on groups.uuid=#{klass.table_name}.owner_uuid and groups.group_class='role'").each do |owned|
+ Link.new(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: owned.owner_uuid,
+ head_uuid: owned.uuid).
+ save!(validate: false)
+ owned.owner_uuid = system_user_uuid
+ owned.save_with_unique_name!
+ end
+ end
+
+ Group.joins("join groups as g2 on g2.uuid=groups.owner_uuid and g2.group_class='role'").each do |owned|
+ Link.new(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: owned.owner_uuid,
+ head_uuid: owned.uuid).
+ save!(validate: false)
+ owned.owner_uuid = system_user_uuid
+ owned.save_with_unique_name!
+ end
+
+ # 4) Projects can't have outgoing permission links. Just
+ # print a warning and delete them.
+ q = ActiveRecord::Base.connection.exec_query %{
+select links.uuid from links, groups where groups.uuid = links.tail_uuid and
+ links.link_class = 'permission' and groups.group_class = 'project'
+}
+ q.each do |lu|
+ ln = Link.find_by_uuid(lu['uuid'])
+ puts "WARNING: Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+ Rails.logger.warn "Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+ ln.destroy!
+ end
+ end
+ end
+ end
+end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-PERMISSION_VIEW = "materialized_permission_view"
-
-def do_refresh_permission_view
- ActiveRecord::Base.transaction do
- ActiveRecord::Base.connection.execute("LOCK TABLE permission_refresh_lock")
- ActiveRecord::Base.connection.execute("REFRESH MATERIALIZED VIEW #{PERMISSION_VIEW}")
- end
-end
-
-def refresh_permission_view(async=false)
- if async and Rails.configuration.API.AsyncPermissionsUpdateInterval > 0
- exp = Rails.configuration.API.AsyncPermissionsUpdateInterval.seconds
- need = false
- Rails.cache.fetch('AsyncRefreshPermissionView', expires_in: exp) do
- need = true
- end
- if need
- # Schedule a new permission update and return immediately
- Thread.new do
- Thread.current.abort_on_exception = false
- begin
- sleep(exp)
- Rails.cache.delete('AsyncRefreshPermissionView')
- do_refresh_permission_view
- rescue => e
- Rails.logger.error "Updating permission view: #{e}\n#{e.backtrace.join("\n\t")}"
- ensure
- ActiveRecord::Base.connection.close
- end
- end
- true
- end
- else
- do_refresh_permission_view
- end
-end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+REVOKE_PERM = 0
+CAN_MANAGE_PERM = 3
+
+def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+ return if Thread.current[:suppress_update_permissions]
+
+ #
+ # Update a subset of the permission table affected by adding or
+ # removing a particular permission relationship (ownership or a
+ # permission link).
+ #
+ # perm_origin_uuid: This is the object that 'gets' the permission.
+ # It is the owner_uuid or tail_uuid.
+ #
+ # starting_uuid: The object we are computing permission for (or head_uuid)
+ #
+ # perm_level: The level of permission that perm_origin_uuid gets for starting_uuid.
+ #
+ # perm_level is a number from 0-3
+ # can_read=1
+ # can_write=2
+ # can_manage=3
+ # or call with perm_level=0 to revoke permissions
+ #
+ # check: for testing/debugging, compare the result of the
+ # incremental update against a full table recompute. Throws an
+ # error if the contents are not identical (ie they produce different
+ # permission results)
+
+ # Theory of operation
+ #
+ # Give a change in a specific permission relationship, we recompute
+ # the set of permissions (for all users) that could possibly be
+ # affected by that relationship. For example, if a project is
+ # shared with another user, we recompute all permissions for all
+ # projects in the hierarchy. This returns a set of updated
+ # permissions, which we stash in a temporary table.
+ #
+ # Then, for each user_uuid/target_uuid in the updated permissions
+ # result set we insert/update a permission row in
+ # materialized_permissions, and delete any rows that exist in
+ # materialized_permissions that are not in the result set or have
+ # perm_level=0.
+ #
+ # see db/migrate/20200501150153_permission_table.rb for details on
+ # how the permissions are computed.
+
+ if edge_id.nil?
+ # For changes of ownership, edge_id is starting_uuid. In turns
+ # out most invocations of update_permissions are for changes of
+ # ownership, so make this parameter optional to reduce
+ # clutter.
+ # For permission links, the uuid of the link object will be passed in for edge_id.
+ edge_id = starting_uuid
+ end
+
+ ActiveRecord::Base.transaction do
+
+ # "Conflicts with the ROW EXCLUSIVE, SHARE UPDATE EXCLUSIVE, SHARE
+ # ROW EXCLUSIVE, EXCLUSIVE, and ACCESS EXCLUSIVE lock modes. This
+ # mode protects a table against concurrent data changes."
+ ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in SHARE MODE"
+
+ # Workaround for
+ # BUG #15160: planner overestimates number of rows in join when there are more than 200 rows coming from CTE
+ # https://www.postgresql.org/message-id/152395805004.19366.3107109716821067806@wrigleys.postgresql.org
+ #
+ # For a crucial join in the compute_permission_subgraph() query, the
+ # planner mis-estimates the number of rows in a Common Table
+ # Expression (CTE, this is a subquery in a WITH clause) and as a
+ # result it chooses the wrong join order. The join starts with the
+ # permissions table because it mistakenly thinks
+ # count(materalized_permissions) < count(new computed permissions)
+ # when actually it is the other way around.
+ #
+ # Because of the incorrect join order, it choose the wrong join
+ # strategy (merge join, which works best when two tables are roughly
+ # the same size). As a workaround, we can tell it not to use that
+ # join strategy, this causes it to pick hash join instead, which
+ # turns out to be a bit better. However, because the join order is
+ # still wrong, we don't get the full benefit of the index.
+ #
+ # This is very unfortunate because it makes the query performance
+ # dependent on the size of the materalized_permissions table, when
+ # the goal of this design was to make permission updates scale-free
+ # and only depend on the number of permissions affected and not the
+ # total table size. In several hours of researching I wasn't able
+ # to find a way to force the correct join order, so I'm calling it
+ # here and I have to move on.
+ #
+ # This is apparently addressed in Postgres 12, but I developed &
+ # tested this on Postgres 9.6, so in the future we should reevaluate
+ # the performance & query plan on Postgres 12.
+ #
+ # https://git.furworks.de/opensourcemirror/postgresql/commit/a314c34079cf06d05265623dd7c056f8fa9d577f
+ #
+ # Disable merge join for just this query (also local for this transaction), then reenable it.
+ ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
+
+ temptable_perms = "temp_perms_#{rand(2**64).to_s(10)}"
+ ActiveRecord::Base.connection.exec_query %{
+create temporary table #{temptable_perms} on commit drop
+as select * from compute_permission_subgraph($1, $2, $3, $4)
+},
+ 'update_permissions.select',
+ [[nil, perm_origin_uuid],
+ [nil, starting_uuid],
+ [nil, perm_level],
+ [nil, edge_id]]
+
+ ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to true;"
+
+ ActiveRecord::Base.connection.exec_delete %{
+delete from #{PERMISSION_VIEW} where
+ target_uuid in (select target_uuid from #{temptable_perms}) and
+ not exists (select 1 from #{temptable_perms}
+ where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+ user_uuid=#{PERMISSION_VIEW}.user_uuid and
+ val>0)
+},
+ "update_permissions.delete"
+
+ ActiveRecord::Base.connection.exec_query %{
+insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
+ select user_uuid, target_uuid, val as perm_level, traverse_owned from #{temptable_perms} where val>0
+on conflict (user_uuid, target_uuid) do update set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned;
+},
+ "update_permissions.insert"
+
+ if perm_level>0
+ check_permissions_against_full_refresh
+ end
+ end
+end
+
+
+def check_permissions_against_full_refresh
+ # No-op except when running tests
+ return unless Rails.env == 'test' and !Thread.current[:no_check_permissions_against_full_refresh] and !Thread.current[:suppress_update_permissions]
+
+ # For checking correctness of the incremental permission updates.
+ # Check contents of the current 'materialized_permission' table
+ # against a from-scratch permission refresh.
+
+ q1 = ActiveRecord::Base.connection.exec_query %{
+select user_uuid, target_uuid, perm_level, traverse_owned from #{PERMISSION_VIEW}
+order by user_uuid, target_uuid
+}, "check_permissions_against_full_refresh.permission_table"
+
+ q2 = ActiveRecord::Base.connection.exec_query %{
+ select pq.origin_uuid as user_uuid, target_uuid, pq.val as perm_level, pq.traverse_owned from (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select uuid, uuid, 3, true, true from users
+},
+:edge_perm => 'edges.val'
+} }) as pq order by origin_uuid, target_uuid
+}, "check_permissions_against_full_refresh.full_recompute"
+
+ if q1.count != q2.count
+ puts "Didn't match incremental+: #{q1.count} != full refresh-: #{q2.count}"
+ end
+
+ if q1.count > q2.count
+ q1.each_with_index do |r, i|
+ if r != q2[i]
+ puts "+#{r}\n-#{q2[i]}"
+ raise "Didn't match"
+ end
+ end
+ else
+ q2.each_with_index do |r, i|
+ if r != q1[i]
+ puts "+#{q1[i]}\n-#{r}"
+ raise "Didn't match"
+ end
+ end
+ end
+end
+
+def skip_check_permissions_against_full_refresh
+ check_perm_was = Thread.current[:no_check_permissions_against_full_refresh]
+ Thread.current[:no_check_permissions_against_full_refresh] = true
+ begin
+ yield
+ ensure
+ Thread.current[:no_check_permissions_against_full_refresh] = check_perm_was
+ end
+end
+
+def batch_update_permissions
+ check_perm_was = Thread.current[:suppress_update_permissions]
+ Thread.current[:suppress_update_permissions] = true
+ begin
+ yield
+ ensure
+ Thread.current[:suppress_update_permissions] = check_perm_was
+ refresh_permissions
+ end
+end
+
+# Used to account for permissions that a user gains by having
+# can_manage on another user.
+#
+# note: in theory a user could have can_manage access to a user
+# through multiple levels, that isn't handled here (would require a
+# recursive query). I think that's okay because users getting
+# transitive access through "can_manage" on a user is is rarely/never
+# used feature and something we probably want to deprecate and remove.
+USER_UUIDS_SUBQUERY_TEMPLATE = %{
+select target_uuid from materialized_permissions where user_uuid in (%{user})
+and target_uuid like '_____-tpzed-_______________' and traverse_owned=true and perm_level >= %{perm_level}
+}
empty:
uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
current_version_uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
- # Empty collection owned by anonymous_group is added with rake db:seed.
portable_data_hash: d41d8cd98f00b204e9800998ecf8427e+0
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-06-11T17:22:54Z
modified_at: 2014-06-11T17:22:54Z
updated_at: 2014-06-11T17:22:54Z
manifest_text: ""
- name: empty_collection
+ name: "empty collection for python test"
foo_collection_in_aproject:
uuid: zzzzz-4zz18-fy296fx3hot09f7
uuid: zzzzz-j7d0g-it30l961gq3t0oi
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
name: Public
- description: Public Group
+ description: Public Project
+ group_class: project
+
+public_role:
+ uuid: zzzzz-j7d0g-jt30l961gq3t0oi
+ owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ name: Public Role
+ description: Public Role
group_class: role
private:
uuid: zzzzz-j7d0g-rew6elm53kancon
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
name: Private
- description: Private Group
+ description: Private Project
+ group_class: project
+
+private_role:
+ uuid: zzzzz-j7d0g-pew6elm53kancon
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: Private Role
+ description: Private Role
group_class: role
private_and_can_read_foofile:
uuid: zzzzz-j7d0g-22xp1wpjul508rk
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
name: Private and Can Read Foofile
description: Another Private Group
group_class: role
uuid: zzzzz-j7d0g-48foin4vonvc2at
owner_uuid: zzzzz-tpzed-000000000000000
name: Administrators of a subset of users
+ group_class: role
aproject:
uuid: zzzzz-j7d0g-v955i6s2oi1cbso
future_project_viewing_group:
uuid: zzzzz-j7d0g-futrprojviewgrp
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-04-21 15:37:48 -0400
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_at: 2014-05-03 18:50:08 -0400
updated_at: 2014-05-03 18:50:08 -0400
name: Owned by bad group b
+ group_class: project
bad_group_has_ownership_cycle_b:
uuid: zzzzz-j7d0g-0077nzts8c178lw
modified_at: 2014-05-03 18:50:08 -0400
updated_at: 2014-05-03 18:50:08 -0400
name: Owned by bad group a
+ group_class: project
anonymous_group:
uuid: zzzzz-j7d0g-anonymouspublic
uuid: zzzzz-j7d0g-ptt1ou6a9lxrv07
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
name: Active user has can_manage
+ group_class: project
# Group for testing granting permission between users who share a group.
group_for_sharing_tests:
description: Test project belonging to FUSE test user
group_class: project
-group_with_no_class:
- uuid: zzzzz-j7d0g-groupwithnoclas
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: 2014-04-21 15:37:48 -0400
- modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
- modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- modified_at: 2014-04-21 15:37:48 -0400
- updated_at: 2014-04-21 15:37:48 -0400
- name: group_with_no_class
- description: This group has no class at all. So rude!
-
# This wouldn't pass model validation, but it enables a workbench
# infinite-loop test. See #4389
project_owns_itself:
trash_at: 2001-01-01T00:00:00Z
delete_at: 2038-03-01T00:00:00Z
is_trashed: false
- modified_at: 2001-01-01T00:00:00Z
\ No newline at end of file
+ modified_at: 2001-01-01T00:00:00Z
head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
properties: {}
-foo_file_readable_by_aproject:
+foo_file_readable_by_project_viewer:
uuid: zzzzz-o0j2j-fp1d8395ldqw22p
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-01-24 20:42:26 -0800
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ tail_uuid: zzzzz-tpzed-projectviewer1a
link_class: permission
name: can_read
head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
name: can_read
head_uuid: zzzzz-4zz18-taggedcolletion
properties: {}
+
+active_manages_viewing_group:
+ uuid: zzzzz-o0j2j-activemanagesvi
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_manage
+ head_uuid: zzzzz-j7d0g-futrprojviewgrp
+ properties: {}
organization: example.com
role: Computational biologist
getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
+has_can_login_permission:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-xabcdjxw79nv3jz
+ email: can-login-user@arvados.local
+ modified_by_client_uuid: zzzzz-ozdt8-teyxzyd8qllg11h
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ first_name: Can_login
+ last_name: User
+ identity_url: https://can-login-user.openid.local
+ is_active: true
+ is_admin: false
+ modified_at: 2015-03-26 12:34:56.789000000 Z
+ username: can-login-user
@controller = Arvados::V1::GroupsController.new
authorize_with :active
post :create, params: {
- group: {},
+ group: {group_class: "project"},
ensure_unique_name: boolparam
}
assert_response :success
post :create, params: {
group: {
name: groups(:aproject).name,
- owner_uuid: groups(:aproject).owner_uuid
+ owner_uuid: groups(:aproject).owner_uuid,
+ group_class: "project"
},
ensure_unique_name: boolparam
}
class Arvados::V1::FiltersTest < ActionController::TestCase
test '"not in" filter passes null values' do
- @controller = Arvados::V1::GroupsController.new
+ @controller = Arvados::V1::ContainerRequestsController.new
authorize_with :admin
get :index, params: {
- filters: [ ['group_class', 'not in', ['project']] ],
- controller: 'groups',
+ filters: [ ['container_uuid', 'not in', ['zzzzz-dz642-queuedcontainer', 'zzzzz-dz642-runningcontainr']] ],
+ controller: 'container_requests',
}
assert_response :success
found = assigns(:objects)
- assert_includes(found.collect(&:group_class), nil,
- "'group_class not in ['project']' filter should pass null")
+ assert_includes(found.collect(&:container_uuid), nil,
+ "'container_uuid not in [zzzzz-dz642-queuedcontainer, zzzzz-dz642-runningcontainr]' filter should pass null")
end
test 'error message for non-array element in filters array' do
end
assert_includes group_uuids, groups(:aproject).uuid
assert_includes group_uuids, groups(:asubproject).uuid
+ assert_includes group_uuids, groups(:private).uuid
assert_not_includes group_uuids, groups(:system_group).uuid
- assert_not_includes group_uuids, groups(:private).uuid
+ assert_not_includes group_uuids, groups(:private_and_can_read_foofile).uuid
end
test "get list of groups that are not projects" do
end
assert_not_includes group_uuids, groups(:aproject).uuid
assert_not_includes group_uuids, groups(:asubproject).uuid
- assert_includes group_uuids, groups(:private).uuid
- assert_includes group_uuids, groups(:group_with_no_class).uuid
end
test "get list of groups with bogus group_class" do
### trashed project tests ###
- [:active, :admin].each do |auth|
+ #
+ # The structure is
+ #
+ # trashed_project (zzzzz-j7d0g-trashedproject1)
+ # trashed_subproject (zzzzz-j7d0g-trashedproject2)
+ # trashed_subproject3 (zzzzz-j7d0g-trashedproject3)
+ # zzzzz-xvhdp-cr5trashedcontr
+
+ [:active,
+ :admin].each do |auth|
# project: to query, to untrash, is visible, parent contents listing success
- [[:trashed_project, [], false, true],
+ [
+ [:trashed_project, [], false, true],
[:trashed_project, [:trashed_project], true, true],
[:trashed_subproject, [], false, false],
[:trashed_subproject, [:trashed_project], true, true],
assert_equal 0, json_response['included'].length
end
- test 'get shared, owned by non-project' do
+ test 'get shared, add permission link' do
authorize_with :user_bar_in_sharing_group
act_as_system_user do
- Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+ Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+ head_uuid: groups(:project_owned_by_foo).uuid,
+ link_class: 'permission',
+ name: 'can_manage')
end
get :shared, params: {:filters => [["group_class", "=", "project"]], :include => "owner_uuid"}
assert_equal 1, json_response['items'].length
- assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+ assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
assert_equal 1, json_response['included'].length
- assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+ assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
end
### contents with exclude_home_project
assert_equal 0, json_response['included'].length
end
- test 'contents, exclude home, owned by non-project' do
+ test 'contents, exclude home, add permission link' do
authorize_with :user_bar_in_sharing_group
act_as_system_user do
- Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+ Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+ head_uuid: groups(:project_owned_by_foo).uuid,
+ link_class: 'permission',
+ name: 'can_manage')
end
get :contents, params: {:include => "owner_uuid", :exclude_home_project => true}
assert_equal 1, json_response['items'].length
- assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+ assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
assert_equal 1, json_response['included'].length
- assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+ assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
end
test 'contents, exclude home, with parent specified' do
headers: auth(:active)
assert_response :success
json_response['items'].each do |svc|
- url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}"
+ url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}/"
assert_equal true, expect_rvz.has_key?(url), "#{url} does not match any configured service: expecting #{expect_rvz}"
rvz = expect_rvz[url]
if rvz.is_a? String
test "get_all_permissions obeys group permissions" do
act_as_user system_user do
r = Repository.create!(name: 'admin/groupcanwrite', owner_uuid: users(:admin).uuid)
- g = Group.create!(group_class: 'group', name: 'repo-writers')
+ g = Group.create!(group_class: 'role', name: 'repo-writers')
u1 = users(:active)
u2 = users(:spectator)
Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_manage')
Link.create!(tail_uuid: u2.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_read')
r = Repository.create!(name: 'admin/groupreadonly', owner_uuid: users(:admin).uuid)
- g = Group.create!(group_class: 'group', name: 'repo-readers')
+ g = Group.create!(group_class: 'role', name: 'repo-readers')
u1 = users(:active)
u2 = users(:spectator)
Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_read')
test "non-admin user gets only safe attributes from users#show" do
g = act_as_system_user do
- create :group
+ create :group, group_class: "role"
end
users = create_list :active_user, 2, join_groups: [g]
token = create :token, user: users[0]
[2, 4].each do |limit|
test "non-admin user can limit index to #{limit}" do
g = act_as_system_user do
- create :group
+ create :group, group_class: "role"
end
users = create_list :active_user, 4, join_groups: [g]
token = create :token, user: users[0]
assert_response :success
end
- test "create request with async=true defers permissions update" do
+ test "create request with async=true does not defer permissions update" do
Rails.configuration.API.AsyncPermissionsUpdateInterval = 1 # second
name = "Random group #{rand(1000)}"
assert_equal nil, Group.find_by_name(name)
+ # Following the implementation of incremental permission updates
+ # (#16007) the async flag is now a no-op. Permission changes are
+ # visible immediately.
+
# Trigger the asynchronous permission update by using async=true parameter.
post "/arvados/v1/groups",
params: {
group: {
- name: name
+ name: name,
+ group_class: "project"
},
async: true
},
headers: auth(:active)
assert_response 202
- # The group exists on the database, but it's not accessible yet.
+ # The group exists in the database
assert_not_nil Group.find_by_name(name)
get "/arvados/v1/groups",
params: {
},
headers: auth(:active)
assert_response 200
- assert_equal 0, json_response['items_available']
+ assert_equal 1, json_response['items_available']
# Wait a bit and try again.
sleep(1)
class PermissionsTest < ActionDispatch::IntegrationTest
include DbCurrentTime
- include CurrentApiClient # for empty_collection
fixtures :users, :groups, :api_client_authorizations, :collections
test "adding and removing direct can_read links" do
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: collections(:foo_file).uuid,
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: collections(:foo_file).uuid,
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: groups(:empty_lonely_group).uuid,
test "active user can read the empty collection" do
# The active user should be able to read the empty collection.
- get("/arvados/v1/collections/#{empty_collection_uuid}",
+ get("/arvados/v1/collections/#{empty_collection_pdh}",
params: {:format => :json},
headers: auth(:active))
assert_response :success
act_as_system_user do
puts("Time spent creating records:", Benchmark.measure do
ActiveRecord::Base.transaction do
- root = Group.create!(owner_uuid: users(:permission_perftest).uuid)
+ root = Group.create!(owner_uuid: users(:permission_perftest).uuid, group_class: "project")
n += 1
a = create_eight root.uuid
n += 8
end
end
end
- User.invalidate_permissions_cache
+ refresh_permissions
end
end)
end
#
# SPDX-License-Identifier: AGPL-3.0
+require 'update_permissions'
+
ENV["RAILS_ENV"] = "test"
unless ENV["NO_COVERAGE_TEST"]
begin
end
# Ensure permissions are computed from the test fixtures.
-User.invalidate_permissions_cache
+refresh_permissions
+refresh_trashed
while longstring.length < 2**16
longstring = longstring + longstring
end
- g = Group.create! name: 'Has a long description', description: longstring
+ g = Group.create! name: 'Has a long description', description: longstring, group_class: "project"
g = Group.find_by_uuid g.uuid
assert_equal g.description, longstring
end
test 'create and retrieve using created_at time' do
set_user_from_auth :active
- group = Group.create! name: 'test create and retrieve group'
+ group = Group.create! name: 'test create and retrieve group', group_class: "project"
assert group.valid?, "group is not valid"
results = Group.where(created_at: group.created_at)
test 'create and update twice and expect different update times' do
set_user_from_auth :active
- group = Group.create! name: 'test create and retrieve group'
+ group = Group.create! name: 'test create and retrieve group', group_class: "project"
assert group.valid?, "group is not valid"
# update 1
test "delete referring links in SweepTrashedObjects" do
uuid = collections(:trashed_on_next_sweep).uuid
act_as_system_user do
+ assert_raises ActiveRecord::RecordInvalid do
+ # Cannot create because :trashed_on_next_sweep is already trashed
+ Link.create!(head_uuid: uuid,
+ tail_uuid: system_user_uuid,
+ link_class: 'whatever',
+ name: 'something')
+ end
+
+ # Bump trash_at to now + 1 minute
+ Collection.where(uuid: uuid).
+ update(trash_at: db_current_time + (1).minute)
+
+ # Not considered trashed now
Link.create!(head_uuid: uuid,
tail_uuid: system_user_uuid,
link_class: 'whatever',
auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
assert_operator auth_exp, :<, db_current_time
+
+ assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
end
test "Exceed maximum lock-unlock cycles" do
# SPDX-License-Identifier: AGPL-3.0
require 'test_helper'
+require 'fix_roles_projects'
class GroupTest < ActiveSupport::TestCase
+ include DbCurrentTime
test "cannot set owner_uuid to object with existing ownership cycle" do
set_user_from_auth :active_trustedclient
test "cannot create a new ownership cycle" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar")
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", group_class: "project")
g_foo.owner_uuid = g_bar.uuid
assert g_foo.save, lambda { g_foo.errors.messages }
test "cannot create a single-object ownership cycle" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "project")
assert g_foo.save
# Ensure I have permission to manage this group even when its owner changes
assert g_foo.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
end
+ test "cannot create a group that is not a 'role' or 'project'" do
+ set_user_from_auth :active_trustedclient
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo")
+ end
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo", group_class: "")
+ end
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo", group_class: "bogus")
+ end
+ end
+
+ test "cannot change group_class on an already created group" do
+ set_user_from_auth :active_trustedclient
+ g = Group.create!(name: "foo", group_class: "role")
+ assert_raises(ActiveRecord::RecordInvalid) do
+ g.update_attributes!(group_class: "project")
+ end
+ end
+
+ test "role cannot own things" do
+ set_user_from_auth :active_trustedclient
+ role = Group.create!(name: "foo", group_class: "role")
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ Collection.create!(name: "bzzz123", owner_uuid: role.uuid)
+ end
+
+ c = Collection.create!(name: "bzzz124")
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ c.update_attributes!(owner_uuid: role.uuid)
+ end
+ end
+
test "trash group hides contents" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "project")
col = Collection.create!(owner_uuid: g_foo.uuid)
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).any?
test "trash group" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trash subgroup" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trash subsubgroup" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trashed does not propagate across permission links" do
set_user_from_auth :admin
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "role")
u_bar = User.create!(first_name: "bar")
assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
set_user_from_auth :active
["", "{SOLIDUS}"].each do |subst|
Rails.configuration.Collections.ForwardSlashNameSubstitution = subst
- g = Group.create
+ proj = Group.create group_class: "project"
+ role = Group.create group_class: "role"
[[nil, true],
["", true],
[".", false],
["../..", subst != ""],
["/", subst != ""],
].each do |name, valid|
- g.name = name
- g.group_class = "role"
- assert_equal true, g.valid?
- g.group_class = "project"
- assert_equal valid, g.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
+ role.name = name
+ assert_equal true, role.valid?
+ proj.name = name
+ assert_equal valid, proj.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
end
end
end
+
+ def insert_group uuid, owner_uuid, name, group_class
+ q = ActiveRecord::Base.connection.exec_query %{
+insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
+ values ('#{uuid}', '#{owner_uuid}',
+ '#{name}', #{if group_class then "'"+group_class+"'" else 'NULL' end},
+ statement_timestamp(), statement_timestamp())
+}
+ uuid
+ end
+
+ test "migration to fix roles and projects" do
+ g1 = insert_group Group.generate_uuid, system_user_uuid, 'group with no class', nil
+ g2 = insert_group Group.generate_uuid, users(:active).uuid, 'role owned by a user', 'role'
+
+ g3 = insert_group Group.generate_uuid, system_user_uuid, 'role that owns a project', 'role'
+ g4 = insert_group Group.generate_uuid, g3, 'the project', 'project'
+
+ g5 = insert_group Group.generate_uuid, users(:active).uuid, 'a project with an outgoing permission link', 'project'
+
+ g6 = insert_group Group.generate_uuid, system_user_uuid, 'name collision', 'role'
+ g7 = insert_group Group.generate_uuid, users(:active).uuid, 'name collision', 'role'
+
+ g8 = insert_group Group.generate_uuid, users(:active).uuid, 'trashed with no class', nil
+ g8obj = Group.find_by_uuid(g8)
+ g8obj.trash_at = db_current_time
+ g8obj.delete_at = db_current_time
+ act_as_system_user do
+ g8obj.save!(validate: false)
+ end
+
+ refresh_permissions
+
+ act_as_system_user do
+ l1 = Link.create!(link_class: 'permission', name: 'can_manage', tail_uuid: g3, head_uuid: g4)
+ q = ActiveRecord::Base.connection.exec_query %{
+update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
+}
+ refresh_permissions
+ end
+
+ assert_equal nil, Group.find_by_uuid(g1).group_class
+ assert_equal nil, Group.find_by_uuid(g8).group_class
+ assert_equal users(:active).uuid, Group.find_by_uuid(g2).owner_uuid
+ assert_equal g3, Group.find_by_uuid(g4).owner_uuid
+ assert !Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+ assert !Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+ assert Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+
+ fix_roles_projects
+
+ assert_equal 'role', Group.find_by_uuid(g1).group_class
+ assert_equal 'role', Group.find_by_uuid(g8).group_class
+ assert_equal system_user_uuid, Group.find_by_uuid(g2).owner_uuid
+ assert_equal system_user_uuid, Group.find_by_uuid(g4).owner_uuid
+ assert Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+ assert Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+ assert !Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+ end
end
Group.all
[User, Group].each do |o_class|
test "create object with legit #{o_class} owner" do
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
i = Specimen.create(owner_uuid: o.uuid)
assert i.valid?, "new item should pass validation"
assert i.uuid, "new item should have an ID"
[User, Group].each do |new_o_class|
test "change owner from legit #{o_class} to legit #{new_o_class} owner" do
- o = o_class.create!
+ o = if o_class == Group
+ o_class.create! group_class: "project"
+ else
+ o_class.create!
+ end
i = Specimen.create!(owner_uuid: o.uuid)
- new_o = new_o_class.create!
+
+ new_o = if new_o_class == Group
+ new_o_class.create! group_class: "project"
+ else
+ new_o_class.create!
+ end
+
assert(Specimen.where(uuid: i.uuid).any?,
"new item should really be in DB")
assert(i.update_attributes(owner_uuid: new_o.uuid),
end
test "delete #{o_class} that owns nothing" do
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
assert(o_class.where(uuid: o.uuid).any?,
"new #{o_class} should really be in DB")
assert(o.destroy, "should delete #{o_class} that owns nothing")
test "change uuid of #{o_class} that owns nothing" do
# (we're relying on our admin credentials here)
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
assert(o_class.where(uuid: o.uuid).any?,
"new #{o_class} should really be in DB")
old_uuid = o.uuid
new_uuid = o.uuid.sub(/..........$/, rand(2**256).to_s(36)[0..9])
- assert(o.update_attributes(uuid: new_uuid),
- "should change #{o_class} uuid from #{old_uuid} to #{new_uuid}")
+ if o.respond_to? :update_uuid
+ o.update_uuid(new_uuid: new_uuid)
+ else
+ assert(o.update_attributes(uuid: new_uuid),
+ "should change #{o_class} uuid from #{old_uuid} to #{new_uuid}")
+ end
assert_equal(false, o_class.where(uuid: old_uuid).any?,
"#{old_uuid} should disappear when renamed to #{new_uuid}")
end
assert_equal(true, Specimen.where(owner_uuid: o.uuid).any?,
"need something to be owned by #{o.uuid} for this test")
- assert_raises(ActiveRecord::DeleteRestrictionError,
- "should not delete #{ofixt} that owns objects") do
- o.destroy
+ skip_check_permissions_against_full_refresh do
+ assert_raises(ActiveRecord::DeleteRestrictionError,
+ "should not delete #{ofixt} that owns objects") do
+ o.destroy
+ end
end
end
assert User.where(uuid: o.uuid).any?, "new User should really be in DB"
assert_equal(true, o.update_attributes(owner_uuid: o.uuid),
"setting owner to self should work")
- assert(o.destroy, "should delete User that owns self")
+
+ skip_check_permissions_against_full_refresh do
+ assert(o.destroy, "should delete User that owns self")
+ end
+
assert_equal(false, User.where(uuid: o.uuid).any?,
"#{o.uuid} should not be in DB after deleting")
+ check_permissions_against_full_refresh
end
test "change uuid of User that owns self" do
"setting owner to self should work")
old_uuid = o.uuid
new_uuid = o.uuid.sub(/..........$/, rand(2**256).to_s(36)[0..9])
- assert(o.update_attributes(uuid: new_uuid),
- "should change uuid of User that owns self")
+ o.update_uuid(new_uuid: new_uuid)
+ o = User.find_by_uuid(new_uuid)
assert_equal(false, User.where(uuid: old_uuid).any?,
"#{old_uuid} should not be in DB after deleting")
assert_equal(true, User.where(uuid: new_uuid).any?,
test "Grant permissions on an object I own" do
set_user_from_auth :active_trustedclient
- ob = Specimen.create
+ ob = Collection.create
assert ob.save
# Ensure I have permission to manage this group even when its owner changes
test "Delete permission links when deleting an object" do
set_user_from_auth :active_trustedclient
- ob = Specimen.create!
+ ob = Collection.create!
Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
link_class: 'permission',
test "permission links owned by root" do
set_user_from_auth :active_trustedclient
- ob = Specimen.create!
+ ob = Collection.create!
perm_link = Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
link_class: 'permission',
end
test "readable_by" do
- set_user_from_auth :active_trustedclient
+ set_user_from_auth :admin
- ob = Specimen.create!
+ ob = Collection.create!
Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
link_class: 'permission',
name: 'can_read')
- assert Specimen.readable_by(users(:active)).where(uuid: ob.uuid).any?, "user does not have read permission"
+ assert Collection.readable_by(users(:active)).where(uuid: ob.uuid).any?, "user does not have read permission"
end
test "writable_by" do
- set_user_from_auth :active_trustedclient
+ set_user_from_auth :admin
- ob = Specimen.create!
+ ob = Collection.create!
Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
link_class: 'permission',
assert ob.writable_by.include?(users(:active).uuid), "user does not have write permission"
end
+ test "update permission link" do
+ set_user_from_auth :admin
+
+ grp = Group.create! name: "blah project", group_class: "project"
+ ob = Collection.create! owner_uuid: grp.uuid
+
+ assert !users(:active).can?(write: ob)
+ assert !users(:active).can?(read: ob)
+
+ l1 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_write')
+
+ assert users(:active).can?(write: ob)
+ assert users(:active).can?(read: ob)
+
+ l1.update_attributes!(name: 'can_read')
+
+ assert !users(:active).can?(write: ob)
+ assert users(:active).can?(read: ob)
+
+ l1.destroy
+
+ assert !users(:active).can?(write: ob)
+ assert !users(:active).can?(read: ob)
+ end
+
test "writable_by reports requesting user's own uuid for a writable project" do
invited_to_write = users(:project_viewer)
group = groups(:asubproject)
test "user owns group, group can_manage object's group, user can add permissions" do
set_user_from_auth :admin
- owner_grp = Group.create!(owner_uuid: users(:active).uuid)
+ owner_grp = Group.create!(owner_uuid: users(:active).uuid, group_class: "role")
- sp_grp = Group.create!
- sp = Specimen.create!(owner_uuid: sp_grp.uuid)
+ sp_grp = Group.create!(group_class: "project")
Link.create!(link_class: 'permission',
name: 'can_manage',
tail_uuid: owner_grp.uuid,
head_uuid: sp_grp.uuid)
+ sp = Collection.create!(owner_uuid: sp_grp.uuid)
+
# active user owns owner_grp, which has can_manage permission on sp_grp
# user should be able to add permissions on sp.
set_user_from_auth :active_trustedclient
skip "can_manage permission on a non-group object" do
set_user_from_auth :admin
- ob = Specimen.create!
+ ob = Collection.create!
# grant can_manage permission to active
perm_link = Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
test "user without can_manage permission may not modify permission link" do
set_user_from_auth :admin
- ob = Specimen.create!
+ ob = Collection.create!
# grant can_manage permission to active
perm_link = Link.create!(tail_uuid: users(:active).uuid,
head_uuid: ob.uuid,
manager = create :active_user, first_name: "Manage", last_name: "Er"
minion = create :active_user, first_name: "Min", last_name: "Ion"
minions_specimen = act_as_user minion do
- Specimen.create!
+ g = Group.create! name: "minon project", group_class: "project"
+ Collection.create! owner_uuid: g.uuid
end
# Manager creates a group. (Make sure it doesn't magically give
# anyone any additional permissions.)
g = nil
act_as_user manager do
- g = create :group, name: "NoBigSecret Lab"
+ g = create :group, name: "NoBigSecret Lab", group_class: "role"
assert_empty(User.readable_by(manager).where(uuid: minion.uuid),
"saw a user I shouldn't see")
assert_raises(ArvadosModel::PermissionDeniedError,
create(:permission_link,
name: 'can_manage', tail_uuid: manager.uuid, head_uuid: minion.uuid)
end
- assert_empty(Specimen
+ assert_empty(Collection
.readable_by(manager)
.where(uuid: minions_specimen.uuid),
"manager saw the minion's private stuff")
act_as_user manager do
# Now, manager can read and write Minion's stuff.
- assert_not_empty(Specimen
+ assert_not_empty(Collection
.readable_by(manager)
.where(uuid: minions_specimen.uuid),
"manager could not find minion's specimen by uuid")
"#{a.first_name} should not be able to see 'b' in the user list")
act_as_system_user do
- g = create :group
+ g = create :group, group_class: "role"
[a,b].each do |u|
create(:permission_link,
name: 'can_read', tail_uuid: u.uuid, head_uuid: g.uuid)
"#{a.first_name} should be able to see 'b' in the user list")
a_specimen = act_as_user a do
- Specimen.create!
+ Collection.create!
end
- assert_not_empty(Specimen.readable_by(a).where(uuid: a_specimen.uuid),
- "A cannot read own Specimen, following test probably useless.")
- assert_empty(Specimen.readable_by(b).where(uuid: a_specimen.uuid),
- "B can read A's Specimen")
+ assert_not_empty(Collection.readable_by(a).where(uuid: a_specimen.uuid),
+ "A cannot read own Collection, following test probably useless.")
+ assert_empty(Collection.readable_by(b).where(uuid: a_specimen.uuid),
+ "B can read A's Collection")
[a,b].each do |u|
assert_empty(User.readable_by(u).where(uuid: other.uuid),
"#{u.first_name} can see OTHER in the user list")
test "cannot create with owner = unwritable user" do
set_user_from_auth :rominiadmin
assert_raises ArvadosModel::PermissionDeniedError, "created with owner = unwritable user" do
- Specimen.create!(owner_uuid: users(:active).uuid)
+ Collection.create!(owner_uuid: users(:active).uuid)
end
end
test "cannot change owner to unwritable user" do
set_user_from_auth :rominiadmin
- ob = Specimen.create!
+ ob = Collection.create!
assert_raises ArvadosModel::PermissionDeniedError, "changed owner to unwritable user" do
ob.update_attributes!(owner_uuid: users(:active).uuid)
end
test "cannot create with owner = unwritable group" do
set_user_from_auth :rominiadmin
assert_raises ArvadosModel::PermissionDeniedError, "created with owner = unwritable group" do
- Specimen.create!(owner_uuid: groups(:aproject).uuid)
+ Collection.create!(owner_uuid: groups(:aproject).uuid)
end
end
test "cannot change owner to unwritable group" do
set_user_from_auth :rominiadmin
- ob = Specimen.create!
+ ob = Collection.create!
assert_raises ArvadosModel::PermissionDeniedError, "changed owner to unwritable group" do
ob.update_attributes!(owner_uuid: groups(:aproject).uuid)
end
assert_not_empty container_logs(:running_older, :anonymous)
end
+
+ test "add user to group, then remove them" do
+ set_user_from_auth :admin
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+ col = Collection.create!(owner_uuid: system_user_uuid)
+
+ l0 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: col.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+ assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+ l1 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+ l2 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: users(:active).uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ l3 = Link.create!(tail_uuid: users(:project_viewer).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+ l4 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: users(:project_viewer).uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid).first
+
+ l1.destroy
+ l2.destroy
+
+ assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+ assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+ end
+
+
+ test "add user to group, then change permission level" do
+ set_user_from_auth :admin
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+ col = Collection.create!(owner_uuid: grp.uuid)
+ assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+ assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+ l1 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert users(:active).can?(write: col.uuid)
+ assert users(:active).can?(manage: col.uuid)
+
+ l1.name = 'can_read'
+ l1.save!
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert !users(:active).can?(write: col.uuid)
+ assert !users(:active).can?(manage: col.uuid)
+
+ l1.name = 'can_write'
+ l1.save!
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert users(:active).can?(write: col.uuid)
+ assert !users(:active).can?(manage: col.uuid)
+ end
+
+
+ test "add user to group, then add overlapping permission link to group" do
+ set_user_from_auth :admin
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+ col = Collection.create!(owner_uuid: grp.uuid)
+ assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+ assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+ l1 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert users(:active).can?(write: col.uuid)
+ assert users(:active).can?(manage: col.uuid)
+
+ l3 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert users(:active).can?(write: col.uuid)
+ assert users(:active).can?(manage: col.uuid)
+
+ l3.destroy!
+
+ assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+ assert users(:active).can?(read: col.uuid)
+ assert users(:active).can?(write: col.uuid)
+ assert users(:active).can?(manage: col.uuid)
+ end
+
+
+ test "add user to group, then add overlapping permission link to subproject" do
+ set_user_from_auth :admin
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+ prj = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+
+ l0 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: prj.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+
+ assert_empty Group.readable_by(users(:active)).where(uuid: prj.uuid)
+ assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+ l1 = Link.create!(tail_uuid: users(:active).uuid,
+ head_uuid: grp.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+ l2 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: users(:active).uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+ assert users(:active).can?(read: prj.uuid)
+ assert users(:active).can?(write: prj.uuid)
+ assert users(:active).can?(manage: prj.uuid)
+
+ l3 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: prj.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
+ assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+ assert users(:active).can?(read: prj.uuid)
+ assert users(:active).can?(write: prj.uuid)
+ assert users(:active).can?(manage: prj.uuid)
+
+ l3.destroy!
+
+ assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+ assert users(:active).can?(read: prj.uuid)
+ assert users(:active).can?(write: prj.uuid)
+ assert users(:active).can?(manage: prj.uuid)
+ end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'test_helper'
+
+class TimeZoneTest < ActiveSupport::TestCase
+ test "Database connection time zone" do
+ # This is pointless if the testing host is already using the UTC
+ # time zone. But if not, the test confirms that
+ # config/initializers/time_zone.rb has successfully changed the
+ # database connection time zone to UTC.
+ assert_equal('UTC', ActiveRecord::Base.connection.select_value("show timezone"))
+ end
+end
if auto_admin_first_user_config
# This test requires no admin users exist (except for the system user)
- users(:admin).delete
+ act_as_system_user do
+ users(:admin).update_attributes!(is_admin: false)
+ end
@all_users = User.where("uuid not like '%-000000000000000'").where(:is_admin => true)
assert_equal 0, @all_users.count, "No admin users should exist (except for the system user)"
end
vm = VirtualMachine.create
- # Set up the bogus Link
- bad_uuid = 'zzzzz-tpzed-xyzxyzxyzxyzxyz'
-
- resp_link = Link.create ({tail_uuid: email, link_class: 'permission',
- name: 'can_login', head_uuid: bad_uuid})
- resp_link.save(validate: false)
-
- verify_link resp_link, 'permission', 'can_login', email, bad_uuid
-
response = user.setup(repo_name: 'foo/testrepo',
vm_uuid: vm.uuid)
cmd = append(cmd, disp.cluster.Containers.CrunchRunArgumentsList...)
if err := disp.submit(ctr, cmd); err != nil {
var text string
- if err, ok := err.(dispatchcloud.ConstraintsNotSatisfiableError); ok {
+ switch err := err.(type) {
+ case dispatchcloud.ConstraintsNotSatisfiableError:
var logBuf bytes.Buffer
fmt.Fprintf(&logBuf, "cannot run container %s: %s\n", ctr.UUID, err)
if len(err.AvailableTypes) == 0 {
}
text = logBuf.String()
disp.UpdateState(ctr.UUID, dispatch.Cancelled)
- } else {
+ default:
text = fmt.Sprintf("Error submitting container %s to slurm: %s", ctr.UUID, err)
}
log.Print(text)
err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()})
c.Check(err, IsNil)
- c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg")
c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
fpm_depends+=(fuse-libs)
;;
debian* | ubuntu*)
- fpm_depends+=(libcurl3-gnutls libpython2.7)
+ fpm_depends+=(libcurl3-gnutls)
;;
esac
super(FuseMagicTest, self).setUp(api=api)
self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
- self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
+ self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
cw = arvados.CollectionWriter()
import (
"bytes"
+ "context"
"crypto/md5"
"fmt"
"io"
defer bal.time("sweep", "wall clock time to run one full sweep")()
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(cluster.Collections.BalanceTimeout.Duration()))
+ defer cancel()
+
var lbFile *os.File
if bal.LostBlocksFile != "" {
tmpfn := bal.LostBlocksFile + ".tmp"
if err = bal.CheckSanityEarly(client); err != nil {
return
}
+
+ // On a big site, indexing and sending trash/pull lists can
+ // take much longer than the usual 5 minute client
+ // timeout. From here on, we rely on the context deadline
+ // instead, aborting the entire operation if any part takes
+ // too long.
+ client.Timeout = 0
+
rs := bal.rendezvousState()
if runOptions.CommitTrash && rs != runOptions.SafeRendezvousState {
if runOptions.SafeRendezvousState != "" {
bal.logf("notice: KeepServices list has changed since last run")
}
bal.logf("clearing existing trash lists, in case the new rendezvous order differs from previous run")
- if err = bal.ClearTrashLists(client); err != nil {
+ if err = bal.ClearTrashLists(ctx, client); err != nil {
return
}
// The current rendezvous state becomes "safe" (i.e.,
// succeed in clearing existing trash lists.
nextRunOptions.SafeRendezvousState = rs
}
- if err = bal.GetCurrentState(client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
+
+ if err = bal.GetCurrentState(ctx, client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
return
}
bal.ComputeChangeSets()
lbFile = nil
}
if runOptions.CommitPulls {
- err = bal.CommitPulls(client)
+ err = bal.CommitPulls(ctx, client)
if err != nil {
// Skip trash if we can't pull. (Too cautious?)
return
}
}
if runOptions.CommitTrash {
- err = bal.CommitTrash(client)
+ err = bal.CommitTrash(ctx, client)
}
return
}
// We avoid this problem if we clear all trash lists before getting
// indexes. (We also assume there is only one rebalancing process
// running at a time.)
-func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
+func (bal *Balancer) ClearTrashLists(ctx context.Context, c *arvados.Client) error {
for _, srv := range bal.KeepServices {
srv.ChangeSet = &ChangeSet{}
}
- return bal.CommitTrash(c)
+ return bal.CommitTrash(ctx, c)
}
// GetCurrentState determines the current replication state, and the
// collection manifests in the database (API server).
//
// It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
+func (bal *Balancer) GetCurrentState(ctx context.Context, c *arvados.Client, pageSize, bufs int) error {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
defer bal.time("get_state", "wall clock time to get current state")()
bal.BlockStateMap = NewBlockStateMap()
go func(mounts []*KeepMount) {
defer wg.Done()
bal.logf("mount %s: retrieve index from %s", mounts[0], mounts[0].KeepService)
- idx, err := mounts[0].KeepService.IndexMount(c, mounts[0].UUID, "")
+ idx, err := mounts[0].KeepService.IndexMount(ctx, c, mounts[0].UUID, "")
if err != nil {
select {
case errs <- fmt.Errorf("%s: retrieve index: %v", mounts[0], err):
default:
}
+ cancel()
return
}
if len(errs) > 0 {
}
for range collQ {
}
+ cancel()
return
}
bal.collScanned++
wg.Add(1)
go func() {
defer wg.Done()
- err = EachCollection(c, pageSize,
+ err = EachCollection(ctx, c, pageSize,
func(coll arvados.Collection) error {
collQ <- coll
if len(errs) > 0 {
case errs <- err:
default:
}
+ cancel()
}
}()
// keepstore servers. This has the effect of increasing replication of
// existing blocks that are either underreplicated or poorly
// distributed according to rendezvous hashing.
-func (bal *Balancer) CommitPulls(c *arvados.Client) error {
+func (bal *Balancer) CommitPulls(ctx context.Context, c *arvados.Client) error {
defer bal.time("send_pull_lists", "wall clock time to send pull lists")()
return bal.commitAsync(c, "send pull list",
func(srv *KeepService) error {
- return srv.CommitPulls(c)
+ return srv.CommitPulls(ctx, c)
})
}
// CommitTrash sends the computed lists of trash requests to the
// keepstore servers. This has the effect of deleting blocks that are
// overreplicated or unreferenced.
-func (bal *Balancer) CommitTrash(c *arvados.Client) error {
+func (bal *Balancer) CommitTrash(ctx context.Context, c *arvados.Client) error {
defer bal.time("send_trash_lists", "wall clock time to send trash lists")()
return bal.commitAsync(c, "send trash list",
func(srv *KeepService) error {
- return srv.CommitTrash(c)
+ return srv.CommitTrash(ctx, c)
})
}
package main
import (
+ "context"
"fmt"
"time"
//
// If pageSize > 0 it is used as the maximum page size in each API
// call; otherwise the maximum allowed page size is requested.
-func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
+func EachCollection(ctx context.Context, c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
if progress == nil {
progress = func(_, _ int) {}
}
for {
progress(callCount, expectCount)
var page arvados.CollectionList
- err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
+ err := c.RequestAndDecodeContext(ctx, &page, "GET", "arvados/v1/collections", nil, params)
if err != nil {
return err
}
package main
import (
+ "context"
"sync"
"time"
longestStreak := 0
var lastMod time.Time
sawUUID := make(map[string]bool)
- err := EachCollection(s.client, pageSize, func(c arvados.Collection) error {
+ err := EachCollection(context.Background(), s.client, pageSize, func(c arvados.Collection) error {
if c.ModifiedAt.IsZero() {
return nil
}
package main
import (
+ "context"
"encoding/json"
"fmt"
"io"
// CommitPulls sends the current list of pull requests to the storage
// server (even if the list is empty).
-func (srv *KeepService) CommitPulls(c *arvados.Client) error {
- return srv.put(c, "pull", srv.ChangeSet.Pulls)
+func (srv *KeepService) CommitPulls(ctx context.Context, c *arvados.Client) error {
+ return srv.put(ctx, c, "pull", srv.ChangeSet.Pulls)
}
// CommitTrash sends the current list of trash requests to the storage
// server (even if the list is empty).
-func (srv *KeepService) CommitTrash(c *arvados.Client) error {
- return srv.put(c, "trash", srv.ChangeSet.Trashes)
+func (srv *KeepService) CommitTrash(ctx context.Context, c *arvados.Client) error {
+ return srv.put(ctx, c, "trash", srv.ChangeSet.Trashes)
}
// Perform a PUT request at path, with data (as JSON) in the request
// body.
-func (srv *KeepService) put(c *arvados.Client, path string, data interface{}) error {
+func (srv *KeepService) put(ctx context.Context, c *arvados.Client, path string, data interface{}) error {
// We'll start a goroutine to do the JSON encoding, so we can
// stream it to the http client through a Pipe, rather than
// keeping the entire encoded version in memory.
}()
url := srv.URLBase() + "/" + path
- req, err := http.NewRequest("PUT", url, ioutil.NopCloser(jsonR))
+ req, err := http.NewRequestWithContext(ctx, "PUT", url, ioutil.NopCloser(jsonR))
if err != nil {
return fmt.Errorf("building request for %s: %v", url, err)
}
}
func (h *handler) setup() {
- h.clientPool = arvadosclient.MakeClientPool()
+ // Errors will be handled at the client pool.
+ arv, _ := arvados.NewClientFromConfig(h.Config.cluster)
+ h.clientPool = arvadosclient.MakeClientPoolWith(arv)
keepclient.RefreshServiceDiscoveryOnSIGHUP()
keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
return fmt.Errorf("Error setting up arvados client %v", err)
}
+ // If a config file is available, use the keepstores defined there
+ // instead of the legacy autodiscover mechanism via the API server
+ for k := range cluster.Services.Keepstore.InternalURLs {
+ arv.KeepServiceURIs = append(arv.KeepServiceURIs, strings.TrimRight(k.String(), "/"))
+ }
+
if cluster.SystemLogs.LogLevel == "debug" {
keepclient.DebugPrintf = log.Printf
}
signal.Notify(term, syscall.SIGINT)
// Start serving requests.
- router = MakeRESTRouter(kc, time.Duration(cluster.API.KeepServiceRequestTimeout), cluster.ManagementToken)
+ router = MakeRESTRouter(kc, time.Duration(keepclient.DefaultProxyRequestTimeout), cluster.ManagementToken)
return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
}
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredConfigYmlSuite{})
+
+// Tests that require the Keep servers running as defined in config.yml
+type ServerRequiredConfigYmlSuite struct{}
+
// Gocheck boilerplate
var _ = Suite(&NoKeepServerSuite{})
arvadostest.StopAPI()
}
+func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
+ arvadostest.StartAPI()
+ // config.yml defines 4 keepstores
+ arvadostest.StartKeep(4, false)
+}
+
+func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
+ arvadostest.StopKeep(4)
+ arvadostest.StopAPI()
+}
+
func (s *NoKeepServerSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
// We need API to have some keep services listed, but the
arvadostest.StopAPI()
}
-func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
+func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, Equals, nil)
cluster, err := cfg.GetCluster("")
c.Assert(err, Equals, nil)
+ if !loadKeepstoresFromConfig {
+ // Do not load Keepstore InternalURLs from the config file
+ cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
+ }
+
cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
listener = nil
}
func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
req, err := http.NewRequest("POST",
}
func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
sr := map[string]string{
}
func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Set up fake keepstore to record request headers
}
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
content := []byte("TestDesiredReplicas")
}
func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
content := []byte("TestPutWrongContentLength")
}
func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
router.(*proxyHandler).timeout = time.Nanosecond
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
}
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
- kc := runProxy(c, true)
+ kc := runProxy(c, true, false)
defer closeListener()
hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
}
func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
{
}
func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
{
// With a valid but non-existing prefix (expect "\n")
// With an invalid prefix (expect error)
func (s *ServerRequiredSuite) TestGetIndex(c *C) {
- kc := runProxy(c, false)
+ getIndexWorker(c, false)
+}
+
+// Test GetIndex
+// Uses config.yml
+// Put one block, with 2 replicas
+// With no prefix (expect the block locator, twice)
+// With an existing prefix (expect the block locator, twice)
+// With a valid but non-existing prefix (expect "\n")
+// With an invalid prefix (expect error)
+func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
+ getIndexWorker(c, true)
+}
+
+func getIndexWorker(c *C, useConfig bool) {
+ kc := runProxy(c, false, useConfig)
defer closeListener()
// Put "index-data" blocks
}
func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash, _, err := kc.PutB([]byte("shareddata"))
c.Check(err, IsNil)
}
func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Put a test block
}
func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Point keepproxy at a non-existent keepstore
}
func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
}
func (s *ServerRequiredSuite) TestPing(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
}
}
+// Test TOUCH requests.
+func (s *HandlerSuite) TestTouchHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ vols := s.handler.volmgr.AllWritable()
+ vols[0].Put(context.Background(), TestHash, TestBlock)
+ vols[0].Volume.(*MockVolume).TouchWithDate(TestHash, time.Now().Add(-time.Hour))
+ afterPut := time.Now()
+ t, err := vols[0].Mtime(TestHash)
+ c.Assert(err, check.IsNil)
+ c.Assert(t.Before(afterPut), check.Equals, true)
+
+ ExpectStatusCode(c,
+ "touch with no credentials",
+ http.StatusUnauthorized,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ }))
+
+ ExpectStatusCode(c,
+ "touch with non-root credentials",
+ http.StatusUnauthorized,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ apiToken: arvadostest.ActiveTokenV2,
+ }))
+
+ ExpectStatusCode(c,
+ "touch non-existent block",
+ http.StatusNotFound,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ apiToken: s.cluster.SystemRootToken,
+ }))
+
+ beforeTouch := time.Now()
+ ExpectStatusCode(c,
+ "touch block",
+ http.StatusOK,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ apiToken: s.cluster.SystemRootToken,
+ }))
+ t, err = vols[0].Mtime(TestHash)
+ c.Assert(err, check.IsNil)
+ c.Assert(t.After(beforeTouch), check.Equals, true)
+}
+
// Test /index requests:
// - unauthenticated /index request
// - unauthenticated /index/prefix request
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
+ // Update timestamp on existing block. Privileged client only.
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleTOUCH).Methods("TOUCH")
// Internals/debugging info (runtime.MemStats)
rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
}
}
+func (rtr *router) handleTOUCH(resp http.ResponseWriter, req *http.Request) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
+ http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
+ return
+ }
+ hash := mux.Vars(req)["hash"]
+ vols := rtr.volmgr.AllWritable()
+ if len(vols) == 0 {
+ http.Error(resp, "no volumes", http.StatusNotFound)
+ return
+ }
+ var err error
+ for _, mnt := range vols {
+ err = mnt.Touch(hash)
+ if err == nil {
+ break
+ }
+ }
+ switch {
+ case err == nil:
+ return
+ case os.IsNotExist(err):
+ http.Error(resp, err.Error(), http.StatusNotFound)
+ default:
+ http.Error(resp, err.Error(), http.StatusInternalServerError)
+ }
+}
+
func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
ctx, cancel := contextForResponse(context.TODO(), resp)
defer cancel()
// S3Volume implements Volume using an S3 bucket.
type S3Volume struct {
- AccessKey string
- SecretKey string
- AuthToken string // populated automatically when IAMRole is used
- AuthExpiration time.Time // populated automatically when IAMRole is used
- IAMRole string
- Endpoint string
- Region string
- Bucket string
- LocationConstraint bool
- IndexPageSize int
- ConnectTimeout arvados.Duration
- ReadTimeout arvados.Duration
- RaceWindow arvados.Duration
- UnsafeDelete bool
+ arvados.S3VolumeDriverParameters
+ AuthToken string // populated automatically when IAMRole is used
+ AuthExpiration time.Time // populated automatically when IAMRole is used
cluster *arvados.Cluster
volume arvados.Volume
func (v *S3Volume) newS3Client() *s3.S3 {
auth := aws.NewAuth(v.AccessKey, v.SecretKey, v.AuthToken, v.AuthExpiration)
client := s3.New(*auth, v.region)
- if v.region.EC2Endpoint.Signer == aws.V4Signature {
- // Currently affects only eu-central-1
+ if !v.V2Signature {
client.Signature = aws.V4Signature
}
client.ConnectTimeout = time.Duration(v.ConnectTimeout)
}
}
+func (s *StubbedS3Suite) TestSignatureVersion(c *check.C) {
+ var header http.Header
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ header = r.Header
+ }))
+ defer stub.Close()
+
+ // Default V4 signature
+ vol := S3Volume{
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ AccessKey: "xxx",
+ SecretKey: "xxx",
+ Endpoint: stub.URL,
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
+ },
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+ }
+ err := vol.check()
+ c.Check(err, check.IsNil)
+ err = vol.Put(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
+ c.Check(err, check.IsNil)
+ c.Check(header.Get("Authorization"), check.Matches, `AWS4-HMAC-SHA256 .*`)
+
+ // Force V2 signature
+ vol = S3Volume{
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ AccessKey: "xxx",
+ SecretKey: "xxx",
+ Endpoint: stub.URL,
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
+ V2Signature: true,
+ },
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+ }
+ err = vol.check()
+ c.Check(err, check.IsNil)
+ err = vol.Put(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
+ c.Check(err, check.IsNil)
+ c.Check(header.Get("Authorization"), check.Matches, `AWS xxx:.*`)
+}
+
func (s *StubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upd := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
w.WriteHeader(http.StatusNotFound)
}))
deadv := &S3Volume{
- IAMRole: s.metadata.URL + "/fake-metadata/test-role",
- Endpoint: "http://localhost:12345",
- Region: "test-region-1",
- Bucket: "test-bucket-name",
- cluster: s.cluster,
- logger: ctxlog.TestLogger(c),
- metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ IAMRole: s.metadata.URL + "/fake-metadata/test-role",
+ Endpoint: "http://localhost:12345",
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
+ },
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err := deadv.check()
c.Check(err, check.ErrorMatches, `.*/fake-metadata/test-role.*`)
v := &TestableS3Volume{
S3Volume: &S3Volume{
- AccessKey: accessKey,
- SecretKey: secretKey,
- IAMRole: iamRole,
- Bucket: TestBucketName,
- Endpoint: endpoint,
- Region: "test-region-1",
- LocationConstraint: true,
- UnsafeDelete: true,
- IndexPageSize: 1000,
- cluster: cluster,
- volume: volume,
- logger: ctxlog.TestLogger(c),
- metrics: metrics,
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ IAMRole: iamRole,
+ AccessKey: accessKey,
+ SecretKey: secretKey,
+ Bucket: TestBucketName,
+ Endpoint: endpoint,
+ Region: "test-region-1",
+ LocationConstraint: true,
+ UnsafeDelete: true,
+ IndexPageSize: 1000,
+ },
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(c),
+ metrics: metrics,
},
c: c,
server: srv,
return e
}
defer v.unlockfile(f)
- ts := syscall.NsecToTimespec(time.Now().UnixNano())
+ ts := time.Now()
v.os.stats.TickOps("utimes")
v.os.stats.Tick(&v.os.stats.UtimesOps)
- err = syscall.UtimesNano(p, []syscall.Timespec{ts, ts})
+ err = os.Chtimes(p, ts, ts)
v.os.stats.TickErr(err)
return err
}
v.os.Remove(tmpfile.Name())
return err
}
+ // ext4 uses a low-precision clock and effectively backdates
+ // files by up to 10 ms, sometimes across a 1-second boundary,
+ // which produces confusing results in logs and tests. We
+ // avoid this by setting the output file's timestamps
+ // explicitly, using a higher resolution clock.
+ ts := time.Now()
+ v.os.stats.TickOps("utimes")
+ v.os.stats.Tick(&v.os.stats.UtimesOps)
+ if err = os.Chtimes(tmpfile.Name(), ts, ts); err != nil {
+ err = fmt.Errorf("error setting timestamps on %s: %s", tmpfile.Name(), err)
+ v.os.Remove(tmpfile.Name())
+ return err
+ }
if err := v.os.Rename(tmpfile.Name(), bpath); err != nil {
err = fmt.Errorf("error renaming %s to %s: %s", tmpfile.Name(), bpath, err)
v.os.Remove(tmpfile.Name())
err := filepath.Walk(v.Root, func(path string, info os.FileInfo, err error) error {
if err != nil {
v.logger.WithError(err).Errorf("EmptyTrash: filepath.Walk(%q) failed", path)
+ // Don't give up -- keep walking other
+ // files/dirs
+ return nil
+ } else if !info.Mode().IsDir() {
+ todo <- dirent{path, info}
return nil
+ } else if path == v.Root || blockDirRe.MatchString(info.Name()) {
+ // Descend into a directory that we might have
+ // put trash in.
+ return nil
+ } else {
+ // Don't descend into other dirs.
+ return filepath.SkipDir
}
- todo <- dirent{path, info}
- return nil
})
close(todo)
wg.Wait()
c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
- c.Check(stats(), check.Matches, `.*"UtimesOps":0,.*`)
+ c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
err = vol.Touch(loc)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":1,.*`)
- c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
+ c.Check(stats(), check.Matches, `.*"UtimesOps":2,.*`)
_, err = vol.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":2,.*`)
}
+
+func (s *UnixVolumeSuite) TestSkipUnusedDirs(c *check.C) {
+ vol := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
+
+ err := os.Mkdir(vol.UnixVolume.Root+"/aaa", 0777)
+ c.Assert(err, check.IsNil)
+ err = os.Mkdir(vol.UnixVolume.Root+"/.aaa", 0777) // EmptyTrash should not look here
+ c.Assert(err, check.IsNil)
+ deleteme := vol.UnixVolume.Root + "/aaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.trash.1"
+ err = ioutil.WriteFile(deleteme, []byte{1, 2, 3}, 0777)
+ c.Assert(err, check.IsNil)
+ skipme := vol.UnixVolume.Root + "/.aaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.trash.1"
+ err = ioutil.WriteFile(skipme, []byte{1, 2, 3}, 0777)
+ c.Assert(err, check.IsNil)
+ vol.EmptyTrash()
+
+ _, err = os.Stat(skipme)
+ c.Check(err, check.IsNil)
+
+ _, err = os.Stat(deleteme)
+ c.Check(err, check.NotNil)
+ c.Check(os.IsNotExist(err), check.Equals, true)
+}
}
func (v *MockVolume) Touch(loc string) error {
+ return v.TouchWithDate(loc, time.Now())
+}
+
+func (v *MockVolume) TouchWithDate(loc string, t time.Time) error {
v.gotCall("Touch")
<-v.Gate
if v.volume.ReadOnly {
return MethodDisabledError
}
+ if _, exists := v.Store[loc]; !exists {
+ return os.ErrNotExist
+ }
if v.Touchable {
- v.Timestamps[loc] = time.Now()
+ v.Timestamps[loc] = t
return nil
}
return errors.New("Touch failed")
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.PostgreSQL.Connection, check.DeepEquals, arvados.PostgreSQLConnection{
mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF" "$RLIBS"
if ! test -d "$ARVADOS_ROOT" ; then
- git clone https://github.com/arvados/arvados.git "$ARVADOS_ROOT"
+ git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
fi
if ! test -d "$SSO_ROOT" ; then
git clone https://github.com/arvados/sso-devise-omniauth-provider.git "$SSO_ROOT"
sv stop keepstore1
sv stop keepproxy
cd /usr/src/arvados/services/api
+export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
export RAILS_ENV=development
bundle exec rake db:drop
rm /var/lib/arvados/api_database_setup
RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/crunch-dispatch-local/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
DefaultReplication: 1
TrustAllContent: true
Login:
- ProviderAppSecret: $sso_app_secret
- ProviderAppID: arvados-server
+ SSO:
+ Enable: true
+ ProviderAppSecret: $sso_app_secret
+ ProviderAppID: arvados-server
Users:
NewUsersAreActive: true
AutoAdminFirstUser: true
popd
if [ "$PYCMD" = "python3" ]; then
- if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
+ if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
pip3 install --prefix /usr/local $1
- fi
+ fi
else
- if ! pip install --no-index --find-links /var/lib/pip $1 ; then
+ if ! pip install --no-index --find-links /var/lib/pip $1 ; then
pip install $1
- fi
+ fi
fi
}
/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
-exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-controller
+exec /usr/local/bin/arvados-controller
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+(cd /usr/local/bin && ln -sf arvados-server arvados-ws)
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-(cd /usr/local/bin && ln -sf arvados-server arvados-ws)
-
-if test "$1" = "--only-deps" ; then
- exit
-fi
-
-/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
-
-exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws
"log"
"net/http"
"os"
+ "os/signal"
+ "syscall"
"time"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
ServiceURL = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
ServiceUUID = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
getVersion = flag.Bool("version", false, "Print version information and exit.")
+ RunTime = flag.Duration("run-time", 0, "time to run (e.g. 60s), or 0 to run indefinitely (default)")
)
func main() {
os.Exit(0)
}
- log.Printf("keep-exercise %s started", version)
+ stderr := log.New(os.Stderr, "", log.LstdFlags)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc.Want_replicas = *Replicas
Transport: &transport,
}
- overrideServices(kc)
+ overrideServices(kc, stderr)
nextLocator := make(chan string, *ReadThreads+*WriteThreads)
- go countBeans(nextLocator)
+ go countBeans(nextLocator, stderr)
for i := 0; i < *WriteThreads; i++ {
nextBuf := make(chan []byte, 1)
- go makeBufs(nextBuf, i)
- go doWrites(kc, nextBuf, nextLocator)
+ go makeBufs(nextBuf, i, stderr)
+ go doWrites(kc, nextBuf, nextLocator, stderr)
}
for i := 0; i < *ReadThreads; i++ {
- go doReads(kc, nextLocator)
+ go doReads(kc, nextLocator, stderr)
}
<-make(chan struct{})
}
// Send struct{}{} to errorsChan when an error happens.
var errorsChan = make(chan struct{})
-func countBeans(nextLocator chan string) {
+func countBeans(nextLocator chan string, stderr *log.Logger) {
t0 := time.Now()
var tickChan <-chan time.Time
+ var endChan <-chan time.Time
+ c := make(chan os.Signal)
+ signal.Notify(c, os.Interrupt, syscall.SIGTERM)
if *StatsInterval > 0 {
tickChan = time.NewTicker(*StatsInterval).C
}
+ if *RunTime > 0 {
+ endChan = time.NewTicker(*RunTime).C
+ }
var bytesIn uint64
var bytesOut uint64
var errors uint64
+ var rateIn, rateOut float64
+ var maxRateIn, maxRateOut float64
+ var abort, printCsv bool
+ csv := log.New(os.Stdout, "", 0)
+ csv.Println("Timestamp,Elapsed,Read (bytes),Avg Read Speed (MiB/s),Peak Read Speed (MiB/s),Written (bytes),Avg Write Speed (MiB/s),Peak Write Speed (MiB/s),Errors,ReadThreads,WriteThreads,VaryRequest,VaryThread,BlockSize,Replicas,StatsInterval,ServiceURL,ServiceUUID,RunTime")
for {
select {
case <-tickChan:
- elapsed := time.Since(t0)
- log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
- elapsed,
- bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
- bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
- errors,
- )
+ printCsv = true
+ case <-endChan:
+ printCsv = true
+ abort = true
+ case <-c:
+ printCsv = true
+ abort = true
+ fmt.Print("\r") // Suppress the ^C print
case i := <-bytesInChan:
bytesIn += i
case o := <-bytesOutChan:
case <-errorsChan:
errors++
}
+ if printCsv {
+ elapsed := time.Since(t0)
+ rateIn = float64(bytesIn) / elapsed.Seconds() / 1048576
+ if rateIn > maxRateIn {
+ maxRateIn = rateIn
+ }
+ rateOut = float64(bytesOut) / elapsed.Seconds() / 1048576
+ if rateOut > maxRateOut {
+ maxRateOut = rateOut
+ }
+ csv.Printf("%v,%v,%v,%.1f,%.1f,%v,%.1f,%.1f,%d,%d,%d,%t,%t,%d,%d,%s,%s,%s,%s",
+ time.Now().Format("2006-01-02 15:04:05"),
+ elapsed,
+ bytesIn, rateIn, maxRateIn,
+ bytesOut, rateOut, maxRateOut,
+ errors,
+ *ReadThreads,
+ *WriteThreads,
+ *VaryRequest,
+ *VaryThread,
+ *BlockSize,
+ *Replicas,
+ *StatsInterval,
+ *ServiceURL,
+ *ServiceUUID,
+ *RunTime,
+ )
+ printCsv = false
+ }
+ if abort {
+ os.Exit(0)
+ }
}
}
-func makeBufs(nextBuf chan<- []byte, threadID int) {
+func makeBufs(nextBuf chan<- []byte, threadID int, stderr *log.Logger) {
buf := make([]byte, *BlockSize)
if *VaryThread {
binary.PutVarint(buf, int64(threadID))
if *VaryRequest {
rnd := make([]byte, randSize)
if _, err := io.ReadFull(rand.Reader, rnd); err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
buf = append(rnd, buf[randSize:]...)
}
}
}
-func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string) {
+func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string, stderr *log.Logger) {
for buf := range nextBuf {
locator, _, err := kc.PutB(buf)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
}
}
-func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
+func doReads(kc *keepclient.KeepClient, nextLocator <-chan string, stderr *log.Logger) {
for locator := range nextLocator {
rdr, size, url, err := kc.Get(locator)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
n, err := io.Copy(ioutil.Discard, rdr)
rdr.Close()
if n != size || err != nil {
- log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
+ stderr.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
errorsChan <- struct{}{}
continue
// Note we don't count the bytes received in
}
}
-func overrideServices(kc *keepclient.KeepClient) {
+func overrideServices(kc *keepclient.KeepClient, stderr *log.Logger) {
roots := make(map[string]string)
if *ServiceURL != "" {
roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
}
}
if len(roots) == 0 {
- log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
+ stderr.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
}
} else {
return
GetItems() []interface{}
}
-// GroupInfo tracks previous and current members of a particular Group
+// GroupPermissions maps permission levels on groups (can_read, can_write, can_manage)
+type GroupPermissions map[string]bool
+
+// GroupInfo tracks previous and current member's permissions on a particular Group
type GroupInfo struct {
Group arvados.Group
- PreviousMembers map[string]bool
- CurrentMembers map[string]bool
+ PreviousMembers map[string]GroupPermissions
+ CurrentMembers map[string]GroupPermissions
}
// GetUserID returns the correct user id value depending on the selector
// Set up usage message
flags.Usage = func() {
- usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns:
- * 1st column: Group name
- * 2nd column: User identifier`
+ usageStr := `Synchronize remote groups into Arvados from a CSV format file with 3 columns:
+ * 1st: Group name
+ * 2nd: User identifier
+ * 3rd (Optional): User permission on the group: can_read, can_write or can_manage. (Default: can_write)`
fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Options:\n")
log.Println("Default parent group not found, creating...")
}
groupData := map[string]string{
- "name": cfg.ParentGroupName,
- "owner_uuid": cfg.SysUserUUID,
+ "name": cfg.ParentGroupName,
+ "owner_uuid": cfg.SysUserUUID,
+ "group_class": "role",
}
if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
// Remove previous members not listed on this run
for groupUUID := range remoteGroups {
gi := remoteGroups[groupUUID]
- evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
+ evictedMemberPerms := subtract(gi.PreviousMembers, gi.CurrentMembers)
groupName := gi.Group.Name
- if len(evictedMembers) > 0 {
- log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
- }
- for evictedUser := range evictedMembers {
- if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
+ if len(evictedMemberPerms) > 0 {
+ log.Printf("Removing permissions from %d users on group %q", len(evictedMemberPerms), groupName)
+ }
+ for member := range evictedMemberPerms {
+ var perms []string
+ completeMembershipRemoval := false
+ if _, ok := gi.CurrentMembers[member]; !ok {
+ completeMembershipRemoval = true
+ membershipsRemoved++
+ } else {
+ // Collect which user->group permission links should be removed
+ for p := range evictedMemberPerms[member] {
+ if evictedMemberPerms[member][p] {
+ perms = append(perms, p)
+ }
+ }
+ membershipsRemoved += len(perms)
+ }
+ if err := RemoveMemberLinksFromGroup(cfg, allUsers[userIDToUUID[member]],
+ perms, completeMembershipRemoval, gi.Group); err != nil {
return err
}
- membershipsRemoved++
}
}
log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
) (groupsCreated, membersAdded, membersSkipped int, err error) {
lineNo := 0
csvReader := csv.NewReader(f)
- csvReader.FieldsPerRecord = 2
+ // Allow variable number of fields.
+ csvReader.FieldsPerRecord = -1
for {
record, e := csvReader.Read()
if e == io.EOF {
err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
return
}
+ // Only allow 2 or 3 fields per record for backwards compatibility.
+ if len(record) < 2 || len(record) > 3 {
+ err = fmt.Errorf("error parsing %q, line %d: found %d fields but only 2 or 3 are allowed", cfg.Path, lineNo, len(record))
+ return
+ }
groupName := strings.TrimSpace(record[0])
groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
- if groupName == "" || groupMember == "" {
- log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
+ groupPermission := "can_write"
+ if len(record) == 3 {
+ groupPermission = strings.ToLower(record[2])
+ }
+ if groupName == "" || groupMember == "" || groupPermission == "" {
+ log.Printf("Warning: CSV record has at least one empty field (%s, %s, %s). Skipping", groupName, groupMember, groupPermission)
+ membersSkipped++
+ continue
+ }
+ if !(groupPermission == "can_read" || groupPermission == "can_write" || groupPermission == "can_manage") {
+ log.Printf("Warning: 3rd field should be 'can_read', 'can_write' or 'can_manage'. Found: %q at line %d, skipping.", groupPermission, lineNo)
membersSkipped++
continue
}
groupNameToUUID[groupName] = newGroup.UUID
remoteGroups[newGroup.UUID] = &GroupInfo{
Group: newGroup,
- PreviousMembers: make(map[string]bool), // Empty set
- CurrentMembers: make(map[string]bool), // Empty set
+ PreviousMembers: make(map[string]GroupPermissions),
+ CurrentMembers: make(map[string]GroupPermissions),
}
groupsCreated++
}
// Both group & user exist, check if user is a member
groupUUID := groupNameToUUID[groupName]
gi := remoteGroups[groupUUID]
- if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] {
+ if !gi.PreviousMembers[groupMember][groupPermission] && !gi.CurrentMembers[groupMember][groupPermission] {
if cfg.Verbose {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
- // User wasn't a member, but should be.
- if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil {
+ // User permissionwasn't there, but should be. Avoid duplicating the
+ // group->user link when necessary.
+ createG2ULink := true
+ if _, ok := gi.PreviousMembers[groupMember]; ok {
+ createG2ULink = false // User is already member of the group
+ }
+ if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group, groupPermission, createG2ULink); e != nil {
err = e
return
}
membersAdded++
}
- gi.CurrentMembers[groupMember] = true
+ if _, ok := gi.CurrentMembers[groupMember]; ok {
+ gi.CurrentMembers[groupMember][groupPermission] = true
+ } else {
+ gi.CurrentMembers[groupMember] = GroupPermissions{groupPermission: true}
+ }
+
}
return
}
return allItems, nil
}
-func subtract(setA map[string]bool, setB map[string]bool) map[string]bool {
- result := make(map[string]bool)
+func subtract(setA map[string]GroupPermissions, setB map[string]GroupPermissions) map[string]GroupPermissions {
+ result := make(map[string]GroupPermissions)
for element := range setA {
- if !setB[element] {
- result[element] = true
+ if _, ok := setB[element]; !ok {
+ result[element] = setA[element]
+ } else {
+ for perm := range setA[element] {
+ if _, ok := setB[element][perm]; !ok {
+ result[element] = GroupPermissions{perm: true}
+ }
+ }
}
}
return result
params := arvados.ResourceListParams{
Filters: []arvados.Filter{{
- Attr: "owner_uuid",
+ Attr: "tail_uuid",
Operator: "=",
Operand: cfg.ParentGroupUUID,
}},
}
- results, err := GetAll(cfg.Client, "groups", params, &GroupList{})
+ results, err := GetAll(cfg.Client, "links", params, &LinkList{})
if err != nil {
return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err)
}
for _, item := range results {
- group := item.(arvados.Group)
+ var group arvados.Group
+ err = GetGroup(cfg, &group, item.(arvados.Link).HeadUUID)
+ if err != nil {
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote group: %s", err)
+ }
// Group -> User filter
g2uFilter := arvados.ResourceListParams{
Filters: []arvados.Filter{{
Operand: "permission",
}, {
Attr: "name",
- Operator: "=",
- Operand: "can_write",
+ Operator: "in",
+ Operand: []string{"can_read", "can_write", "can_manage"},
}, {
Attr: "head_uuid",
Operator: "=",
}
g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
if err != nil {
- return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err)
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting group->user 'can_read' links for group %q: %s", group.Name, err)
}
u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
if err != nil {
- return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err)
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting user->group links for group %q: %s", group.Name, err)
}
- // Build a list of user ids (email or username) belonging to this group
- membersSet := make(map[string]bool)
- u2gLinkSet := make(map[string]bool)
+ // Build a list of user ids (email or username) belonging to this group.
+ membersSet := make(map[string]GroupPermissions)
+ u2gLinkSet := make(map[string]GroupPermissions)
for _, l := range u2gLinks {
- linkedMemberUUID := l.(arvados.Link).TailUUID
- u2gLinkSet[linkedMemberUUID] = true
+ link := l.(arvados.Link)
+ // Also save the member's group access level.
+ if _, ok := u2gLinkSet[link.TailUUID]; ok {
+ u2gLinkSet[link.TailUUID][link.Name] = true
+ } else {
+ u2gLinkSet[link.TailUUID] = GroupPermissions{link.Name: true}
+ }
}
for _, item := range g2uLinks {
link := item.(arvados.Link)
if err != nil {
return remoteGroups, groupNameToUUID, err
}
- membersSet[memberID] = true
+ membersSet[memberID] = u2gLinkSet[link.HeadUUID]
}
remoteGroups[group.UUID] = &GroupInfo{
Group: group,
PreviousMembers: membersSet,
- CurrentMembers: make(map[string]bool), // Empty set
+ CurrentMembers: make(map[string]GroupPermissions),
}
groupNameToUUID[group.Name] = group.UUID
}
return remoteGroups, groupNameToUUID, nil
}
-// RemoveMemberFromGroup remove all links related to the membership
-func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+// RemoveMemberLinksFromGroup remove all links related to the membership
+func RemoveMemberLinksFromGroup(cfg *ConfigParams, user arvados.User, linkNames []string, completeRemoval bool, group arvados.Group) error {
if cfg.Verbose {
log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID)
}
var links []interface{}
- // Search for all group<->user links (both ways)
- for _, filterset := range [][]arvados.Filter{
- // Group -> User
- {{
- Attr: "link_class",
- Operator: "=",
- Operand: "permission",
- }, {
- Attr: "tail_uuid",
- Operator: "=",
- Operand: group.UUID,
- }, {
- Attr: "head_uuid",
- Operator: "=",
- Operand: user.UUID,
- }},
- // Group <- User
- {{
- Attr: "link_class",
- Operator: "=",
- Operand: "permission",
- }, {
- Attr: "tail_uuid",
- Operator: "=",
- Operand: user.UUID,
- }, {
- Attr: "head_uuid",
- Operator: "=",
- Operand: group.UUID,
- }},
- } {
+ var filters [][]arvados.Filter
+ if completeRemoval {
+ // Search for all group<->user links (both ways)
+ filters = [][]arvados.Filter{
+ // Group -> User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }},
+ // Group <- User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }},
+ }
+ } else {
+ // Search only for the requested Group <- User permission links
+ filters = [][]arvados.Filter{
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "name",
+ Operator: "in",
+ Operand: linkNames,
+ }},
+ }
+ }
+
+ for _, filterset := range filters {
l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
if err != nil {
userID, _ := GetUserID(user, cfg.UserID)
}
// AddMemberToGroup create membership links
-func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group, perm string, createG2ULink bool) error {
var newLink arvados.Link
- linkData := map[string]string{
- "owner_uuid": cfg.SysUserUUID,
- "link_class": "permission",
- "name": "can_read",
- "tail_uuid": group.UUID,
- "head_uuid": user.UUID,
- }
- if err := CreateLink(cfg, &newLink, linkData); err != nil {
- userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+ var linkData map[string]string
+ if createG2ULink {
+ linkData = map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "can_read",
+ "tail_uuid": group.UUID,
+ "head_uuid": user.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+ }
}
linkData = map[string]string{
"owner_uuid": cfg.SysUserUUID,
"link_class": "permission",
- "name": "can_write",
+ "name": perm,
"tail_uuid": user.UUID,
"head_uuid": group.UUID,
}
if err := CreateLink(cfg, &newLink, linkData); err != nil {
userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err)
+ return fmt.Errorf("error adding user %q -> group %q %s permission: %s", userID, group.Name, perm, err)
}
return nil
}
}
// GroupMembershipExists checks that both needed links exist between user and group
-func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string) bool {
+func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string, perm string) bool {
ll := LinkList{}
// Check Group -> User can_read permission
params := arvados.ResourceListParams{
}, {
Attr: "name",
Operator: "=",
- Operand: "can_write",
+ Operand: perm,
}, {
Attr: "tail_uuid",
Operator: "=",
}, {
Attr: "owner_uuid",
Operator: "=",
- Operand: cfg.ParentGroupUUID,
+ Operand: cfg.SysUserUUID,
}, {
Attr: "group_class",
Operator: "=",
groupUUID, err := RemoteGroupExists(s.cfg, groupName)
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
}
+// Error out when records have <2 or >3 records
+func (s *TestSuite) TestWrongNumberOfFields(c *C) {
+ for _, testCase := range [][][]string{
+ {{"field1"}},
+ {{"field1", "field2", "field3", "field4"}},
+ {{"field1", "field2", "field3", "field4", "field5"}},
+ } {
+ tmpfile, err := MakeTempCSVFile(testCase)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, Not(IsNil))
+ }
+}
+
+// Check different membership permissions
+func (s *TestSuite) TestMembershipLevels(c *C) {
+ userEmail := s.users[arvadostest.ActiveUserUUID].Email
+ userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ data := [][]string{
+ {"TestGroup1", userEmail, "can_read"},
+ {"TestGroup2", userEmail, "can_write"},
+ {"TestGroup3", userEmail, "can_manage"},
+ {"TestGroup4", userEmail, "invalid_permission"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ for _, record := range data {
+ groupName := record[0]
+ permLevel := record[2]
+ if permLevel != "invalid_permission" {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, permLevel), Equals, true)
+ } else {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ }
+ }
+}
+
+// Check membership level change
+func (s *TestSuite) TestMembershipLevelUpdate(c *C) {
+ userEmail := s.users[arvadostest.ActiveUserUUID].Email
+ userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ groupName := "TestGroup1"
+ // Give read permissions
+ tmpfile, err := MakeTempCSVFile([][]string{{groupName, userEmail, "can_read"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+ // Give write permissions
+ tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_write"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+ // Give manage permissions
+ tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_manage"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, true)
+}
+
// The absence of a user membership on the CSV file implies its removal
func (s *TestSuite) TestMembershipRemoval(c *C) {
localUserEmail := s.users[arvadostest.ActiveUserUUID].Email
groupUUID, err := RemoteGroupExists(s.cfg, groupName)
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
}
// New CSV with some previous membership missing
data = [][]string{
groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, false)
// Confirm TestGroup1 memberships
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup2")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, false)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
}
// If a group doesn't exist on the system, create it before adding users
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
// active user should be a member
- c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID, "can_write"), Equals, true)
}
// Users listed on the file that don't exist on the system are ignored
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
// Users listed on the file that don't exist on the system are ignored
activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
// Confirm that group doesn't exist
- groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
- c.Assert(err, IsNil)
- c.Assert(groupUUID, Equals, "")
+ for _, groupName := range []string{"TestGroup4", "TestGroup5"} {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ }
// Create file & run command
data := [][]string{
- {"", activeUserEmail}, // Empty field
- {"TestGroup5", ""}, // Empty field
+ {"", activeUserEmail}, // Empty field
+ {"TestGroup5", ""}, // Empty field
+ {"TestGroup5", activeUserEmail, ""}, // Empty 3rd field: is optional but cannot be empty
{"TestGroup4", activeUserEmail},
}
tmpfile, err := MakeTempCSVFile(data)
s.cfg.Path = tmpfile.Name()
err = doMain(s.cfg)
c.Assert(err, IsNil)
- // Confirm that memberships exist
+ // Confirm that records about TestGroup5 were skipped
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup5")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Confirm that membership exists
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
// Instead of emails, use username as identifier
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup1")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
projects / arvados.git / commitdiff