attrs['state'] = "Uncommitted"
# required
- attrs['command'] = ["arvados-cwl-runner", "--local", "--api=containers", "/var/lib/cwl/workflow.json#main", "/var/lib/cwl/cwl.input.json"]
+ attrs['command'] = ["arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--project-uuid=#{params['work_unit']['owner_uuid']}",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"]
attrs['container_image'] = "arvados/jobs"
attrs['cwd'] = "/var/spool/cwl"
attrs['output_path'] = "/var/spool/cwl"
link
end
end
- assert_equal(['foo'], hrefs.compact.sort,
+ assert_equal(['./foo'], hrefs.compact.sort,
"download page did provide strictly file links")
click_link "foo"
assert_text "foo\nfile\n"
# SPDX-License-Identifier: AGPL-3.0
#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
-debian8,debian9,ubuntu1204,centos7|python-gflags|2.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.6.2|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|apache-libcloud|2.3.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|rsa|3.4.2|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|uritemplate|3.0.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|httplib2|0.9.2|3|python|all
-debian8,debian9,ubuntu1204,centos7|ws4py|0.3.5|2|python|all
-debian8,debian9,ubuntu1204,centos7|pykka|1.2.1|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|six|1.10.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|ciso8601|1.0.3|3|python|amd64
-debian8,debian9,ubuntu1204,centos7|pycrypto|2.6.1|3|python|amd64
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604|backports.ssl_match_hostname|3.5.0.1|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|llfuse|1.2|3|python|amd64
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|pyyaml|3.12|2|python|amd64
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.2|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|mistune|0.7.3|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|typing|3.5.3.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|avro|1.8.1|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|cachecontrol|0.11.7|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|pathlib2|2.1.0|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|docker-py|1.7.2|2|python3|all
-debian8,debian9,ubuntu1204,centos7|six|1.10.0|2|python3|all
-debian8,debian9,ubuntu1204,ubuntu1404,centos7|requests|2.12.4|2|python3|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|websocket-client|0.37.0|2|python3|all
-ubuntu1204,ubuntu1404|requests|2.4.3|2|python|all
-ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
-ubuntu1204,centos7|isodate|0.5.4|2|python|all
+debian8,debian9,centos7|python-gflags|2.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.6.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|apache-libcloud|2.3.0|3|python|all|--depends 'python-requests >= 2.4.3'
+debian8,debian9,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
+debian8,debian9,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
+debian8,debian9,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|rsa|3.4.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|uritemplate|3.0.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|httplib2|0.9.2|3|python|all
+debian8,debian9,centos7|ws4py|0.3.5|2|python|all
+debian8,debian9,centos7|pykka|1.2.1|2|python|all
+debian8,debian9,ubuntu1404,centos7|six|1.10.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|ciso8601|1.0.6|3|python|amd64
+debian8,debian9,centos7|pycrypto|2.6.1|3|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604|backports.ssl_match_hostname|3.5.0.1|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|llfuse|1.2|3|python|amd64
+debian8,debian9,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|pyyaml|3.12|2|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.2|2|python|all
+debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|mistune|0.7.3|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|typing|3.5.3.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|avro|1.8.1|2|python|all
+debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|cachecontrol|0.11.7|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|pathlib2|2.3.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|scandir|1.7|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|docker-py|1.7.2|2|python3|all
+debian8,debian9,centos7|six|1.10.0|2|python3|all
+debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|websocket-client|0.37.0|2|python3|all
+ubuntu1404|requests|2.4.3|2|python|all
+centos7|contextlib2|0.5.4|2|python|all
+centos7|isodate|0.5.4|2|python|all
centos7|python-daemon|2.1.2|1|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
-debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
-debian8,ubuntu1404,centos7|subprocess32|3.2.7|2|python|all
-all|ruamel.yaml|0.13.7|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|subprocess32|3.5.0rc1|2|python|all
+all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
all|cwltest|1.0.20180416154033|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32'
all|junit-xml|1.8|3|python|all
all|rdflib-jsonld|0.4.0|2|python|all
# SPDX-License-Identifier: AGPL-3.0
LIBCLOUD_PIN=2.3.0
+
+using_fork=false
+if [[ $using_fork = true ]]; then
+ LIBCLOUD_PIN_SRC="https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip"
+else
+ LIBCLOUD_PIN_SRC=""
+fi
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
cp -rlt ubuntu1604/generated common-generated/*
-GOTARBALL=go1.8.3.linux-amd64.tar.gz
+GOTARBALL=go1.10.1.linux-amd64.tar.gz
NODETARBALL=node-v6.11.2-linux-x64.tar.xz
common-generated-all: common-generated/$(GOTARBALL) common-generated/$(NODETARBALL)
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.8.3.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.8.3.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.8.3.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.8.3.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
/usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
# Install golang binary
-ADD generated/go1.8.3.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ENV DEBIAN_FRONTEND noninteractive
-# Install RVM
+# Install dependencies and RVM
RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates && \
+ apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates && \
gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
+++ /dev/null
-deb-common-test-packages.sh
\ No newline at end of file
cd docker/jobs
docker build $NOCACHE \
--build-arg python_sdk_version=${python_sdk_version}-2 \
- --build-arg cwl_runner_version=${cwl_runner_version}-3 \
+ --build-arg cwl_runner_version=${cwl_runner_version}-4 \
-t arvados/jobs:$cwl_runner_version .
ECODE=$?
fi
echo
echo "START: $p test on $IMAGE" >&2
- if docker run --rm \
+ # ulimit option can be removed when debian8 and ubuntu1404 are retired
+ if docker run --ulimit nofile=4096:4096 \
+ --rm \
"${docker_volume_args[@]}" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "TARGET=$TARGET" \
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
- # Build packages
- if docker run --rm \
+ # Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired
+ if docker run --ulimit nofile=4096:4096 \
+ --rm \
"${docker_volume_args[@]}" \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
debian9)
FORMAT=deb
;;
- ubuntu1204)
- FORMAT=deb
- ;;
ubuntu1404)
FORMAT=deb
;;
fi
)
-# On older platforms we need to publish a backport of libfuse >=2.9.2,
-# and we need to build and install it here in order to even build an
-# llfuse package.
-cd $WORKSPACE/packages/$TARGET
-if [[ $TARGET =~ ubuntu1204 ]]; then
- # port libfuse 2.9.2 to Ubuntu 12.04
- LIBFUSE_DIR=$(mktemp -d)
- (
- cd $LIBFUSE_DIR
- # download fuse 2.9.2 ubuntu 14.04 source package
- file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
- file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
- file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
-
- # install dpkg-source and dpkg-buildpackage commands
- apt-get install -y --no-install-recommends dpkg-dev
-
- # extract source and apply patches
- dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
- rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
-
- # add new version to changelog
- cd fuse-2.9.2
- (
- echo "fuse (2.9.2-5) precise; urgency=low"
- echo
- echo " * Backported from trusty-security to precise"
- echo
- echo " -- Joshua Randall <jcrandall@alum.mit.edu> Thu, 4 Feb 2016 11:31:00 -0000"
- echo
- cat debian/changelog
- ) > debian/changelog.new
- mv debian/changelog.new debian/changelog
-
- # install build-deps and build
- apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
- dpkg-buildpackage -rfakeroot -b
- )
- fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
- dpkg -i \
- "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
- "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
- "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
- apt-get -y --no-install-recommends -f install
- rm -rf $LIBFUSE_DIR
-fi
-
# Go binaries
cd $WORKSPACE/packages/$TARGET
export GOPATH=$(mktemp -d)
arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)}
declare -a iterargs=()
if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
- arvados_cwl_runner_iteration=3
+ arvados_cwl_runner_iteration=4
iterargs+=(--iteration $arvados_cwl_runner_iteration)
else
arvados_cwl_runner_iteration=
fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0rc1" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
echo -n 'go: '
go version \
|| fatal "No go binary. See http://golang.org/doc/install"
- [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 8 ]] \
- || fatal "Go >= 1.8 required. See http://golang.org/doc/install"
+ [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
+ || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
echo -n 'gcc: '
gcc --version | egrep ^gcc \
|| fatal "No gcc. Try: apt-get install build-essential"
local venvdest="$1"; shift
if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+ elif [[ -n "$short" ]]; then
+ return
fi
if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then
# pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy
export R_LIBS
export GOPATH
-mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-for d in \
- "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
- "$GOPATH/src/git.curoverse.com/arvados.git"; do
- [[ -d "$d" ]] && rmdir "$d"
- [[ -h "$d" ]] && rm "$d"
-done
-ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
- || fatal "symlink failed"
-go get -v github.com/kardianos/govendor \
- || fatal "govendor install failed"
-cd "$GOPATH/src/git.curoverse.com/arvados.git" \
- || fatal
-# Remove cached source dirs in workdir. Otherwise, they won't qualify
-# as +missing or +external below, and we won't be able to detect that
-# they're missing from vendor/vendor.json.
-rm -r vendor/*/
-go get -v -d ...
-"$GOPATH/bin/govendor" sync \
- || fatal "govendor sync failed"
-[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
- || fatal "vendor/vendor.json has unused or missing dependencies -- try:
-* govendor remove +unused
-* govendor add +missing +external
-"
-cd "$WORKSPACE"
-
+(
+ set -e
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ rmdir -v --parents --ignore-fail-on-non-empty "${temp}/GOPATH"
+ for d in \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+ "$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -d "$d" ]] && rmdir "$d"
+ [[ -h "$d" ]] && rm "$d"
+ done
+ ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+ go get -v github.com/kardianos/govendor
+ cd "$GOPATH/src/git.curoverse.com/arvados.git"
+ if [[ -n "$short" ]]; then
+ go get -v -d ...
+ "$GOPATH/bin/govendor" sync
+ else
+ # Remove cached source dirs in workdir. Otherwise, they will
+ # not qualify as +missing or +external below, and we won't be
+ # able to detect that they're missing from vendor/vendor.json.
+ rm -rf vendor/*/
+ go get -v -d ...
+ "$GOPATH/bin/govendor" sync
+ [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+ || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+
+(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
+
+";
+ fi
+) || fatal "Go setup failed"
setup_virtualenv "$VENVDIR" --python python2.7
. "$VENVDIR/bin/activate"
# Needed for run_test_server.py which is used by certain (non-Python) tests.
-pip freeze 2>/dev/null | egrep ^PyYAML= \
- || pip install --no-cache-dir PyYAML >/dev/null \
+pip install --no-cache-dir PyYAML \
|| fatal "pip install PyYAML failed"
-# Preinstall libcloud, because nodemanager "pip install"
-# won't pick it up by default.
-pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
- || pip install --pre --ignore-installed --no-cache-dir apache-libcloud>=$LIBCLOUD_PIN >/dev/null \
- || fatal "pip install apache-libcloud failed"
-
-# We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks.
-pip freeze | grep -x llfuse==1.2.0 || (
- set -e
- yes | pip uninstall llfuse || true
- cython --version || fatal "no cython; try sudo apt-get install cython"
- cd "$temp"
- (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse)
- cd python-llfuse
- git checkout 620722fd990ea642ddb8e7412676af482c090c0c
- git checkout setup.py
- sed -i -e "s:'1\\.2':'1.2.0':" setup.py
- python setup.py build_cython
- python setup.py install --force
-) || fatal "llfuse fork failed"
-pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???"
+# Preinstall libcloud if using a fork; otherwise nodemanager "pip
+# install" won't pick it up by default.
+if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then
+ pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
+ || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \
+ || fatal "pip install apache-libcloud failed"
+fi
# Deactivate Python 2 virtualenv
deactivate
# is a postgresql superuser.
cd "$WORKSPACE/services/api" \
&& test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
- && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
mkdir -p "$WORKSPACE/services/api/tmp/pids"
title: "R SDK Overview"
...
-#{rd.read.gsub("```", "\n~~~\n").gsub(/^[ ]+/, "")}
+#{rd.read.gsub(/^```$/, "~~~").gsub(/^```(\w)$/, "~~~\\1")}
EOF
)
end
exclude: ["Rakefile", "tmp", "vendor"]
navbar:
- #start:
- #- Getting Started:
- #- start/index.html.textile.liquid
- #- Quickstart:
- #- start/getting_started/publicproject.html.textile.liquid
- #- start/getting_started/firstpipeline.html.textile.liquid
- #- Common Use Cases:
- #- start/getting_started/sharedata.html.textile.liquid
- #- Next Steps:
- #- start/getting_started/nextsteps.html.textile.liquid
-
userguide:
- Welcome:
- user/index.html.textile.liquid
- Running workflows at the command line:
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- - user/topics/running-pipeline-command-line.html.textile.liquid
- - user/topics/arv-run.html.textile.liquid
- Working with git repositories:
- user/tutorials/add-new-repository.html.textile.liquid
- user/tutorials/git-arvados-guide.html.textile.liquid
- user/cwl/cwl-style.html.textile.liquid
- user/cwl/cwl-extensions.html.textile.liquid
- user/topics/arv-docker.html.textile.liquid
+ - Reference:
+ - user/reference/cookbook.html.textile.liquid
+ - Arvados License:
+ - user/copying/copying.html.textile.liquid
+ - user/copying/agpl-3.0.html
+ - user/copying/LICENSE-2.0.html
+ - user/copying/by-sa-3.0.html
+ - Obsolete documentation:
+ - user/topics/running-pipeline-command-line.html.textile.liquid
+ - user/topics/arv-run.html.textile.liquid
- user/tutorials/running-external-program.html.textile.liquid
- user/topics/crunch-tools-overview.html.textile.liquid
- user/tutorials/tutorial-firstscript.html.textile.liquid
- user/tutorials/tutorial-submit-job.html.textile.liquid
- user/topics/tutorial-parallel.html.textile.liquid
- - Develop a web service:
- - user/topics/arv-web.html.textile.liquid
- - Reference:
- - user/reference/cookbook.html.textile.liquid
- user/topics/run-command.html.textile.liquid
- user/reference/job-pipeline-ref.html.textile.liquid
- user/examples/crunch-examples.html.textile.liquid
- - Admin tools:
- - user/topics/arvados-sync-groups.html.textile.liquid
- - admin/change-account-owner.html.textile.liquid
- - admin/merge-remote-account.html.textile.liquid
- - Query the metadata database:
- user/topics/tutorial-trait-search.html.textile.liquid
- - Arvados License:
- - user/copying/copying.html.textile.liquid
- - user/copying/agpl-3.0.html
- - user/copying/LICENSE-2.0.html
- - user/copying/by-sa-3.0.html
sdk:
- Overview:
- sdk/index.html.textile.liquid
- sdk/python/sdk-python.html.textile.liquid
- sdk/python/example.html.textile.liquid
- sdk/python/python.html.textile.liquid
- - sdk/python/crunch-utility-libraries.html.textile.liquid
- sdk/python/arvados-fuse.html.textile.liquid
- sdk/python/events.html.textile.liquid
- sdk/python/cookbook.html.textile.liquid
+ - sdk/python/crunch-utility-libraries.html.textile.liquid
- CLI:
- sdk/cli/install.html.textile.liquid
- sdk/cli/index.html.textile.liquid
- api/requests.html.textile.liquid
- api/methods.html.textile.liquid
- api/resources.html.textile.liquid
- - api/permission-model.html.textile.liquid
- - api/storage.html.textile.liquid
- - api/execution.html.textile.liquid
- Permission and authentication:
- api/methods/api_client_authorizations.html.textile.liquid
- api/methods/api_clients.html.textile.liquid
- api/methods/humans.html.textile.liquid
- api/methods/specimens.html.textile.liquid
- api/methods/traits.html.textile.liquid
+ architecture:
+ - Topics:
+ - architecture/index.html.textile.liquid
+ - api/storage.html.textile.liquid
+ - api/execution.html.textile.liquid
+ - api/permission-model.html.textile.liquid
+ admin:
+ - Topics:
+ - admin/index.html.textile.liquid
+ - admin/upgrading.html.textile.liquid
+ - install/cheat_sheet.html.textile.liquid
+ - user/topics/arvados-sync-groups.html.textile.liquid
+ - admin/merge-remote-account.html.textile.liquid
+ - install/migrate-docker19.html.textile.liquid
installguide:
- Overview:
- install/index.html.textile.liquid
- Jobs API support (deprecated):
- install/install-crunch-dispatch.html.textile.liquid
- install/install-compute-node.html.textile.liquid
- - Helpful hints:
- - install/copy_pipeline_from_curoverse.html.textile.liquid
- - install/cheat_sheet.html.textile.liquid
- - Migrating from Docker 1.9:
- - install/migrate-docker19.html.textile.liquid
+++ /dev/null
-#!/usr/bin/env python
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-import arvados
-import arvados.events
-
-# 'ev' is a dict containing the log table record describing the change.
-def on_message(ev):
- if ev.get("event_type") == "create" and ev.get("object_kind") == "arvados#collection":
- print "A new collection was created: %s" % ev["object_uuid"]
-
-api = arvados.api("v1")
-ws = arvados.events.subscribe(api, [], on_message)
-ws.run_forever()
+++ /dev/null
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-import (
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
-)
<!--<li {% if page.navsection == 'start' %} class="active" {% endif %}><a href="{{ site.baseurl }}/start/index.html">Getting Started</a></li>-->
<li {% if page.navsection == 'userguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/user/index.html">User Guide</a></li>
<li {% if page.navsection == 'sdk' %} class="active" {% endif %}><a href="{{ site.baseurl }}/sdk/index.html">SDKs</a></li>
+ <li {% if page.navsection == 'architecture' %} class="active" {% endif %}><a href="{{ site.baseurl }}/architecture/index.html">Architecture</a></li>
<li {% if page.navsection == 'api' %} class="active" {% endif %}><a href="{{ site.baseurl }}/api/index.html">API</a></li>
+ <li {% if page.navsection == 'admin' %} class="active" {% endif %}><a href="{{ site.baseurl }}/admin/index.html">Admin</a></li>
<li {% if page.navsection == 'installguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/install/index.html">Install</a></li>
<li><a href="https://arvados.org" style="padding-left: 2em">arvados.org »</a></li>
</ul>
<link href="{{ site.baseurl }}/css/font-awesome.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/carousel-override.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/button-override.css" rel="stylesheet">
+ <link href="{{ site.baseurl }}/css/images.css" rel="stylesheet">
<style>
html {
height:100%;
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: "Changing account ownership"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-It is sometimes necessary to reassign an existing Arvados user account to a new Google account.
-
-Examples:
-* A user’s email address has changed from <code>person@old.example.com</code> to <code>person@new.example.com</code>.
-* A user who used to authenticate via LDAP is switching to Google login.
-
-This can be done by an administrator using Arvados APIs.
-
-First, determine the user’s existing UUID, e.g., @aaaaa-tpzed-abcdefghijklmno@.
-
-Ensure the new email address is not already associated with a different Arvados account. If it is, disassociate it by clearing that account’s @identity_url@ and @email@ fields.
-
-Clear the @identity_url@ field of the existing user record.
-
-Create a Link object with the following attributes (where @tail_uuid@ is the new email address, and @head_uuid@ is the existing user UUID):
-
-<notextile>
-<pre><code>{
- "link_class":"permission",
- "name":"can_login",
- "tail_uuid":"<span class="userinput">person@new.example.com</span>",
- "head_uuid":"<span class="userinput">aaaaa-tpzed-abcdefghijklmno</span>",
- "properties":{
- "identity_url_prefix":"https://www.google.com/"
- }
-}
-</code></pre>
-</notextile>
-
-Have the user log in using their <code>person@new.example.com</code> Google account. You can verify this by checking that the @identity_url@ field has been populated.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: "Arvados admin overview"
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This section describes how to administer an Arvados cluster. Cluster admins should already be familiar with the "Arvados architecture.":{{site.baseurl}}/architecture/index.html For instructions on installing and configuring an Arvados cluster, see the "install guide.":{{site.baseurl}}/install/index.html
---
layout: default
-navsection: userguide
-title: "Merging a remote account"
+navsection: admin
+title: "Migrating a user to a federated account"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: "Upgrading Arvados and Release notes"
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+What you need to know and do in order to upgrade your Arvados installation.
+
+h2. General process
+
+# Wait for the cluster to be idle and stop Arvados services.
+# Install new packages using @apt-get upgrade@ or @yum upgrade@.
+# Package installation scripts will perform any necessary data migrations.
+# Consult upgrade notes below to see if any manual configuration updates are necessary.
+# Restart Arvados services.
+
+h2. Upgrade notes
+
+Some versions introduce changes that require special attention when upgrading: e.g., there is a new service to install, or there is a change to the default configuration that you might need to override in order to preserve the old behavior.
+
+{% comment %}
+Note to developers: Add new items at the top. Include the date, issue number, commit, and considerations/instructions for those about to upgrade.
+
+TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
+{% endcomment %}
+
+h3. 2018-04-05: v1.1.4 regression in arvados-cwl-runner for workflows that rely on implicit discovery of secondaryFiles
+
+h4. Secondary files missing from toplevel workflow inputs
+
+If a workflow input does not declare @secondaryFiles@ corresponding to the @secondaryFiles@ of workflow steps which use the input, the workflow would inconsistently succeed or fail depending on whether the input values were specified as local files or referenced an existing collection (and whether the existing collection contained the secondary files or not). To ensure consistent behavior, the workflow is now required to declare in the top level workflow inputs any secondaryFiles that are expected by workflow steps.
+
+As an example, the following workflow will fail because the @toplevel_input@ does not declare the @secondaryFiles@ that are expected by @step_input@:
+
+<pre>
+class: Workflow
+cwlVersion: v1.0
+inputs:
+ toplevel_input: File
+outputs: []
+steps:
+ step1:
+ in:
+ step_input: toplevel_input
+ out: []
+ run:
+ id: sub
+ class: CommandLineTool
+ inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ outputs: []
+ baseCommand: echo
+</pre>
+
+When run, this produces an error like this:
+
+<pre>
+cwltool ERROR: [step step1] Cannot make job: Missing required secondary file 'hello.txt.idx' from file object: {
+ "basename": "hello.txt",
+ "class": "File",
+ "location": "keep:ade9d0e032044bd7f58daaecc0d06bc6+51/hello.txt",
+ "size": 0,
+ "nameroot": "hello",
+ "nameext": ".txt",
+ "secondaryFiles": []
+}
+</pre>
+
+To fix this error, add the appropriate @secondaryFiles@ section to @toplevel_input@
+
+<notextile>
+<pre><code>class: Workflow
+cwlVersion: v1.0
+inputs:
+ <span class="userinput">toplevel_input:
+ type: File
+ secondaryFiles:
+ - .idx</span>
+outputs: []
+steps:
+ step1:
+ in:
+ step_input: toplevel_input
+ out: []
+ run:
+ id: sub
+ class: CommandLineTool
+ inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ outputs: []
+ baseCommand: echo
+</code></pre>
+</notextile>
+
+h4. Secondary files on default file inputs
+
+Due to a bug in Arvados v1.1.4, @File@ inputs that have default values and also expect @secondaryFiles@ and will fail to upload default @secondaryFiles@. As an example, the following case will fail:
+
+<pre>
+class: CommandLineTool
+inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ default:
+ class: File
+ location: hello.txt
+outputs: []
+baseCommand: echo
+</pre>
+
+When run, this produces an error like this:
+
+<pre>
+2018-05-03 10:58:47 cwltool ERROR: Unhandled error, try again with --debug for more information:
+ [Errno 2] File not found: u'hello.txt.idx'
+</pre>
+
+To fix this, manually upload the primary and secondary files to keep and explicitly declare @secondaryFiles@ on the default primary file:
+
+<notextile>
+<pre><code>class: CommandLineTool
+inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ <span class="userinput">default:
+ class: File
+ location: keep:4d8a70b1e63b2aad6984e40e338e2373+69/hello.txt
+ secondaryFiles:
+ - class: File
+ location: keep:4d8a70b1e63b2aad6984e40e338e2373+69/hello.txt.idx</span>
+outputs: []
+baseCommand: echo
+</code></pre>
+</notextile>
+
+This bug will be fixed in an upcoming release of Arvados.
+
+h3. 2017-12-08: #11908 commit:8f987a9271 now requires minimum of Postgres 9.4 (previously 9.3)
+* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
+* Ubuntu 16.04 (pg 9.5) does not require an upgrade
+* Ubuntu 14.04 (pg 9.3) requires upgrade to Postgres 9.4: https://www.postgresql.org/download/linux/ubuntu/
+* CentOS 7 and RHEL7 (pg 9.2) require upgrade to Postgres 9.4. It is necessary to migrate of the contents of your database: https://www.postgresql.org/docs/9.0/static/migration.html
+*# Create a database backup using @pg_dump@
+*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
+*# Restore from the backup using @psql@
+
+h3. 2017-09-25: #12032 commit:68bdf4cbb now requires minimum of Postgres 9.3 (previously 9.1)
+* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
+* Ubuntu 16.04 (pg 9.5) does not require an upgrade
+* Ubuntu 14.04 (pg 9.3) is compatible, however upgrading to Postgres 9.4 is recommended: https://www.postgresql.org/download/linux/ubuntu/
+* CentOS 7 and RHEL7 (pg 9.2) should upgrade to Postgres 9.4. It is necessary to migrate of the contents of your database: https://www.postgresql.org/docs/9.0/static/migration.html
+*# Create a database backup using @pg_dump@
+*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
+*# Restore from the backup using @psql@
+
+h3. 2017-06-30: #11807 commit:55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
+* The migration can take some time if your database contains a substantial number of YAML-serialized rows (i.e., you installed Arvados before March 3, 2017 commit:660a614 and used the jobs/pipelines APIs). Otherwise, the upgrade will be no slower than usual.
+* The conversion runs as a database migration, i.e., during the deb/rpm package upgrade process, while your API server is unavailable.
+* Expect it to take about 1 minute per 20K jobs that have ever been created/run.
+
+h3. 2017-06-05: #9005 commit:cb230b0 reduces service discovery overhead in keep-web requests.
+* When upgrading keep-web _or keepproxy_ to/past this version, make sure to update API server as well. Otherwise, a bad token in a request can cause keep-web to fail future requests until either keep-web restarts or API server gets upgraded.
+
+h3. 2017-04-12: #11349 commit:2c094e2 adds a "management" http server to nodemanager.
+* To enable it, add to your configuration file: <pre>[Manage]
+ address = 127.0.0.1
+ port = 8989</pre> (see example configuration files in source:services/nodemanager/doc or https://doc.arvados.org/install/install-nodemanager.html for more info)
+* The server responds to @http://{address}:{port}/status.json@ with a summary of how many nodes are in each state (booting, busy, shutdown, etc.)
+
+h3. 2017-03-23: #10766 commit:e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
+* See http://doc.arvados.org/install/install-ws.html for install/upgrade instructions.
+* Remove the old puma server after the upgrade is complete. Example, with runit: <pre>
+$ sudo sv down /etc/sv/puma
+$ sudo rm -r /etc/sv/puma
+</pre> Example, with systemd: <pre>
+$ systemctl disable puma
+$ systemctl stop puma
+</pre>
+
+h3. 2017-03-06: #11168 commit:660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
+* Aside from a slight performance improvement, this should have no externally visible effect.
+* Downgrading past this version is not supported, and is likely to cause errors. If this happens, the solution is to upgrade past this version.
+* After upgrading, make sure to restart puma and crunch-dispatch-* processes.
+
+h3. 2017-02-03: #10969 commit:74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
+* If your compute nodes run a version of *docker older than 1.10* you must override the default by adding to your API server configuration (@/etc/arvados/api/application.yml@): <pre><code class="yaml">docker_image_formats: ["v1"]</code></pre>
+* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue #10969 for more detail.
+* *NOTE:* This does *not* include any support for migrating existing Docker images from v1 to v2 format. This will come later: for now, sites running Docker 1.9 or earlier should still *avoid upgrading Docker further than 1.9.*
+
+h3. 2016-09-27: several Debian and RPM packages -- keep-balance (commit:d9eec0b), keep-web (commit:3399e63), keepproxy (commit:6de67b6), and arvados-git-httpd (commit:9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags (commit:3bbe1cd).
+* On Debian-based systems using systemd, services are enabled automatically when packages are installed.
+* On RedHat-based systems using systemd, unit files are installed but services must be enabled explicitly: e.g., <code>"sudo systemctl enable keep-web; sudo systemctl start keep-web"</code>.
+* The new systemd-supervised services will not start up successfully until configuration files are installed in /etc/arvados/: e.g., <code>"Sep 26 18:23:55 62751f5bb946 keep-web[74]: 2016/09/26 18:23:55 open /etc/arvados/keep-web/keep-web.yml: no such file or directory"</code>
+* To migrate from runit to systemd after installing the new packages, we recommend the following procedure:
+*# Bring down the runit service: "sv down /etc/sv/keep-web"
+*# Create a JSON configuration file (e.g., /etc/arvados/keep-web/keep-web.yml -- see "keep-web -help")
+*# Ensure the service is running correctly under systemd: "systemctl status keep-web" / "journalctl -u keep-web"
+*# Remove the runit service so it doesn't start at next boot
+* Affected services:
+** keep-balance - /etc/arvados/keep-balance/keep-balance.yml
+** keep-web - /etc/arvados/keep-web/keep-web.yml
+** keepproxy - /etc/arvados/keepproxy/keepproxy.yml
+** arvados-git-httpd - /etc/arvados/arv-git-httpd/arv-git-httpd.yml
+
+h3. 2016-05-31: commit:ae72b172c8 and commit:3aae316c25 install Python modules and scripts to different locations on the filesystem.
+* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See #9242 for more background.
+* If you simply import Python modules from scripts, or call Python tools relying on $PATH, you don't need to make any changes. If you have hardcoded full paths to some of these files (e.g., in symbolic links or configuration files), you will need to update those paths after this upgrade.
+
+h3. 2016-04-25: commit:eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
+* On each Debian-based compute node and shell node, run: @sudo apt-get install crunchrunner@
+* On each Red Hat-based compute node and shell node, run: @sudo yum install crunchrunner@
+
+h3. 2016-04-21: commit:3c88abd changes the Keep permission signature algorithm.
+* All software components that generate signatures must be upgraded together. These are: keepstore, API server, keep-block-check, and keep-rsync. For example, if keepstore < 0.1.20160421183420 but API server >= 0.1.20160421183420, clients will not be able to read or write data in Keep.
+* Jobs and client operations that are in progress during the upgrade (including arv-put's "resume cache") will fail.
+
+h3. 2015-01-05: commit:e1276d6e disables Workbench's "Getting Started" popup by default.
+* If you want new users to continue seeing this popup, set @enable_getting_started_popup: true@ in Workbench's @application.yml@ configuration.
+
+h3. 2015-12-03: commit:5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see #7751)
+* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge commit:346a558
+* Jobs will fail if the API server (in particular crunch-job from the arvados-cli gem) is upgraded without upgrading arvados-fuse on compute nodes.
+
+h3. 2015-11-11: commit:1e2ace5 changes recommended config for keep-web (see #5824)
+* proxy/dns/ssl config should be updated to route "https://download.uuid_prefix.arvadosapi.com/" requests to keep-web (alongside the existing "collections" routing)
+* keep-web command line adds @-attachment-only-host download.uuid_prefix.arvadosapi.com@
+* Workbench config adds @keep_web_download_url@
+* More info on the (still beta/non-TOC-linked) "keep-web doc page":http://doc.arvados.org/install/install-keep-web.html
+
+h3. 2015-11-04: commit:1d1c6de removes stopped containers (see #7444)
+* arvados-docker-cleaner removes _all_ docker containers as soon as they exit, effectively making @docker run@ default to @--rm@. If you run arvados-docker-cleaner on a host that does anything other than run crunch-jobs, and you still want to be able to use @docker start@, read the "new doc page":http://doc.arvados.org/install/install-compute-node.html to learn how to turn this off before upgrading.
+
+h3. 2015-11-04: commit:21006cf adds a keep-web service (see #5824)
+* Nothing relies on it yet, but early adopters can install it now by following http://doc.arvados.org/install/install-keep-web.html (it is not yet linked in the TOC).
---
layout: default
-navsection: api
+navsection: architecture
title: Computing with Crunch
...
{% comment %}
h2. Container API
-Note: although the preferred API for Arvados going forward, the Container API may not yet be available on all installations.
-
# To submit work, create a "container request":{{site.baseurl}}/api/methods/container_requests.html in the @Committed@ state.
# The system will fufill the container request by creating or reusing a "Container object":{{site.baseurl}}/api/methods/containers.html and assigning it to the @container_uuid@ field. If the same request has been submitted in the past, it may reuse an existing container. The reuse behavior can be suppressed with @use_existing: false@ in the container request.
# The dispatcher process will notice a new container in @Queued@ state and submit a container executor to the underlying work queuing system (such as SLURM).
# When the container associated with the container request is completed, the container request will go into the @Final@ state.
# The @output_uuid@ field of the container request contains the uuid of output collection produced by container request.
-!{{site.baseurl}}/images/Crunch_dispatch.svg!
+!(full-width){{site.baseurl}}/images/Crunch_dispatch.svg!
h2. Job API (deprecated)
---
layout: default
-navsection: api
+navsection: architecture
navmenu: Concepts
title: "Permission model"
...
h2. Example
-!{{site.baseurl}}/images/Arvados_Permissions.svg!
+!(full-width){{site.baseurl}}/images/Arvados_Permissions.svg!
---
layout: default
-navsection: api
+navsection: architecture
title: Storage in Keep
...
{% comment %}
Keep clients are applications such as @arv-get@, @arv-put@ and @arv-mount@ which store and retrieve data from Keep. In doing so, these programs interact with both the API server (which stores file metadata in form of Collection objects) and individual Keep servers (which store the actual data blocks).
-!{{site.baseurl}}/images/Keep_reading_writing_block.svg!
+!(full-width){{site.baseurl}}/images/Keep_reading_writing_block.svg!
h2. Storing a file
# The client creates a "collection":{{site.baseurl}}/api/methods/collections.html and provides the @manifest_text@
# The API server accepts the collection after validating the signed tokens (proof of knowledge) for each block.
-!{{site.baseurl}}/images/Keep_manifests.svg!
+!(full-width){{site.baseurl}}/images/Keep_manifests.svg!
h2. Fetching a file
# The client sends the data block request to the keep server, along with the token signature from the API which proves to Keep servers that the client is permitted to read a given block.
# The server provides the block data after validating the token signature for the block (if the server does not have the block, it returns a 404 and the client tries the next highest priority server)
-!{{site.baseurl}}/images/Keep_rendezvous_hashing.svg!
+!(full-width){{site.baseurl}}/images/Keep_rendezvous_hashing.svg!
Each @keep_service@ resource has an assigned uuid. To determine priority assignments of blocks to servers, for each keep service compute the MD5 sum of the string concatenation of the block locator (hex-coded hash part only) and service uuid, then sort this list in descending order. Blocks are preferentially placed on servers with the highest weight.
--- /dev/null
+---
+layout: default
+navsection: architecture
+title: "Arvados components"
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+!(full-width){{site.baseurl}}/images/Arvados_arch.svg!
+
+h3. Services
+
+Located in @arvados/services@ except for Workbench which is located in @arvados/apps/workbench@.
+
+table(table table-bordered table-condensed).
+|_. Component|_. Description|
+|api|The API server is the core of Arvados. It is backed by a Postgres database and manages information such as metadata for storage, a record of submitted compute jobs, users, groups, and associated permissions.|
+|arv-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.|
+|crunch-dispatch-local|Get compute requests submitted to the API server and execute them locally.|
+|crunch-dispatch-slurm|Get compute requests submitted to the API server and submit them to slurm.|
+|crunch-run|Dispatched by crunch-dispatch, executes a single compute run: setting up a Docker container, running it, and collecting the output.|
+|dockercleaner|Daemon for cleaning up Docker containers and images.|
+|fuse|Filesystem in USErspace (FUSE) filesystem driver for Keep.|
+|health|Health check proxy, contacts configured Arvados services at their health check endpoints and reports results.|
+|keep-balance|Perform storage utilization reporting, optimization and garbage collection. Moves data blocks to their optimum location, ensures correct replication and storage class, and trashes unreferenced blocks.|
+|keepproxy|Provides low-level access to keepstore services (block-level data access) for clients outside the internal (private) network.|
+|keepstore|Provides access to underlying storage (filesystem or object storage such as Amazon S3 or Azure Blob) with Arvados permissions.|
+|keep-web|Provides high-level WebDAV access to collections (file-level data access).|
+|login-sync|Synchronize virtual machine users with Arvados users and permissions.|
+|nodemanager|Provide elastic computing by creating and destroying cloud based virtual machines on compute demand.|
+|ws|Publishes API server change events over websockets.|
+|workbench|Web application providing user interface to Arvados services.|
+
+h3. Tools
+
+The @arv@ command is located in @arvados/sdk/ruby@, the @arv-*@ tools are located in @arvados/sdk/python@, the rest are located in @arvados/tools@.
+
+table(table table-bordered table-condensed).
+|_. Component|_. Description |
+|arv|Provides command line access to API, also provides some purpose utilities.|
+|arv-copy|Copy a collection from one cluster to another|
+|arv-get|Get files from a collection.|
+|arv-keepdocker|Upload Docker images from local Docker daemon to Keep.|
+|arv-ls|List files in a collection|
+|arv-migrate-docker19|Migrate Docker images in Keep from v1 format (Docker 1.9 or earlier) to v2 format (Docker 1.10 or later)|
+|arv-normalize|Read manifest text on stdin and produce normalized manifest text on stdout.|
+|arv-put|Upload files to a collection.|
+|arv-ws|Print events from Arvados websocket event source.|
+|arvbash|Helpful @bash@ macros for using Arvados at the command line.|
+|arvbox|Dockerized Arvados environment for development and testing.|
+|crunchstat-summary|Read execution metrics (cpu %, ram, network, etc) collected from a compute container and produce a report.|
+|keep-block-check|Given a list of keep block locators, check that each block exists on one of the configured keepstore servers and verify the block hash.|
+|keep-exercise|Benchmarking tool to test throughput and reliability of keepstores under various usage patterns.|
+|keep-rsync|Get lists of blocks from two clusters, copy blocks which exist on source cluster but are missing from destination cluster.|
+|sync-groups|Take a CSV file listing (group, username) pairs and synchronize membership in Arvados groups.|
--- /dev/null
+img.full-width {
+ width: 100%
+}
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.2" width="280mm" height="210mm" viewBox="0 0 28000 21000" preserveAspectRatio="xMidYMid" fill-rule="evenodd" stroke-width="28.222" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" xmlns:ooo="http://xml.openoffice.org/svg/export" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:presentation="http://sun.com/xmlns/staroffice/presentation" xmlns:smil="http://www.w3.org/2001/SMIL20/" xmlns:anim="urn:oasis:names:tc:opendocument:xmlns:animation:1.0" xml:space="preserve">
+ <defs class="ClipPathGroup">
+ <clipPath id="presentation_clip_path" clipPathUnits="userSpaceOnUse">
+ <rect x="0" y="0" width="28000" height="21000"/>
+ </clipPath>
+ <clipPath id="presentation_clip_path_shrink" clipPathUnits="userSpaceOnUse">
+ <rect x="28" y="21" width="27944" height="20958"/>
+ </clipPath>
+ </defs>
+ <defs>
+ <font id="EmbeddedFont_1" horiz-adv-x="2048">
+ <font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="normal" font-style="normal" ascent="1852" descent="423"/>
+ <missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
+ <glyph unicode="y" horiz-adv-x="1059" d="M 604,1 C 579,-64 553,-123 527,-175 500,-227 471,-272 438,-309 405,-346 369,-374 329,-394 289,-413 243,-423 191,-423 168,-423 147,-423 128,-423 109,-423 88,-420 67,-414 L 67,-279 C 80,-282 94,-284 110,-284 126,-284 140,-284 151,-284 204,-284 253,-264 298,-225 343,-186 383,-124 417,-38 L 434,5 5,1082 197,1082 425,484 C 432,466 440,442 451,412 461,382 471,352 482,322 492,292 501,265 509,241 517,217 522,202 523,196 525,203 530,218 538,240 545,261 554,285 564,312 573,339 583,366 593,393 603,420 611,444 618,464 L 830,1082 1020,1082 604,1 Z"/>
+ <glyph unicode="x" horiz-adv-x="1033" d="M 801,0 L 510,444 217,0 23,0 408,556 41,1082 240,1082 510,661 778,1082 979,1082 612,558 1002,0 801,0 Z"/>
+ <glyph unicode="w" horiz-adv-x="1535" d="M 1174,0 L 965,0 792,698 C 787,716 781,738 776,765 770,792 764,818 759,843 752,872 746,903 740,934 734,904 728,874 721,845 716,820 710,793 704,766 697,739 691,715 686,694 L 508,0 300,0 -3,1082 175,1082 358,347 C 363,332 367,313 372,291 377,268 381,246 386,225 391,200 396,175 401,149 406,174 412,199 418,223 423,244 429,265 434,286 439,307 444,325 448,339 L 644,1082 837,1082 1026,339 C 1031,322 1036,302 1041,280 1046,258 1051,237 1056,218 1061,195 1067,172 1072,149 1077,174 1083,199 1088,223 1093,244 1098,265 1103,288 1108,310 1112,330 1117,347 L 1308,1082 1484,1082 1174,0 Z"/>
+ <glyph unicode="v" horiz-adv-x="1059" d="M 613,0 L 400,0 7,1082 199,1082 437,378 C 442,363 447,346 454,325 460,304 466,282 473,259 480,236 486,215 492,194 497,173 502,155 506,141 510,155 515,173 522,194 528,215 534,236 541,258 548,280 555,302 562,323 569,344 575,361 580,376 L 826,1082 1017,1082 613,0 Z"/>
+ <glyph unicode="u" horiz-adv-x="901" d="M 314,1082 L 314,396 C 314,343 318,299 326,264 333,229 346,200 363,179 380,157 403,142 432,133 460,124 495,119 537,119 580,119 618,127 653,142 687,157 716,178 741,207 765,235 784,270 797,312 810,353 817,401 817,455 L 817,1082 997,1082 997,228 C 997,205 997,181 998,156 998,131 998,107 999,85 1000,62 1000,43 1001,27 1002,11 1002,3 1003,3 L 833,3 C 832,6 832,15 831,30 830,44 830,61 829,79 828,98 827,117 826,136 825,156 825,172 825,185 L 822,185 C 805,154 786,125 765,100 744,75 720,53 693,36 666,18 634,4 599,-6 564,-15 523,-20 476,-20 416,-20 364,-13 321,2 278,17 242,39 214,70 186,101 166,140 153,188 140,236 133,294 133,361 L 133,1082 314,1082 Z"/>
+ <glyph unicode="t" horiz-adv-x="531" d="M 554,8 C 527,1 499,-5 471,-10 442,-14 409,-16 372,-16 228,-16 156,66 156,229 L 156,951 31,951 31,1082 163,1082 216,1324 336,1324 336,1082 536,1082 536,951 336,951 336,268 C 336,216 345,180 362,159 379,138 408,127 450,127 467,127 484,128 501,131 517,134 535,137 554,141 L 554,8 Z"/>
+ <glyph unicode="s" horiz-adv-x="927" d="M 950,299 C 950,248 940,203 921,164 901,124 872,91 835,64 798,37 752,16 698,2 643,-13 581,-20 511,-20 448,-20 392,-15 342,-6 291,4 247,20 209,41 171,62 139,91 114,126 88,161 69,203 57,254 L 216,285 C 231,227 263,185 311,158 359,131 426,117 511,117 550,117 585,120 618,125 650,130 678,140 701,153 724,166 743,183 756,205 769,226 775,253 775,285 775,318 767,345 752,366 737,387 715,404 688,418 661,432 628,444 589,455 550,465 507,476 460,489 417,500 374,513 331,527 288,541 250,560 216,583 181,606 153,634 132,668 111,702 100,745 100,796 100,895 135,970 206,1022 276,1073 378,1099 513,1099 632,1099 727,1078 798,1036 868,994 912,927 931,834 L 769,814 C 763,842 752,866 736,885 720,904 701,919 678,931 655,942 630,951 602,956 573,961 544,963 513,963 432,963 372,951 333,926 294,901 275,864 275,814 275,785 282,761 297,742 311,723 331,707 357,694 382,681 413,669 449,660 485,650 525,640 568,629 597,622 626,614 656,606 686,597 715,587 744,576 772,564 799,550 824,535 849,519 870,500 889,478 908,456 923,430 934,401 945,372 950,338 950,299 Z"/>
+ <glyph unicode="r" horiz-adv-x="556" d="M 142,0 L 142,830 C 142,853 142,876 142,900 141,923 141,946 140,968 139,990 139,1011 138,1030 137,1049 137,1067 136,1082 L 306,1082 C 307,1067 308,1049 309,1030 310,1010 311,990 312,969 313,948 313,929 314,910 314,891 314,874 314,861 L 318,861 C 331,902 344,938 359,969 373,999 390,1024 409,1044 428,1063 451,1078 478,1088 505,1097 537,1102 575,1102 590,1102 604,1101 617,1099 630,1096 641,1094 648,1092 L 648,927 C 636,930 622,933 606,935 590,936 572,937 552,937 511,937 476,928 447,909 418,890 394,865 376,832 357,799 344,759 335,714 326,668 322,618 322,564 L 322,0 142,0 Z"/>
+ <glyph unicode="p" horiz-adv-x="953" d="M 1053,546 C 1053,464 1046,388 1033,319 1020,250 998,190 967,140 936,90 895,51 844,23 793,-6 730,-20 655,-20 578,-20 510,-5 452,24 394,53 350,101 319,168 L 314,168 C 315,167 315,161 316,150 316,139 316,126 317,110 317,94 317,76 318,57 318,37 318,17 318,-2 L 318,-425 138,-425 138,864 C 138,891 138,916 138,940 137,964 137,986 136,1005 135,1025 135,1042 134,1056 133,1070 133,1077 132,1077 L 306,1077 C 307,1075 308,1068 309,1057 310,1045 311,1031 312,1014 313,998 314,980 315,961 316,943 316,925 316,908 L 320,908 C 337,943 356,972 377,997 398,1021 423,1041 450,1057 477,1072 508,1084 542,1091 575,1098 613,1101 655,1101 730,1101 793,1088 844,1061 895,1034 936,997 967,949 998,900 1020,842 1033,774 1046,705 1053,629 1053,546 Z M 864,542 C 864,609 860,668 852,720 844,772 830,816 811,852 791,888 765,915 732,934 699,953 658,962 609,962 569,962 531,956 496,945 461,934 430,912 404,880 377,848 356,804 341,748 326,691 318,618 318,528 318,451 324,387 337,334 350,281 368,238 393,205 417,172 447,149 483,135 519,120 560,113 607,113 657,113 699,123 732,142 765,161 791,189 811,226 830,263 844,308 852,361 860,414 864,474 864,542 Z"/>
+ <glyph unicode="o" horiz-adv-x="980" d="M 1053,542 C 1053,353 1011,212 928,119 845,26 724,-20 565,-20 490,-20 422,-9 363,14 304,37 254,71 213,118 172,165 140,223 119,294 97,364 86,447 86,542 86,915 248,1102 571,1102 655,1102 728,1090 789,1067 850,1044 900,1009 939,962 978,915 1006,857 1025,787 1044,717 1053,635 1053,542 Z M 864,542 C 864,626 858,695 845,750 832,805 813,848 788,881 763,914 732,937 696,950 660,963 619,969 574,969 528,969 487,962 450,949 413,935 381,912 355,879 329,846 309,802 296,747 282,692 275,624 275,542 275,458 282,389 297,334 312,279 332,235 358,202 383,169 414,146 449,133 484,120 522,113 563,113 609,113 651,120 688,133 725,146 757,168 783,201 809,234 829,278 843,333 857,388 864,458 864,542 Z"/>
+ <glyph unicode="n" horiz-adv-x="900" d="M 825,0 L 825,686 C 825,739 821,783 814,818 806,853 793,882 776,904 759,925 736,941 708,950 679,959 644,963 602,963 559,963 521,956 487,941 452,926 423,904 399,876 374,847 355,812 342,771 329,729 322,681 322,627 L 322,0 142,0 142,853 C 142,876 142,900 142,925 141,950 141,974 140,996 139,1019 139,1038 138,1054 137,1070 137,1078 136,1078 L 306,1078 C 307,1075 307,1066 308,1052 309,1037 310,1021 311,1002 312,984 312,965 313,945 314,926 314,910 314,897 L 317,897 C 334,928 353,957 374,982 395,1007 419,1029 446,1047 473,1064 505,1078 540,1088 575,1097 616,1102 663,1102 723,1102 775,1095 818,1080 861,1065 897,1043 925,1012 953,981 974,942 987,894 1000,845 1006,788 1006,721 L 1006,0 825,0 Z"/>
+ <glyph unicode="m" horiz-adv-x="1456" d="M 768,0 L 768,686 C 768,739 765,783 758,818 751,853 740,882 725,904 709,925 688,941 663,950 638,959 607,963 570,963 532,963 498,956 467,941 436,926 410,904 389,876 367,847 350,812 339,771 327,729 321,681 321,627 L 321,0 142,0 142,853 C 142,876 142,900 142,925 141,950 141,974 140,996 139,1019 139,1038 138,1054 137,1070 137,1078 136,1078 L 306,1078 C 307,1075 307,1066 308,1052 309,1037 310,1021 311,1002 312,984 312,965 313,945 314,926 314,910 314,897 L 317,897 C 333,928 350,957 369,982 388,1007 410,1029 435,1047 460,1064 488,1078 521,1088 553,1097 590,1102 633,1102 715,1102 780,1086 828,1053 875,1020 908,968 927,897 L 930,897 C 946,928 964,957 984,982 1004,1007 1027,1029 1054,1047 1081,1064 1111,1078 1144,1088 1177,1097 1215,1102 1258,1102 1313,1102 1360,1095 1400,1080 1439,1065 1472,1043 1497,1012 1522,981 1541,942 1553,894 1565,845 1571,788 1571,721 L 1571,0 1393,0 1393,686 C 1393,739 1390,783 1383,818 1376,853 1365,882 1350,904 1334,925 1313,941 1288,950 1263,959 1232,963 1195,963 1157,963 1123,956 1092,942 1061,927 1035,906 1014,878 992,850 975,815 964,773 952,731 946,682 946,627 L 946,0 768,0 Z"/>
+ <glyph unicode="l" horiz-adv-x="187" d="M 138,0 L 138,1484 318,1484 318,0 138,0 Z"/>
+ <glyph unicode="k" horiz-adv-x="927" d="M 816,0 L 450,494 318,385 318,0 138,0 138,1484 318,1484 318,557 793,1082 1004,1082 565,617 1027,0 816,0 Z"/>
+ <glyph unicode="j" horiz-adv-x="372" d="M 137,1312 L 137,1484 317,1484 317,1312 137,1312 Z M 317,-132 C 317,-174 314,-212 307,-247 300,-283 287,-313 269,-339 251,-365 227,-386 196,-401 165,-416 125,-423 77,-423 54,-423 32,-423 11,-423 -11,-423 -31,-421 -50,-416 L -50,-277 C -41,-278 -31,-280 -19,-281 -7,-282 3,-283 12,-283 37,-283 58,-280 75,-273 91,-266 104,-256 113,-242 122,-227 129,-209 132,-187 135,-164 137,-138 137,-107 L 137,1082 317,1082 317,-132 Z"/>
+ <glyph unicode="i" horiz-adv-x="187" d="M 137,1312 L 137,1484 317,1484 317,1312 137,1312 Z M 137,0 L 137,1082 317,1082 317,0 137,0 Z"/>
+ <glyph unicode="h" horiz-adv-x="874" d="M 317,897 C 337,934 359,965 382,991 405,1016 431,1037 459,1054 487,1071 518,1083 551,1091 584,1098 622,1102 663,1102 732,1102 789,1093 834,1074 878,1055 913,1029 939,996 964,962 982,922 992,875 1001,828 1006,777 1006,721 L 1006,0 825,0 825,686 C 825,732 822,772 817,807 811,842 800,871 784,894 768,917 745,934 716,946 687,957 649,963 602,963 559,963 521,955 487,940 452,925 423,903 399,875 374,847 355,813 342,773 329,733 322,688 322,638 L 322,0 142,0 142,1484 322,1484 322,1098 C 322,1076 322,1054 321,1032 320,1010 320,990 319,971 318,952 317,937 316,924 315,911 315,902 314,897 L 317,897 Z"/>
+ <glyph unicode="g" horiz-adv-x="954" d="M 548,-425 C 486,-425 431,-419 383,-406 335,-393 294,-375 260,-352 226,-328 198,-300 177,-267 156,-234 140,-198 131,-158 L 312,-132 C 324,-182 351,-220 392,-248 433,-274 486,-288 553,-288 594,-288 631,-282 664,-271 697,-260 726,-241 749,-217 772,-191 790,-159 803,-119 816,-79 822,-30 822,27 L 822,201 820,201 C 807,174 790,148 771,123 751,98 727,75 699,56 670,37 637,21 600,10 563,-2 520,-8 472,-8 403,-8 345,4 296,27 247,50 207,84 176,130 145,176 122,233 108,302 93,370 86,449 86,539 86,626 93,704 108,773 122,842 145,901 178,950 210,998 252,1035 304,1061 355,1086 418,1099 492,1099 569,1099 635,1082 692,1047 748,1012 791,962 822,897 L 824,897 C 824,914 825,933 826,953 827,974 828,994 829,1012 830,1031 831,1046 832,1060 833,1073 835,1080 836,1080 L 1007,1080 C 1006,1074 1006,1064 1005,1050 1004,1035 1004,1018 1003,998 1002,978 1002,956 1002,932 1001,907 1001,882 1001,856 L 1001,30 C 1001,-121 964,-234 890,-311 815,-387 701,-425 548,-425 Z M 822,541 C 822,616 814,681 798,735 781,788 760,832 733,866 706,900 676,925 642,941 607,957 572,965 536,965 490,965 451,957 418,941 385,925 357,900 336,866 314,831 298,787 288,734 277,680 272,616 272,541 272,463 277,398 288,345 298,292 314,249 335,216 356,183 383,160 416,146 449,132 488,125 533,125 569,125 604,133 639,148 673,163 704,188 731,221 758,254 780,297 797,350 814,403 822,466 822,541 Z"/>
+ <glyph unicode="f" horiz-adv-x="557" d="M 361,951 L 361,0 181,0 181,951 29,951 29,1082 181,1082 181,1204 C 181,1243 185,1280 192,1314 199,1347 213,1377 233,1402 252,1427 279,1446 313,1461 347,1475 391,1482 445,1482 466,1482 489,1481 512,1479 535,1477 555,1474 572,1470 L 572,1333 C 561,1335 548,1337 533,1339 518,1340 504,1341 492,1341 465,1341 444,1337 427,1330 410,1323 396,1312 387,1299 377,1285 370,1268 367,1248 363,1228 361,1205 361,1179 L 361,1082 572,1082 572,951 361,951 Z"/>
+ <glyph unicode="e" horiz-adv-x="980" d="M 276,503 C 276,446 282,394 294,347 305,299 323,258 348,224 372,189 403,163 441,144 479,125 525,115 578,115 656,115 719,131 766,162 813,193 844,233 861,281 L 1019,236 C 1008,206 992,176 972,146 951,115 924,88 890,64 856,39 814,19 763,4 712,-12 650,-20 578,-20 418,-20 296,28 213,123 129,218 87,360 87,548 87,649 100,735 125,806 150,876 185,933 229,977 273,1021 324,1053 383,1073 442,1092 504,1102 571,1102 662,1102 738,1087 799,1058 860,1029 909,988 946,937 983,885 1009,824 1025,754 1040,684 1048,608 1048,527 L 1048,503 276,503 Z M 862,641 C 852,755 823,838 775,891 727,943 658,969 568,969 538,969 507,964 474,955 441,945 410,928 382,903 354,878 330,845 311,803 292,760 281,706 278,641 L 862,641 Z"/>
+ <glyph unicode="d" horiz-adv-x="954" d="M 821,174 C 788,105 744,55 689,25 634,-5 565,-20 484,-20 347,-20 247,26 183,118 118,210 86,349 86,536 86,913 219,1102 484,1102 566,1102 634,1087 689,1057 744,1027 788,979 821,914 L 823,914 C 823,921 823,931 823,946 822,960 822,975 822,991 821,1006 821,1021 821,1035 821,1049 821,1059 821,1065 L 821,1484 1001,1484 1001,219 C 1001,193 1001,168 1002,143 1002,119 1002,97 1003,77 1004,57 1004,40 1005,26 1006,11 1006,4 1007,4 L 835,4 C 834,11 833,20 832,32 831,44 830,58 829,73 828,89 827,105 826,123 825,140 825,157 825,174 L 821,174 Z M 275,542 C 275,467 280,403 289,350 298,297 313,253 334,219 355,184 381,159 413,143 445,127 484,119 530,119 577,119 619,127 656,142 692,157 722,182 747,217 771,251 789,296 802,351 815,406 821,474 821,554 821,631 815,696 802,749 789,802 771,844 746,877 721,910 691,933 656,948 620,962 579,969 532,969 488,969 450,961 418,946 386,931 359,906 338,872 317,838 301,794 291,740 280,685 275,619 275,542 Z"/>
+ <glyph unicode="c" horiz-adv-x="875" d="M 275,546 C 275,484 280,427 289,375 298,323 313,278 334,241 355,203 384,174 419,153 454,132 497,122 548,122 612,122 666,139 709,173 752,206 778,258 788,328 L 970,328 C 964,283 951,239 931,197 911,155 884,118 850,86 815,54 773,28 724,9 675,-10 618,-20 553,-20 468,-20 396,-6 337,23 278,52 230,91 193,142 156,192 129,251 112,320 95,388 87,462 87,542 87,615 93,679 105,735 117,790 134,839 156,881 177,922 203,957 232,986 261,1014 293,1037 328,1054 362,1071 398,1083 436,1091 474,1098 512,1102 551,1102 612,1102 666,1094 713,1077 760,1060 801,1038 836,1009 870,980 898,945 919,906 940,867 955,824 964,779 L 779,765 C 770,825 746,873 708,908 670,943 616,961 546,961 495,961 452,953 418,936 383,919 355,893 334,859 313,824 298,781 289,729 280,677 275,616 275,546 Z"/>
+ <glyph unicode="b" horiz-adv-x="953" d="M 1053,546 C 1053,169 920,-20 655,-20 573,-20 505,-5 451,25 396,54 352,102 318,168 L 316,168 C 316,150 316,132 315,113 314,94 313,77 312,61 311,45 310,31 309,19 308,8 307,2 306,2 L 132,2 C 133,8 133,18 134,32 135,47 135,64 136,84 137,104 137,126 138,150 138,174 138,199 138,225 L 138,1484 318,1484 318,1061 C 318,1041 318,1022 318,1004 317,985 317,969 316,955 315,938 315,923 314,908 L 318,908 C 351,977 396,1027 451,1057 506,1087 574,1102 655,1102 792,1102 892,1056 957,964 1021,872 1053,733 1053,546 Z M 864,540 C 864,615 859,679 850,732 841,785 826,829 805,864 784,898 758,923 726,939 694,955 655,963 609,963 562,963 520,955 484,940 447,925 417,900 393,866 368,832 350,787 337,732 324,677 318,609 318,529 318,452 324,387 337,334 350,281 368,239 393,206 417,173 447,149 483,135 519,120 560,113 607,113 651,113 689,121 721,136 753,151 780,176 801,210 822,244 838,288 849,343 859,397 864,463 864,540 Z"/>
+ <glyph unicode="a" horiz-adv-x="1060" d="M 414,-20 C 305,-20 224,9 169,66 114,124 87,203 87,303 87,375 101,434 128,480 155,526 190,562 234,588 277,614 327,632 383,642 439,652 496,657 554,657 L 797,657 797,717 C 797,762 792,800 783,832 774,863 759,889 740,908 721,928 697,942 668,951 639,960 604,965 565,965 530,965 499,963 471,958 443,953 419,944 398,931 377,918 361,900 348,878 335,855 327,827 323,793 L 135,810 C 142,853 154,892 173,928 192,963 218,994 253,1020 287,1046 330,1066 382,1081 433,1095 496,1102 569,1102 705,1102 807,1071 876,1009 945,946 979,856 979,738 L 979,272 C 979,219 986,179 1000,152 1014,125 1041,111 1080,111 1090,111 1100,112 1110,113 1120,114 1130,116 1139,118 L 1139,6 C 1116,1 1094,-3 1072,-6 1049,-9 1025,-10 1000,-10 966,-10 937,-5 913,4 888,13 868,26 853,45 838,63 826,86 818,113 810,140 805,171 803,207 L 797,207 C 778,172 757,141 734,113 711,85 684,61 653,42 622,22 588,7 549,-4 510,-15 465,-20 414,-20 Z M 455,115 C 512,115 563,125 606,146 649,167 684,194 713,226 741,259 762,294 776,332 790,371 797,408 797,443 L 797,531 600,531 C 556,531 514,528 475,522 435,517 400,506 370,489 340,472 316,449 299,418 281,388 272,349 272,300 272,241 288,195 320,163 351,131 396,115 455,115 Z"/>
+ <glyph unicode="W" horiz-adv-x="1906" d="M 1511,0 L 1283,0 1039,895 C 1032,920 1024,950 1016,985 1007,1020 1000,1053 993,1084 985,1121 977,1158 969,1196 960,1157 952,1120 944,1083 937,1051 929,1018 921,984 913,950 905,920 898,895 L 652,0 424,0 9,1409 208,1409 461,514 C 472,472 483,430 494,389 504,348 513,311 520,278 529,239 537,203 544,168 554,214 564,259 575,304 580,323 584,342 589,363 594,384 599,404 604,424 609,444 614,463 619,482 624,500 628,517 632,532 L 877,1409 1060,1409 1305,532 C 1309,517 1314,500 1319,482 1324,463 1329,444 1334,425 1339,405 1343,385 1348,364 1353,343 1357,324 1362,305 1373,260 1383,215 1393,168 1394,168 1397,180 1402,203 1407,226 1414,254 1422,289 1430,324 1439,361 1449,402 1458,442 1468,479 1478,514 L 1727,1409 1926,1409 1511,0 Z"/>
+ <glyph unicode="S" horiz-adv-x="1139" d="M 1272,389 C 1272,330 1261,275 1238,225 1215,175 1179,132 1131,96 1083,59 1023,31 950,11 877,-10 790,-20 690,-20 515,-20 378,11 280,72 182,133 120,222 93,338 L 278,375 C 287,338 302,305 321,275 340,245 367,219 400,198 433,176 473,159 522,147 571,135 629,129 697,129 754,129 806,134 853,144 900,153 941,168 975,188 1009,208 1036,234 1055,266 1074,297 1083,335 1083,379 1083,425 1073,462 1052,491 1031,520 1001,543 963,562 925,581 880,596 827,609 774,622 716,635 652,650 613,659 573,668 534,679 494,689 456,701 420,716 383,730 349,747 317,766 285,785 257,809 234,836 211,863 192,894 179,930 166,965 159,1006 159,1053 159,1120 173,1177 200,1225 227,1272 264,1311 312,1342 360,1373 417,1395 482,1409 547,1423 618,1430 694,1430 781,1430 856,1423 918,1410 980,1396 1032,1375 1075,1348 1118,1321 1152,1287 1178,1247 1203,1206 1224,1159 1239,1106 L 1051,1073 C 1042,1107 1028,1137 1011,1164 993,1191 970,1213 941,1231 912,1249 878,1263 837,1272 796,1281 747,1286 692,1286 627,1286 572,1280 528,1269 483,1257 448,1241 421,1221 394,1201 374,1178 363,1151 351,1124 345,1094 345,1063 345,1021 356,987 377,960 398,933 426,910 462,892 498,874 540,859 587,847 634,835 685,823 738,811 781,801 825,791 868,781 911,770 952,758 991,744 1030,729 1067,712 1102,693 1136,674 1166,650 1191,622 1216,594 1236,561 1251,523 1265,485 1272,440 1272,389 Z"/>
+ <glyph unicode="P" horiz-adv-x="1086" d="M 1258,985 C 1258,924 1248,867 1228,814 1207,761 1177,715 1137,676 1096,637 1046,606 985,583 924,560 854,549 773,549 L 359,549 359,0 168,0 168,1409 761,1409 C 844,1409 917,1399 979,1379 1041,1358 1093,1330 1134,1293 1175,1256 1206,1211 1227,1159 1248,1106 1258,1048 1258,985 Z M 1066,983 C 1066,1072 1039,1140 984,1187 929,1233 847,1256 738,1256 L 359,1256 359,700 746,700 C 856,700 937,724 989,773 1040,822 1066,892 1066,983 Z"/>
+ <glyph unicode="L" horiz-adv-x="900" d="M 168,0 L 168,1409 359,1409 359,156 1071,156 1071,0 168,0 Z"/>
+ <glyph unicode="I" horiz-adv-x="186" d="M 189,0 L 189,1409 380,1409 380,0 189,0 Z"/>
+ <glyph unicode="F" horiz-adv-x="1006" d="M 359,1253 L 359,729 1145,729 1145,571 359,571 359,0 168,0 168,1409 1169,1409 1169,1253 359,1253 Z"/>
+ <glyph unicode="E" horiz-adv-x="1112" d="M 168,0 L 168,1409 1237,1409 1237,1253 359,1253 359,801 1177,801 1177,647 359,647 359,156 1278,156 1278,0 168,0 Z"/>
+ <glyph unicode="C" horiz-adv-x="1297" d="M 792,1274 C 712,1274 641,1261 580,1234 518,1207 466,1169 425,1120 383,1071 351,1011 330,942 309,873 298,796 298,711 298,626 310,549 333,479 356,408 389,348 432,297 475,246 527,207 590,179 652,151 722,137 800,137 855,137 905,144 950,159 995,173 1035,193 1072,219 1108,245 1140,276 1169,312 1198,347 1223,387 1245,430 L 1401,352 C 1376,299 1344,250 1307,205 1270,160 1226,120 1176,87 1125,54 1068,28 1005,9 941,-10 870,-20 791,-20 677,-20 577,-2 492,35 406,71 334,122 277,187 219,252 176,329 147,418 118,507 104,605 104,711 104,821 119,920 150,1009 180,1098 224,1173 283,1236 341,1298 413,1346 498,1380 583,1413 681,1430 790,1430 940,1430 1065,1401 1166,1342 1267,1283 1341,1196 1388,1081 L 1207,1021 C 1194,1054 1176,1086 1153,1117 1130,1147 1102,1174 1068,1197 1034,1220 994,1239 949,1253 903,1267 851,1274 792,1274 Z"/>
+ <glyph unicode="A" horiz-adv-x="1350" d="M 1167,0 L 1006,412 364,412 202,0 4,0 579,1409 796,1409 1362,0 1167,0 Z M 768,1026 C 757,1053 747,1080 738,1107 728,1134 719,1159 712,1182 705,1204 699,1223 694,1238 689,1253 686,1262 685,1265 684,1262 681,1252 676,1237 671,1222 665,1203 658,1180 650,1157 641,1132 632,1105 622,1078 612,1051 602,1024 L 422,561 949,561 768,1026 Z"/>
+ <glyph unicode="3" horiz-adv-x="980" d="M 1049,389 C 1049,324 1039,267 1018,216 997,165 966,123 926,88 885,53 835,26 776,8 716,-11 648,-20 571,-20 484,-20 410,-9 351,13 291,34 242,63 203,99 164,134 135,175 116,221 97,266 84,313 78,362 L 264,379 C 269,342 279,308 294,277 308,246 327,220 352,198 377,176 407,159 443,147 479,135 522,129 571,129 662,129 733,151 785,196 836,241 862,307 862,395 862,447 851,489 828,521 805,552 776,577 742,595 707,612 670,624 630,630 589,636 552,639 518,639 L 416,639 416,795 514,795 C 548,795 583,799 620,806 657,813 690,825 721,844 751,862 776,887 796,918 815,949 825,989 825,1038 825,1113 803,1173 759,1217 714,1260 648,1282 561,1282 482,1282 418,1262 369,1221 320,1180 291,1123 283,1049 L 102,1063 C 109,1125 126,1179 153,1225 180,1271 214,1309 255,1340 296,1370 342,1393 395,1408 448,1423 504,1430 563,1430 642,1430 709,1420 766,1401 823,1381 869,1354 905,1321 941,1287 968,1247 985,1202 1002,1157 1010,1108 1010,1057 1010,1016 1004,977 993,941 982,905 964,873 940,844 916,815 886,791 849,770 812,749 767,734 715,723 L 715,719 C 772,713 821,700 863,681 905,661 940,636 967,607 994,578 1015,544 1029,507 1042,470 1049,430 1049,389 Z"/>
+ <glyph unicode="0" horiz-adv-x="980" d="M 1059,705 C 1059,570 1046,456 1021,364 995,271 960,197 916,140 871,83 819,42 759,17 699,-8 635,-20 567,-20 498,-20 434,-8 375,17 316,42 264,82 221,139 177,196 143,270 118,363 93,455 80,569 80,705 80,847 93,965 118,1058 143,1151 177,1225 221,1280 265,1335 317,1374 377,1397 437,1419 502,1430 573,1430 640,1430 704,1419 763,1397 822,1374 873,1335 917,1280 961,1225 996,1151 1021,1058 1046,965 1059,847 1059,705 Z M 876,705 C 876,817 869,910 856,985 843,1059 823,1118 797,1163 771,1207 739,1238 702,1257 664,1275 621,1284 573,1284 522,1284 478,1275 439,1256 400,1237 368,1206 342,1162 315,1117 295,1058 282,984 269,909 262,816 262,705 262,597 269,506 283,432 296,358 316,299 343,254 369,209 401,176 439,157 477,137 520,127 569,127 616,127 659,137 697,157 735,176 767,209 794,254 820,299 840,358 855,432 869,506 876,597 876,705 Z"/>
+ <glyph unicode="." horiz-adv-x="186" d="M 187,0 L 187,219 382,219 382,0 187,0 Z"/>
+ <glyph unicode="-" horiz-adv-x="504" d="M 91,464 L 91,624 591,624 591,464 91,464 Z"/>
+ <glyph unicode="," horiz-adv-x="212" d="M 385,219 L 385,51 C 385,16 384,-16 381,-46 378,-74 373,-101 366,-127 359,-151 351,-175 342,-197 332,-219 320,-241 307,-262 L 184,-262 C 214,-219 237,-175 254,-131 270,-87 278,-43 278,0 L 190,0 190,219 385,219 Z"/>
+ <glyph unicode=" " horiz-adv-x="556"/>
+ </font>
+ </defs>
+ <defs>
+ <font id="EmbeddedFont_2" horiz-adv-x="2048">
+ <font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="bold" font-style="normal" ascent="1852" descent="423"/>
+ <missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
+ <glyph unicode="x" horiz-adv-x="1139" d="M 819,0 L 567,392 313,0 14,0 410,559 33,1082 336,1082 567,728 797,1082 1102,1082 725,562 1124,0 819,0 Z"/>
+ <glyph unicode="w" horiz-adv-x="1615" d="M 436,255 L 645,1082 946,1082 1153,255 1337,1082 1597,1082 1313,0 1016,0 797,882 571,0 274,0 -6,1082 258,1082 436,255 Z"/>
+ <glyph unicode="v" horiz-adv-x="1139" d="M 565,227 L 836,1082 1130,1082 731,0 395,0 8,1082 305,1082 565,227 Z"/>
+ <glyph unicode="t" horiz-adv-x="636" d="M 420,-18 C 337,-18 274,5 229,50 184,95 162,163 162,254 L 162,892 25,892 25,1082 176,1082 264,1336 440,1336 440,1082 645,1082 645,892 440,892 440,330 C 440,277 450,239 470,214 490,189 521,176 563,176 580,176 596,177 610,180 624,183 640,186 657,190 L 657,16 C 622,5 586,-4 547,-10 508,-15 466,-18 420,-18 Z"/>
+ <glyph unicode="s" horiz-adv-x="980" d="M 1055,316 C 1055,264 1044,217 1023,176 1001,135 969,100 928,71 887,42 836,19 776,4 716,-12 648,-20 571,-20 502,-20 440,-15 385,-5 330,5 281,22 240,45 198,68 163,97 135,134 107,171 86,216 72,270 L 319,307 C 327,277 338,253 352,234 366,215 383,201 404,191 425,181 449,174 477,171 504,168 536,166 571,166 603,166 633,168 661,172 688,175 712,182 733,191 753,200 769,212 780,229 791,245 797,265 797,290 797,318 789,340 773,357 756,373 734,386 706,397 677,407 644,416 606,424 567,431 526,440 483,450 438,460 393,472 349,486 305,500 266,519 231,543 196,567 168,598 147,635 126,672 115,718 115,775 115,826 125,872 145,913 165,953 194,987 233,1016 272,1044 320,1066 377,1081 434,1096 499,1103 573,1103 632,1103 686,1098 737,1087 788,1076 833,1058 873,1035 913,1011 947,981 974,944 1001,907 1019,863 1030,811 L 781,785 C 776,811 768,833 756,850 744,867 729,880 712,890 694,900 673,907 650,911 627,914 601,916 573,916 506,916 456,908 423,891 390,874 373,845 373,805 373,780 380,761 394,746 407,731 427,719 452,710 477,700 506,692 541,685 575,678 612,669 653,659 703,648 752,636 801,622 849,607 892,588 930,563 967,538 998,505 1021,466 1044,427 1055,377 1055,316 Z"/>
+ <glyph unicode="r" horiz-adv-x="662" d="M 143,0 L 143,833 C 143,856 143,881 143,907 142,933 142,958 141,982 140,1006 139,1027 138,1046 137,1065 136,1075 135,1075 L 403,1075 C 404,1067 406,1054 407,1035 408,1016 410,995 411,972 412,950 414,927 415,905 416,883 416,865 416,851 L 420,851 C 434,890 448,926 462,957 476,988 493,1014 512,1036 531,1057 553,1074 580,1086 607,1097 640,1103 679,1103 696,1103 712,1102 729,1099 745,1096 757,1092 766,1088 L 766,853 C 748,857 730,861 712,864 693,867 671,868 646,868 576,868 522,840 483,783 444,726 424,642 424,531 L 424,0 143,0 Z"/>
+ <glyph unicode="p" horiz-adv-x="1059" d="M 1167,546 C 1167,464 1159,388 1143,319 1126,250 1101,190 1067,140 1033,90 990,51 938,23 885,-6 823,-20 752,-20 720,-20 688,-17 657,-10 625,-3 595,8 566,23 537,38 511,57 487,82 462,106 441,136 424,172 L 418,172 C 419,169 419,160 420,147 421,134 421,118 422,101 423,83 423,64 424,45 424,25 424,7 424,-10 L 424,-425 143,-425 143,833 C 143,888 142,938 141,981 139,1024 137,1058 135,1082 L 408,1082 C 409,1077 411,1068 413,1055 414,1042 416,1026 417,1009 418,992 418,974 419,955 420,936 420,920 420,906 L 424,906 C 458,977 505,1028 564,1059 623,1090 692,1105 770,1105 839,1105 898,1091 948,1063 998,1035 1039,996 1072,947 1104,898 1128,839 1144,771 1159,702 1167,627 1167,546 Z M 874,546 C 874,669 855,761 818,821 781,880 725,910 651,910 623,910 595,904 568,893 540,881 515,861 494,833 472,804 454,766 441,719 427,671 420,611 420,538 420,467 427,409 440,362 453,315 471,277 493,249 514,221 539,201 566,190 593,178 621,172 649,172 685,172 717,179 745,194 773,208 797,230 816,261 835,291 849,330 859,377 869,424 874,481 874,546 Z"/>
+ <glyph unicode="o" horiz-adv-x="1086" d="M 1171,542 C 1171,459 1160,384 1137,315 1114,246 1079,187 1033,138 987,88 930,49 861,22 792,-6 712,-20 621,-20 533,-20 455,-6 388,21 321,48 264,87 219,136 173,185 138,245 115,314 92,383 80,459 80,542 80,623 91,697 114,766 136,834 170,893 215,943 260,993 317,1032 386,1060 455,1088 535,1102 627,1102 724,1102 807,1088 876,1060 945,1032 1001,993 1045,944 1088,894 1120,835 1141,767 1161,698 1171,623 1171,542 Z M 877,542 C 877,671 856,764 814,822 772,880 711,909 631,909 548,909 485,880 441,821 397,762 375,669 375,542 375,477 381,422 393,375 404,328 421,290 442,260 463,230 489,208 519,194 549,179 582,172 618,172 659,172 696,179 729,194 761,208 788,230 810,260 832,290 849,328 860,375 871,422 877,477 877,542 Z"/>
+ <glyph unicode="n" horiz-adv-x="1006" d="M 844,0 L 844,607 C 844,649 841,688 834,723 827,758 816,788 801,813 786,838 766,857 741,871 716,885 686,892 651,892 617,892 586,885 559,870 531,855 507,833 487,806 467,778 452,745 441,707 430,668 424,626 424,580 L 424,0 143,0 143,845 C 143,868 143,892 143,917 142,942 142,966 141,988 140,1010 139,1031 138,1048 137,1066 136,1075 135,1075 L 403,1075 C 404,1067 406,1055 407,1038 408,1021 410,1002 411,981 412,961 414,940 415,919 416,899 416,881 416,867 L 420,867 C 458,950 506,1010 563,1047 620,1084 689,1103 768,1103 833,1103 889,1092 934,1071 979,1050 1015,1020 1044,983 1072,946 1092,902 1105,851 1118,800 1124,746 1124,687 L 1124,0 844,0 Z"/>
+ <glyph unicode="l" horiz-adv-x="292" d="M 143,0 L 143,1484 424,1484 424,0 143,0 Z"/>
+ <glyph unicode="k" horiz-adv-x="1033" d="M 834,0 L 545,490 424,406 424,0 143,0 143,1484 424,1484 424,634 810,1082 1112,1082 732,660 1141,0 834,0 Z"/>
+ <glyph unicode="i" horiz-adv-x="292" d="M 143,1277 L 143,1484 424,1484 424,1277 143,1277 Z M 143,0 L 143,1082 424,1082 424,0 143,0 Z"/>
+ <glyph unicode="g" horiz-adv-x="1060" d="M 596,-434 C 525,-434 462,-427 408,-413 353,-398 307,-378 269,-353 230,-327 200,-296 177,-261 154,-225 138,-186 129,-143 L 410,-110 C 420,-153 442,-187 475,-212 508,-237 551,-249 604,-249 637,-249 668,-244 696,-235 723,-226 747,-210 767,-188 786,-165 802,-136 813,-99 824,-62 829,-17 829,37 829,56 829,75 829,94 829,113 829,131 830,147 831,166 831,184 831,201 L 829,201 C 796,131 751,80 692,49 633,18 562,2 481,2 412,2 353,16 304,43 254,70 213,107 180,156 147,204 123,262 108,329 92,396 84,469 84,550 84,633 92,709 109,777 126,844 151,902 186,951 220,1000 263,1037 316,1064 368,1090 430,1103 502,1103 574,1103 639,1088 696,1057 753,1026 797,977 829,908 L 834,908 C 834,922 835,939 836,957 837,976 838,994 839,1011 840,1029 842,1044 844,1058 845,1071 847,1078 848,1078 L 1114,1078 C 1113,1054 1111,1020 1110,977 1109,934 1108,885 1108,829 L 1108,32 C 1108,-47 1097,-115 1074,-173 1051,-231 1018,-280 975,-318 931,-357 877,-386 814,-405 750,-424 677,-434 596,-434 Z M 831,556 C 831,624 824,681 811,726 798,771 780,808 759,835 738,862 713,882 686,893 658,904 630,910 602,910 566,910 534,903 507,889 479,875 455,853 436,824 417,795 402,757 392,712 382,667 377,613 377,550 377,433 396,345 433,286 470,227 526,197 600,197 628,197 656,203 684,214 711,225 736,244 758,272 780,299 798,336 811,382 824,428 831,486 831,556 Z"/>
+ <glyph unicode="f" horiz-adv-x="663" d="M 473,892 L 473,0 193,0 193,892 35,892 35,1082 193,1082 193,1195 C 193,1236 198,1275 208,1310 218,1345 235,1375 259,1401 283,1427 315,1447 356,1462 397,1477 447,1484 508,1484 540,1484 572,1482 603,1479 634,1476 661,1472 686,1468 L 686,1287 C 674,1290 661,1292 646,1294 631,1295 617,1296 604,1296 578,1296 557,1293 540,1288 523,1283 509,1275 500,1264 490,1253 483,1240 479,1224 475,1207 473,1188 473,1167 L 473,1082 686,1082 686,892 473,892 Z"/>
+ <glyph unicode="e" horiz-adv-x="980" d="M 586,-20 C 508,-20 438,-8 376,15 313,38 260,73 216,120 172,167 138,226 115,297 92,368 80,451 80,546 80,649 94,736 122,807 149,878 187,935 234,979 281,1022 335,1054 396,1073 457,1092 522,1102 590,1102 675,1102 748,1087 809,1057 869,1027 918,986 957,932 996,878 1024,814 1042,739 1060,664 1069,582 1069,491 L 1069,491 375,491 C 375,445 379,402 387,363 395,323 408,289 426,261 444,232 467,209 496,193 525,176 559,168 600,168 649,168 690,179 721,200 752,221 775,253 788,297 L 1053,274 C 1041,243 1024,211 1003,176 981,141 952,110 916,81 880,52 835,28 782,9 728,-10 663,-20 586,-20 Z M 586,925 C 557,925 531,920 506,911 481,901 459,886 441,865 422,844 407,816 396,783 385,750 378,710 377,663 L 797,663 C 792,750 771,816 734,860 697,903 648,925 586,925 Z"/>
+ <glyph unicode="c" horiz-adv-x="1007" d="M 594,-20 C 508,-20 433,-7 369,20 304,47 251,84 208,133 165,182 133,240 112,309 91,377 80,452 80,535 80,625 92,705 115,776 138,846 172,905 216,954 260,1002 314,1039 379,1064 443,1089 516,1102 598,1102 668,1102 730,1092 785,1073 839,1054 886,1028 925,995 964,963 996,924 1021,879 1045,834 1062,786 1071,734 L 788,734 C 780,787 760,830 728,861 696,893 651,909 592,909 517,909 462,878 427,816 392,754 375,664 375,546 375,297 449,172 596,172 649,172 694,188 730,221 766,253 788,302 797,366 L 1079,366 C 1072,315 1057,267 1034,220 1010,174 978,133 938,97 897,62 848,33 791,12 734,-9 668,-20 594,-20 Z"/>
+ <glyph unicode="a" horiz-adv-x="1112" d="M 393,-20 C 341,-20 295,-13 254,2 213,16 178,37 149,65 120,93 98,127 83,168 68,208 60,255 60,307 60,371 71,425 94,469 116,513 146,548 185,575 224,602 269,622 321,634 373,647 428,653 487,653 L 720,653 720,709 C 720,748 717,782 710,808 703,835 692,857 679,873 666,890 649,902 630,909 610,916 587,920 562,920 539,920 518,918 500,913 481,909 465,901 452,890 439,879 428,864 420,845 411,826 405,803 402,774 L 109,774 C 117,822 132,866 153,906 174,946 204,981 242,1010 279,1039 326,1062 381,1078 436,1094 500,1102 574,1102 641,1102 701,1094 754,1077 807,1060 851,1036 888,1003 925,970 953,929 972,881 991,833 1001,777 1001,714 L 1001,320 C 1001,295 1002,272 1005,252 1007,232 1011,215 1018,202 1024,188 1033,178 1045,171 1056,164 1071,160 1090,160 1111,160 1132,162 1152,166 L 1152,14 C 1135,10 1120,6 1107,3 1094,0 1080,-3 1067,-5 1054,-7 1040,-9 1025,-10 1010,-11 992,-12 972,-12 901,-12 849,5 816,40 782,75 762,126 755,193 L 749,193 C 712,126 664,73 606,36 547,-1 476,-20 393,-20 Z M 720,499 L 576,499 C 546,499 518,497 491,493 464,490 440,482 420,470 399,459 383,442 371,420 359,397 353,367 353,329 353,277 365,239 389,214 412,189 444,176 483,176 519,176 552,184 581,199 610,214 635,234 656,259 676,284 692,312 703,345 714,377 720,411 720,444 L 720,499 Z"/>
+ <glyph unicode="S" horiz-adv-x="1218" d="M 1286,406 C 1286,342 1274,284 1251,232 1228,179 1192,134 1143,97 1094,60 1031,31 955,11 878,-10 787,-20 682,-20 589,-20 506,-12 435,5 364,22 303,46 252,79 201,112 159,152 128,201 96,249 73,304 59,367 L 344,414 C 352,383 364,354 379,328 394,302 416,280 443,261 470,242 503,227 544,217 584,206 633,201 690,201 790,201 867,216 920,247 973,277 999,324 999,389 999,428 988,459 967,484 946,509 917,529 882,545 847,561 806,574 760,585 714,596 666,606 616,616 576,625 536,635 496,645 456,655 418,667 382,681 345,695 311,712 280,731 249,750 222,774 199,803 176,831 158,864 145,902 132,940 125,985 125,1036 125,1106 139,1166 167,1216 195,1266 234,1307 284,1339 333,1370 392,1393 461,1408 530,1423 605,1430 686,1430 778,1430 857,1423 923,1409 988,1394 1043,1372 1088,1343 1132,1314 1167,1277 1193,1233 1218,1188 1237,1136 1249,1077 L 963,1038 C 948,1099 919,1144 874,1175 829,1206 764,1221 680,1221 628,1221 585,1217 551,1208 516,1199 489,1186 469,1171 448,1156 434,1138 425,1118 416,1097 412,1076 412,1053 412,1018 420,990 437,968 454,945 477,927 507,912 537,897 573,884 615,874 656,863 702,853 752,842 796,833 840,823 883,813 926,802 968,790 1007,776 1046,762 1083,745 1117,725 1151,705 1181,681 1206,652 1231,623 1250,588 1265,548 1279,508 1286,461 1286,406 Z"/>
+ <glyph unicode="I" horiz-adv-x="292" d="M 137,0 L 137,1409 432,1409 432,0 137,0 Z"/>
+ <glyph unicode="E" horiz-adv-x="1139" d="M 137,0 L 137,1409 1245,1409 1245,1181 432,1181 432,827 1184,827 1184,599 432,599 432,228 1286,228 1286,0 137,0 Z"/>
+ <glyph unicode=")" horiz-adv-x="583" d="M 2,-425 C 55,-347 101,-270 139,-196 177,-120 208,-44 233,33 257,110 275,190 286,272 297,353 303,439 303,530 303,620 297,706 286,788 275,869 257,949 233,1026 208,1103 177,1180 139,1255 101,1330 55,1407 2,1484 L 283,1484 C 334,1410 379,1337 416,1264 453,1191 484,1116 509,1039 533,962 551,882 563,799 574,716 580,626 580,531 580,436 574,347 563,264 551,180 533,99 509,22 484,-55 453,-131 416,-204 379,-277 334,-351 283,-425 L 2,-425 Z"/>
+ <glyph unicode="(" horiz-adv-x="583" d="M 399,-425 C 348,-351 303,-277 266,-204 229,-131 198,-55 174,22 149,99 131,180 120,264 108,347 102,436 102,531 102,626 108,716 120,799 131,882 149,962 174,1039 198,1116 229,1191 266,1264 303,1337 348,1410 399,1484 L 680,1484 C 627,1407 581,1330 543,1255 505,1180 474,1103 450,1026 425,949 407,869 396,788 385,706 379,620 379,530 379,439 385,353 396,272 407,190 425,110 450,33 474,-44 505,-120 543,-196 581,-270 627,-347 680,-425 L 399,-425 Z"/>
+ <glyph unicode=" " horiz-adv-x="556"/>
+ </font>
+ </defs>
+ <defs class="TextShapeIndex">
+ <g ooo:slide="id1" ooo:id-list="id3 id4 id5 id6 id7 id8 id9 id10 id11 id12 id13 id14 id15 id16 id17 id18 id19 id20 id21 id22 id23 id24 id25 id26 id27 id28 id29 id30 id31 id32 id33 id34 id35 id36 id37 id38 id39 id40 id41 id42"/>
+ </defs>
+ <defs class="EmbeddedBulletChars">
+ <g id="bullet-char-template(57356)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 580,1141 L 1163,571 580,0 -4,571 580,1141 Z"/>
+ </g>
+ <g id="bullet-char-template(57354)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 8,1128 L 1137,1128 1137,0 8,0 8,1128 Z"/>
+ </g>
+ <g id="bullet-char-template(10146)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 174,0 L 602,739 174,1481 1456,739 174,0 Z M 1358,739 L 309,1346 659,739 1358,739 Z"/>
+ </g>
+ <g id="bullet-char-template(10132)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 2015,739 L 1276,0 717,0 1260,543 174,543 174,936 1260,936 717,1481 1274,1481 2015,739 Z"/>
+ </g>
+ <g id="bullet-char-template(10007)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 0,-2 C -7,14 -16,27 -25,37 L 356,567 C 262,823 215,952 215,954 215,979 228,992 255,992 264,992 276,990 289,987 310,991 331,999 354,1012 L 381,999 492,748 772,1049 836,1024 860,1049 C 881,1039 901,1025 922,1006 886,937 835,863 770,784 769,783 710,716 594,584 L 774,223 C 774,196 753,168 711,139 L 727,119 C 717,90 699,76 672,76 641,76 570,178 457,381 L 164,-76 C 142,-110 111,-127 72,-127 30,-127 9,-110 8,-76 1,-67 -2,-52 -2,-32 -2,-23 -1,-13 0,-2 Z"/>
+ </g>
+ <g id="bullet-char-template(10004)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 285,-33 C 182,-33 111,30 74,156 52,228 41,333 41,471 41,549 55,616 82,672 116,743 169,778 240,778 293,778 328,747 346,684 L 369,508 C 377,444 397,411 428,410 L 1163,1116 C 1174,1127 1196,1133 1229,1133 1271,1133 1292,1118 1292,1087 L 1292,965 C 1292,929 1282,901 1262,881 L 442,47 C 390,-6 338,-33 285,-33 Z"/>
+ </g>
+ <g id="bullet-char-template(9679)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 813,0 C 632,0 489,54 383,161 276,268 223,411 223,592 223,773 276,916 383,1023 489,1130 632,1184 813,1184 992,1184 1136,1130 1245,1023 1353,916 1407,772 1407,592 1407,412 1353,268 1245,161 1136,54 992,0 813,0 Z"/>
+ </g>
+ <g id="bullet-char-template(8226)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 346,457 C 273,457 209,483 155,535 101,586 74,649 74,723 74,796 101,859 155,911 209,963 273,989 346,989 419,989 480,963 531,910 582,859 608,796 608,723 608,648 583,586 532,535 482,483 420,457 346,457 Z"/>
+ </g>
+ <g id="bullet-char-template(8211)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M -4,459 L 1135,459 1135,606 -4,606 -4,459 Z"/>
+ </g>
+ <g id="bullet-char-template(61548)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 173,740 C 173,903 231,1043 346,1159 462,1274 601,1332 765,1332 928,1332 1067,1274 1183,1159 1299,1043 1357,903 1357,740 1357,577 1299,437 1183,322 1067,206 928,148 765,148 601,148 462,206 346,322 231,437 173,577 173,740 Z"/>
+ </g>
+ </defs>
+ <defs class="TextEmbeddedBitmaps"/>
+ <g>
+ <g id="id2" class="Master_Slide">
+ <g id="bg-id2" class="Background"/>
+ <g id="bo-id2" class="BackgroundObjects"/>
+ </g>
+ </g>
+ <g class="SlideGroup">
+ <g>
+ <g id="container-id1">
+ <g id="id1" class="Slide" clip-path="url(#presentation_clip_path)">
+ <g class="Page">
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id3">
+ <rect class="BoundingBox" stroke="none" fill="none" x="16493" y="6587" width="2416" height="2289"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 16494,6588 L 18907,8874"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id4">
+ <rect class="BoundingBox" stroke="none" fill="none" x="13572" y="1506" width="2036" height="1909"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14589,1507 C 15165,1507 15605,1919 15605,2459 15605,2999 15165,3412 14589,3412 14013,3412 13573,2999 13573,2459 13573,1919 14013,1507 14589,1507 Z M 13573,1507 L 13573,1507 Z M 15606,3413 L 15606,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14589,1507 C 15165,1507 15605,1919 15605,2459 15605,2999 15165,3412 14589,3412 14013,3412 13573,2999 13573,2459 13573,1919 14013,1507 14589,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 13573,1507 L 13573,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 15606,3413 L 15606,3413 Z"/>
+ <path fill="rgb(91,127,166)" stroke="none" d="M 14258,2005 C 14311,2005 14352,2076 14352,2169 14352,2262 14311,2333 14258,2333 14205,2333 14165,2262 14165,2169 14165,2076 14205,2005 14258,2005 Z M 13573,1507 L 13573,1507 Z M 15606,3413 L 15606,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14258,2005 C 14311,2005 14352,2076 14352,2169 14352,2262 14311,2333 14258,2333 14205,2333 14165,2262 14165,2169 14165,2076 14205,2005 14258,2005 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 13573,1507 L 13573,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 15606,3413 L 15606,3413 Z"/>
+ <path fill="rgb(91,127,166)" stroke="none" d="M 14916,2005 C 14969,2005 15010,2076 15010,2169 15010,2262 14969,2333 14916,2333 14863,2333 14823,2262 14823,2169 14823,2076 14863,2005 14916,2005 Z M 13573,1507 L 13573,1507 Z M 15606,3413 L 15606,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14916,2005 C 14969,2005 15010,2076 15010,2169 15010,2262 14969,2333 14916,2333 14863,2333 14823,2262 14823,2169 14823,2076 14863,2005 14916,2005 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 13573,1507 L 13573,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 15606,3413 L 15606,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14031,2787 C 14389,3141 14789,3141 15147,2787"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 13573,1507 L 13573,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 15606,3413 L 15606,3413 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id5">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7349" y="1506" width="2036" height="1909"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 8366,1507 C 8942,1507 9382,1919 9382,2459 9382,2999 8942,3412 8366,3412 7790,3412 7350,2999 7350,2459 7350,1919 7790,1507 8366,1507 Z M 7350,1507 L 7350,1507 Z M 9383,3413 L 9383,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 8366,1507 C 8942,1507 9382,1919 9382,2459 9382,2999 8942,3412 8366,3412 7790,3412 7350,2999 7350,2459 7350,1919 7790,1507 8366,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7350,1507 L 7350,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 9383,3413 L 9383,3413 Z"/>
+ <path fill="rgb(91,127,166)" stroke="none" d="M 8035,2005 C 8088,2005 8129,2076 8129,2169 8129,2262 8088,2333 8035,2333 7982,2333 7942,2262 7942,2169 7942,2076 7982,2005 8035,2005 Z M 7350,1507 L 7350,1507 Z M 9383,3413 L 9383,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 8035,2005 C 8088,2005 8129,2076 8129,2169 8129,2262 8088,2333 8035,2333 7982,2333 7942,2262 7942,2169 7942,2076 7982,2005 8035,2005 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7350,1507 L 7350,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 9383,3413 L 9383,3413 Z"/>
+ <path fill="rgb(91,127,166)" stroke="none" d="M 8693,2005 C 8746,2005 8787,2076 8787,2169 8787,2262 8746,2333 8693,2333 8640,2333 8600,2262 8600,2169 8600,2076 8640,2005 8693,2005 Z M 7350,1507 L 7350,1507 Z M 9383,3413 L 9383,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 8693,2005 C 8746,2005 8787,2076 8787,2169 8787,2262 8746,2333 8693,2333 8640,2333 8600,2262 8600,2169 8600,2076 8640,2005 8693,2005 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7350,1507 L 7350,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 9383,3413 L 9383,3413 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7808,2787 C 8166,3141 8566,3141 8924,2787"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7350,1507 L 7350,1507 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 9383,3413 L 9383,3413 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id6">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12682" y="5570" width="4194" height="1400"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14779,6968 L 12683,6968 12683,5571 16874,5571 16874,6968 14779,6968 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14779,6968 L 12683,6968 12683,5571 16874,5571 16874,6968 14779,6968 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="13528" y="6441"><tspan fill="rgb(0,0,0)" stroke="none">Workbench</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id7">
+ <rect class="BoundingBox" stroke="none" fill="none" x="5824" y="8618" width="4194" height="1654"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 7921,10270 L 5825,10270 5825,8619 10016,8619 10016,10270 7921,10270 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7921,10270 L 5825,10270 5825,8619 10016,8619 10016,10270 7921,10270 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="6784" y="9339"><tspan fill="rgb(0,0,0)" stroke="none">keepproxy</tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="6850" y="9894"><tspan fill="rgb(0,0,0)" stroke="none">keep-web</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id8">
+ <rect class="BoundingBox" stroke="none" fill="none" x="22080" y="8492" width="4194" height="1781"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 24177,10271 L 22081,10271 22081,8493 26272,8493 26272,10271 24177,10271 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 24177,10271 L 22081,10271 22081,8493 26272,8493 26272,10271 24177,10271 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="22856" y="9554"><tspan fill="rgb(0,0,0)" stroke="none">arv-git-httpd</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id9">
+ <rect class="BoundingBox" stroke="none" fill="none" x="17635" y="8492" width="4194" height="1781"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 19732,10271 L 17636,10271 17636,8493 21827,8493 21827,10271 19732,10271 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 19732,10271 L 17636,10271 17636,8493 21827,8493 21827,10271 19732,10271 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="19008" y="9554"><tspan fill="rgb(0,0,0)" stroke="none">arv-ws</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id10">
+ <rect class="BoundingBox" stroke="none" fill="none" x="5825" y="15730" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 7604,18144 L 5826,18144 5826,15731 9382,15731 9382,18144 7604,18144 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7604,18144 L 5826,18144 5826,15731 9382,15731 9382,18144 7604,18144 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id11">
+ <rect class="BoundingBox" stroke="none" fill="none" x="6079" y="16111" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 7858,18525 L 6080,18525 6080,16112 9636,16112 9636,18525 7858,18525 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 7858,18525 L 6080,18525 6080,16112 9636,16112 9636,18525 7858,18525 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id12">
+ <rect class="BoundingBox" stroke="none" fill="none" x="6460" y="16492" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 8239,18906 L 6461,18906 6461,16493 10017,16493 10017,18906 8239,18906 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 8239,18906 L 6461,18906 6461,16493 10017,16493 10017,18906 8239,18906 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="7149" y="17871"><tspan fill="rgb(0,0,0)" stroke="none">keepstore</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id13">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12556" y="15730" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14335,18144 L 12557,18144 12557,15731 16113,15731 16113,18144 14335,18144 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14335,18144 L 12557,18144 12557,15731 16113,15731 16113,18144 14335,18144 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id14">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12810" y="16111" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14589,18525 L 12811,18525 12811,16112 16367,16112 16367,18525 14589,18525 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14589,18525 L 12811,18525 12811,16112 16367,16112 16367,18525 14589,18525 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id15">
+ <rect class="BoundingBox" stroke="none" fill="none" x="13191" y="16492" width="3559" height="2416"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14970,18906 L 13192,18906 13192,16493 16748,16493 16748,18906 14970,18906 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14970,18906 L 13192,18906 13192,16493 16748,16493 16748,18906 14970,18906 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="13671" y="17871"><tspan fill="rgb(0,0,0)" stroke="none">compute0...</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id16">
+ <rect class="BoundingBox" stroke="none" fill="none" x="15477" y="10143" width="5972" height="5972"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 15478,10144 L 21447,16113"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id17">
+ <rect class="BoundingBox" stroke="none" fill="none" x="14588" y="6968" width="3" height="1527"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 14589,6969 L 14589,8493"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id18">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7984" y="10270" width="3" height="5464"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 7985,10271 L 7985,15732"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id19">
+ <rect class="BoundingBox" stroke="none" fill="none" x="10016" y="17382" width="2543" height="3"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 10017,17383 L 12557,17383"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id20">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12047" y="13064" width="5210" height="1781"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14652,14843 L 12048,14843 12048,13065 17255,13065 17255,14843 14652,14843 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14652,14843 L 12048,14843 12048,13065 17255,13065 17255,14843 14652,14843 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="12209" y="14126"><tspan fill="rgb(0,0,0)" stroke="none">crunch-dispatch-slurm</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id21">
+ <rect class="BoundingBox" stroke="none" fill="none" x="14588" y="10143" width="3" height="2924"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 14589,10144 L 14589,13065"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id22">
+ <rect class="BoundingBox" stroke="none" fill="none" x="14588" y="14842" width="3" height="892"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 14589,14843 L 14589,15732"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id23">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1582" y="12123" width="24872" height="107"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 1635,12176 L 1844,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 1978,12176 L 2187,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 2322,12176 L 2531,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 2665,12176 L 2874,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 3009,12176 L 3218,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 3352,12176 L 3561,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 3696,12176 L 3904,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 4039,12176 L 4248,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 4383,12176 L 4591,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 4726,12176 L 4935,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 5069,12176 L 5278,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 5413,12176 L 5622,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 5756,12176 L 5965,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 6100,12176 L 6309,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 6443,12176 L 6652,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 6787,12176 L 6995,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 7130,12176 L 7339,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 7473,12176 L 7682,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 7817,12176 L 8026,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 8160,12176 L 8369,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 8504,12176 L 8713,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 8847,12176 L 9056,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 9191,12176 L 9399,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 9534,12176 L 9743,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 9878,12176 L 10086,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 10221,12176 L 10430,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 10564,12176 L 10773,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 10908,12176 L 11117,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 11251,12176 L 11460,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 11595,12176 L 11804,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 11938,12176 L 12147,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 12282,12176 L 12490,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 12625,12176 L 12834,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 12969,12176 L 13177,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 13312,12176 L 13521,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 13655,12176 L 13864,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 13999,12176 L 14208,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 14342,12176 L 14551,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 14686,12176 L 14895,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 15029,12176 L 15238,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 15373,12176 L 15581,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 15716,12176 L 15925,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 16059,12176 L 16268,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 16403,12176 L 16612,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 16746,12176 L 16955,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 17090,12176 L 17299,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 17433,12176 L 17642,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 17777,12176 L 17986,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 18120,12176 L 18329,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 18464,12176 L 18672,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 18807,12176 L 19016,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 19150,12176 L 19359,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 19494,12176 L 19703,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 19837,12176 L 20046,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 20181,12176 L 20390,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 20524,12176 L 20733,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 20868,12176 L 21076,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 21211,12176 L 21420,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 21555,12176 L 21763,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 21898,12176 L 22107,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 22241,12176 L 22450,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 22585,12176 L 22794,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 22928,12176 L 23137,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 23272,12176 L 23481,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 23615,12176 L 23824,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 23959,12176 L 24167,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 24302,12176 L 24511,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 24645,12176 L 24854,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 24989,12176 L 25198,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 25332,12176 L 25541,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 25676,12176 L 25885,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 26019,12176 L 26228,12176"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="106" stroke-linejoin="round" d="M 26363,12176 L 26400,12176"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id24">
+ <rect class="BoundingBox" stroke="none" fill="none" x="16366" y="9381" width="1273" height="3"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 16367,9382 L 17637,9382"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id25">
+ <rect class="BoundingBox" stroke="none" fill="none" x="22462" y="12936" width="3306" height="2417"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 24114,12937 C 23213,12937 22463,13074 22463,13238 L 22463,15049 C 22463,15213 23213,15351 24114,15351 25015,15351 25766,15213 25766,15049 L 25766,13238 C 25766,13074 25015,12937 24114,12937 L 24114,12937 Z M 22463,12937 L 22463,12937 Z M 25766,15351 L 25766,15351 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 24114,12937 C 23213,12937 22463,13074 22463,13238 L 22463,15049 C 22463,15213 23213,15351 24114,15351 25015,15351 25766,15213 25766,15049 L 25766,13238 C 25766,13074 25015,12937 24114,12937 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 22463,12937 L 22463,12937 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 25766,15351 L 25766,15351 Z"/>
+ <path fill="rgb(165,195,226)" stroke="none" d="M 24114,12937 C 23213,12937 22463,13074 22463,13238 22463,13403 23213,13540 24114,13540 25015,13540 25766,13403 25766,13238 25766,13074 25015,12937 24114,12937 L 24114,12937 Z M 22463,12937 L 22463,12937 Z M 25766,15351 L 25766,15351 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 24114,12937 C 23213,12937 22463,13074 22463,13238 22463,13403 23213,13540 24114,13540 25015,13540 25766,13403 25766,13238 25766,13074 25015,12937 24114,12937 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 22463,12937 L 22463,12937 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 25766,15351 L 25766,15351 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="23162" y="14466"><tspan fill="rgb(0,0,0)" stroke="none">git repos</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id26">
+ <rect class="BoundingBox" stroke="none" fill="none" x="23986" y="10270" width="3" height="2670"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 23987,10271 L 23987,12938"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id27">
+ <rect class="BoundingBox" stroke="none" fill="none" x="14588" y="4301" width="3" height="1273"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 14589,4302 L 14589,5572"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id28">
+ <rect class="BoundingBox" stroke="none" fill="none" x="9381" y="4809" width="3432" height="3686"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 12811,8493 L 9382,4810"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id29">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7984" y="4809" width="3" height="3813"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 7985,8620 L 7985,4810"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id30">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7350" y="3666" width="2541" height="636"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="7956" y="4105"><tspan fill="rgb(0,0,0)" stroke="none">CLI user</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id31">
+ <rect class="BoundingBox" stroke="none" fill="none" x="13319" y="3539" width="2541" height="636"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="13831" y="3978"><tspan fill="rgb(0,0,0)" stroke="none">Web user</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id32">
+ <rect class="BoundingBox" stroke="none" fill="none" x="5445" y="10651" width="2541" height="636"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="5502" y="11090"><tspan fill="rgb(0,0,0)" stroke="none">Storage access</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id33">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1254" y="10524" width="2541" height="1398"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1783" y="10950"><tspan fill="rgb(0,0,0)" stroke="none">External </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1957" y="11344"><tspan fill="rgb(0,0,0)" stroke="none">facing </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1824" y="11738"><tspan fill="rgb(0,0,0)" stroke="none">services</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id34">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1123" y="12556" width="2811" height="1398"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1889" y="12982"><tspan fill="rgb(0,0,0)" stroke="none">Internal</tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1756" y="13376"><tspan fill="rgb(0,0,0)" stroke="none">Services </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="700"><tspan class="TextPosition" x="1106" y="13770"><tspan fill="rgb(0,0,0)" stroke="none">(private network)</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id35">
+ <rect class="BoundingBox" stroke="none" fill="none" x="17636" y="10525" width="3938" height="1017"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="17792" y="10957"><tspan fill="rgb(0,0,0)" stroke="none">Publish change events </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="18294" y="11351"><tspan fill="rgb(0,0,0)" stroke="none">over websockets</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id36">
+ <rect class="BoundingBox" stroke="none" fill="none" x="11508" y="10271" width="2855" height="1525"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="11492" y="10760"><tspan fill="rgb(0,0,0)" stroke="none">Storage metadata,</tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="11801" y="11154"><tspan fill="rgb(0,0,0)" stroke="none">Compute jobs,</tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="11977" y="11548"><tspan fill="rgb(0,0,0)" stroke="none">Permissions</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id37">
+ <rect class="BoundingBox" stroke="none" fill="none" x="5444" y="19033" width="5462" height="636"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="5526" y="19472"><tspan fill="rgb(0,0,0)" stroke="none">Content-addressed object storage</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id38">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12811" y="19033" width="4065" height="636"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-weight="400"><tspan class="TextPosition" x="13074" y="19472"><tspan fill="rgb(0,0,0)" stroke="none">Elastic compute nodes</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id39">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1000" y="1127" width="5843" height="2033"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="1190" y="2008"><tspan fill="rgb(0,0,0)" stroke="none">An Arvados cluster </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="1595" y="2719"><tspan fill="rgb(0,0,0)" stroke="none">From 30000 feet</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id40">
+ <rect class="BoundingBox" stroke="none" fill="none" x="19795" y="15985" width="3814" height="3306"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 21701,15986 C 20662,15986 19796,16173 19796,16398 L 19796,18876 C 19796,19101 20662,19289 21701,19289 22740,19289 23607,19101 23607,18876 L 23607,16398 C 23607,16173 22740,15986 21701,15986 L 21701,15986 Z M 19796,15986 L 19796,15986 Z M 23607,19289 L 23607,19289 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 21701,15986 C 20662,15986 19796,16173 19796,16398 L 19796,18876 C 19796,19101 20662,19289 21701,19289 22740,19289 23607,19101 23607,18876 L 23607,16398 C 23607,16173 22740,15986 21701,15986 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 19796,15986 L 19796,15986 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 23607,19289 L 23607,19289 Z"/>
+ <path fill="rgb(165,195,226)" stroke="none" d="M 21701,15986 C 20662,15986 19796,16173 19796,16398 19796,16624 20662,16811 21701,16811 22740,16811 23607,16624 23607,16398 23607,16173 22740,15986 21701,15986 L 21701,15986 Z M 19796,15986 L 19796,15986 Z M 23607,19289 L 23607,19289 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 21701,15986 C 20662,15986 19796,16173 19796,16398 19796,16624 20662,16811 21701,16811 22740,16811 23607,16624 23607,16398 23607,16173 22740,15986 21701,15986 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 19796,15986 L 19796,15986 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 23607,19289 L 23607,19289 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="20377" y="18015"><tspan fill="rgb(0,0,0)" stroke="none">Postgres db</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.LineShape">
+ <g id="id41">
+ <rect class="BoundingBox" stroke="none" fill="none" x="10016" y="9381" width="2924" height="3"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 10017,9382 L 12938,9382"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id42">
+ <rect class="BoundingBox" stroke="none" fill="none" x="12810" y="8491" width="3559" height="1654"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 14589,10143 L 12811,10143 12811,8492 16367,8492 16367,10143 14589,10143 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 14589,10143 L 12811,10143 12811,8492 16367,8492 16367,10143 14589,10143 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="14189" y="9489"><tspan fill="rgb(0,0,0)" stroke="none">API</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+</svg>
\ No newline at end of file
</div>
<div class="col-sm-6" style="border-left: solid; border-width: 1px">
- <p><strong>Quickstart</strong>
- <p>
- Try any pipeline from the <a href="https://cloud.curoverse.com/projects/public">list of public pipelines</a>. For instance, the <a href="http://curover.se/pathomap">Pathomap Pipeline</a> links to these <a href="https://dev.arvados.org/projects/arvados/wiki/pathomap_tutorial/">step-by-step instructions</a> for trying Arvados out right in your browser using Curoverse's <a href="http://lp.curoverse.com/beta-signup/">public Arvados instance</a>.
- </p>
- <!--<p>-->
- <!--<ol>-->
- <!--<li>-->
- <!--Go to <a href="{{site.arvados_workbench_host}}/" target="_blank">{{site.arvados_workbench_host}}/</a>-->
- <!--</li><li>-->
- <!--Register with any Google account-->
- <!--</li><li>-->
- <!--Follow the Getting Started guide-->
- <!--<br>-->
- <!--<em>Tip: Don't see the guide? You can find it by clicking (in the upper-right corner) <span class="fa fa-lg fa-question-circle"></span> > Getting Started)</em>-->
- <!--</li>-->
- <!--</ol>-->
- <!--</p>-->
- <p><strong>
- Pipeline Developer Quickstart
- </strong></p>
- <p>
- Want to port your pipeline to Arvados? Check out the step-by-step <a href="https://dev.arvados.org/projects/arvados/wiki/Port_a_Pipeline">Port-a-Pipeline</a> guide on the Arvados wiki.
- </p>
<p><strong>More in-depth guides
</strong></p>
<!--<p>-->
<p>
<a href="{{ site.baseurl }}/sdk/index.html">SDK Reference</a> — Details about the accessing Arvados from various programming languages.
</p>
+ <p>
+ <a href="{{ site.baseurl }}/architecture/index.html">Arvados Architecture</a> — Details about the the Arvados components and architecture.
+ </p>
<p>
<a href="{{ site.baseurl }}/api/index.html">API Reference</a> — Details about the the Arvados REST API.
</p>
<p>
- <a href="{{ site.baseurl }}/install/index.html">Install Guide</a> — How to install Arvados on a cloud platform.
+ <a href="{{ site.baseurl }}/admin/index.html">Admin Guide</a> — Details about administering an Arvados cluster.
+ </p>
+ <p>
+ <a href="{{ site.baseurl }}/install/index.html">Install Guide</a> — How to install Arvados.
</p>
</div>
</div>
---
layout: default
-navsection: installguide
-title: Cheat Sheet
+navsection: admin
+title: User management
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
<pre><code>#!/bin/sh
exec 2>&1
-exec GOGC=10 keepstore \
+GOGC=10 exec keepstore \
-enforce-permissions=true \
-blob-signing-key-file=<span class="userinput">/etc/keepstore/blob-signing.key</span> \
-max-buffers=<span class="userinput">100</span> \
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+h2. Supported Cloud and HPC platforms
+
+Arvados can run in a variety of configurations. For compute scheduling, Arvados supports HPC clusters using @slurm@, and supports elastic cloud computing on AWS, Google and Azure. For storage, Arvados can store blocks on regular file systems such as ext4 or xfs, on network file systems such as GPFS, or object storage such as Azure blob storage, Amazon S3, and other object storage that supports the S3 API including Google Cloud Storage and Ceph.
+
h2. Hardware (or virtual machines)
This guide assumes you have seven systems available in the same network subnet:
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.3* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
+Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.4* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
* Install PostgreSQL on the the same host as the SSO server, and dedicate that install to hosting the SSO database. This provides the best security for the SSO server, because the database does not have to accept any client connections over the network. Typical load on the SSO server is light enough that deploying both it and its database on the same host does not compromise performance.
* If you want to provide the most scalability for your Arvados cluster, install PostgreSQL for the API server on a dedicated host. This gives you the most flexibility to avoid resource contention, and tune performance separately for the API server and its database. If performance is less of a concern for your installation, you can install PostgreSQL on the API server host directly, as with the SSO server.
Find the section for your distribution below, and follow it to install PostgreSQL on each host where you will deploy it. Then follow the steps in the later section(s) to set up PostgreSQL for the Arvados service(s) that need it.
-h2. Install PostgreSQL 9.3+
+It is important to make sure that autovacuum is enabled for the PostgreSQL database that backs the API server. Autovacuum is enabled by default since PostgreSQL 8.3.
-The API server requires at least version *9.3* of PostgreSQL.
+h2. Install PostgreSQL 9.4+
+
+The API server requires at least version *9.4* of PostgreSQL.
h3(#centos7). CentOS 7
{% assign rh_version = "7" %}
h3(#debian). Debian or Ubuntu
-Debian 8 (Jessie) and Ubuntu 14.04 (Trusty) and later versions include a sufficiently recent version of Postgres.
+Debian 8 (Jessie) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
+
+Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/
# Install PostgreSQL:
<notextile><pre>~$ <span class="userinput">sudo apt-get install postgresql</span></pre></notextile>
---
layout: default
-navsection: installguide
-title: Migrating Docker images
+navsection: admin
+title: Migrating from Docker 1.9
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
Use @go get git.curoverse.com/arvados.git/sdk/go/arvadosclient@. The go tools will fetch the relevant code and dependencies for you.
-<notextile>{% code 'example_sdk_go_imports' as go %}</notextile>
+{% codeblock as go %}
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+)
+{% endcodeblock %}
If you need pre-release client code, you can use the latest version from the repo by following "these instructions.":https://dev.arvados.org/projects/arvados/wiki/Go#Using-Go-with-Arvados
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'pipeline_deprecation_notice' %}
+
Several utility libraries are included with Arvados. They are intended to make it quicker and easier to write your own crunch scripts.
* "Python SDK extras":#pythonsdk
...
}
</pre>
-
-
Arvados applications can subscribe to a live event stream from the database. Events are described in the "Log resource.":{{site.baseurl}}/api/methods/logs.html
-<notextile>
-{% code 'events_py' as python %}
-</notextile>
+{% codeblock as python %}
+#!/usr/bin/env python
+
+import arvados
+import arvados.events
+
+# 'ev' is a dict containing the log table record describing the change.
+def on_message(ev):
+ if ev.get("event_type") == "create" and ev.get("object_kind") == "arvados#collection":
+ print "A new collection was created: %s" % ev["object_uuid"]
+
+api = arvados.api("v1")
+ws = arvados.events.subscribe(api, [], on_message)
+ws.run_forever()
+{% endcodeblock %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-_If you are new to Arvados, please try the Quickstart on <a href="http://doc.arvados.org">the documentation homepage</a> instead of this detailed User Guide._
-
This guide provides a reference for using Arvados to solve big data bioinformatics problems, including:
* Robust storage of very large files, such as whole genome sequences, using the "Arvados Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html content-addressable cluster file system.
* Running compute-intensive genomic analysis pipelines, such as alignment and variant calls using the "Arvados Crunch":{{site.baseurl}}/user/tutorials/intro-crunch.html cluster compute engine.
-* Storing and querying metadata about genome sequence files, such as human subjects and their phenotypic traits using the "Arvados Metadata Database.":{{site.baseurl}}/user/topics/tutorial-trait-search.html
* Accessing, organizing, and sharing data, pipelines and results using the "Arvados Workbench":{{site.baseurl}}/user/getting_started/workbench.html web application.
The examples in this guide use the public Arvados instance located at <a href="{{site.arvados_workbench_host}}/" target="_blank">{{site.arvados_workbench_host}}</a>. If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
---
layout: default
-navsection: userguide
-title: "Using arvados-sync-groups"
+navsection: admin
+title: "Synchronizing external groups"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The @arvados-sync-groups@ tool allows to synchronize remote groups into Arvados from an external source.
+The @arvados-sync-groups@ tool allows to synchronize groups in Arvados from an external source.
h1. Using arvados-sync-groups
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin_warning' %}
+The humans, specimens and traits tables are deprecated and will be removed in a future release. The recommended way to store and search on user-defined metadata is using the "properties" field of Arvados resources.
+{% include 'notebox_end' %}
+
This tutorial introduces the Arvados Metadata Database. The Metadata Database stores information about files in Keep. This example will use the Python SDK to find public WGS (Whole Genome Sequencing) data for people who have reported a certain medical condition.
{% include 'tutorial_expectations' %}
In the tutorial examples, three angle brackets (>>>) will be used to denote code to enter at the interactive Python prompt.
-Start by running Python.
+Start by running Python.
<notextile>
<pre><code>~$ <span class="userinput">python</span>
-Python 2.7.3 (default, Jan 2 2013, 13:56:14)
+Python 2.7.3 (default, Jan 2 2013, 13:56:14)
[GCC 4.7.2] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
</code></pre>
</notextile>
-
+
If everything is set up correctly, you will be able to import the arvados SDK.
notextile. <pre><code>>>> <span class="userinput">import arvados</span></pre></code>
job_uuid = job[collection_uuid]['uuid']
job_output = arvados.api('v1').jobs().get(uuid=job_uuid).execute()['output']
output_files = arvados.api('v1').collections().get(uuid=job_output).execute()['files']
- # Test the output size. If greater than zero, that means 'grep' found the variant
+ # Test the output size. If greater than zero, that means 'grep' found the variant
if output_files[0][2] > 0:
print("%s has variant rs1126809" % (pgpid[collection_uuid]))
else:
### Installation
-```install.packages("ArvadosR", repos=c("http://r.arvados.org", getOption("repos")["CRAN"]), dependencies=TRUE)```
+```{r include=FALSE}
+knitr::opts_chunk$set(eval=FALSE)
+```
+
+```{r}
+install.packages("ArvadosR", repos=c("http://r.arvados.org", getOption("repos")["CRAN"]), dependencies=TRUE)
+```
Note: on Linux, you may have to install supporting packages.
On Centos 7, this is:
-```yum install libxml2-devel openssl-devel curl-devel```
+```{bash}
+yum install libxml2-devel openssl-devel curl-devel
+```
On Debian, this is:
-```apt-get install build-essential libxml2-dev libssl-dev libcurl4-gnutls-dev```
+```{bash}
+apt-get install build-essential libxml2-dev libssl-dev libcurl4-gnutls-dev
+```
### Usage
#### Initializing API
-```{r include=FALSE}
-knitr::opts_chunk$set(eval = FALSE)
-```
-
* Load Library and Initialize API:
- ```{r}
- library('ArvadosR')
- # use environment variables ARVADOS_API_TOKEN and ARVADOS_API_HOST
- arv <- Arvados$new()
+```{r}
+library('ArvadosR')
+# use environment variables ARVADOS_API_TOKEN and ARVADOS_API_HOST
+arv <- Arvados$new()
- # provide them explicitly
- arv <- Arvados$new("your Arvados token", "example.arvadosapi.com")
- ```
+# provide them explicitly
+arv <- Arvados$new("your Arvados token", "example.arvadosapi.com")
+```
- Optionally, add numRetries parameter to specify number of times to retry failed service requests.
- Default is 0.
+Optionally, add numRetries parameter to specify number of times to retry failed service requests.
+Default is 0.
- ```{r}
- arv <- Arvados$new("your Arvados token", "example.arvadosapi.com", numRetries = 3)
- ```
+```{r}
+arv <- Arvados$new("your Arvados token", "example.arvadosapi.com", numRetries = 3)
+```
- This parameter can be set at any time using setNumRetries
+This parameter can be set at any time using setNumRetries
- ```{r}
- arv$setNumRetries(5)
- ```
+```{r}
+arv$setNumRetries(5)
+```
#### Working with collections
* Get a collection:
- ```{r}
- collection <- arv$collections.get("uuid")
- ```
+```{r}
+collection <- arv$collections.get("uuid")
+```
* List collections:
- ```{r}
- # offset of 0 and default limit of 100
- collectionList <- arv$collections.list(list(list("name", "like", "Test%")))
+```{r}
+# offset of 0 and default limit of 100
+collectionList <- arv$collections.list(list(list("name", "like", "Test%")))
- collectionList <- arv$collections.list(list(list("name", "like", "Test%")), limit = 10, offset = 2)
- ```
+collectionList <- arv$collections.list(list(list("name", "like", "Test%")), limit = 10, offset = 2)
+```
- ```{r}
- # count of total number of items (may be more than returned due to paging)
- collectionList$items_available
+```{r}
+# count of total number of items (may be more than returned due to paging)
+collectionList$items_available
- # items which match the filter criteria
- collectionList$items
- ```
+# items which match the filter criteria
+collectionList$items
+```
* List all collections even if the number of items is greater than maximum API limit:
- ```{r}
- collectionList <- listAll(arv$collections.list, list(list("name", "like", "Test%")))
- ```
+```{r}
+collectionList <- listAll(arv$collections.list, list(list("name", "like", "Test%")))
+```
* Delete a collection:
- ```{r}
- deletedCollection <- arv$collections.delete("uuid")
- ```
+```{r}
+deletedCollection <- arv$collections.delete("uuid")
+```
* Update a collection's metadata:
- ```{r}
- updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), "uuid")
- ```
+```{r}
+updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), "uuid")
+```
* Create collection:
- ```{r}
- newCollection <- arv$collections.create(list(name = "Example", description = "This is a test collection"))
- ```
+```{r}
+newCollection <- arv$collections.create(list(name = "Example", description = "This is a test collection"))
+```
#### Manipulating collection content
* Create collection object:
- ```{r}
- collection <- Collection$new(arv, "uuid")
- ```
+```{r}
+collection <- Collection$new(arv, "uuid")
+```
* Get list of files:
- ```{r}
- files <- collection$getFileListing()
- ```
+```{r}
+files <- collection$getFileListing()
+```
* Get ArvadosFile or Subcollection from internal tree-like structure:
- ```{r}
- arvadosFile <- collection$get("location/to/my/file.cpp")
- ```
+```{r}
+arvadosFile <- collection$get("location/to/my/file.cpp")
+```
or
- ```{r}
- arvadosSubcollection <- collection$get("location/to/my/directory/")
- ```
+```{r}
+arvadosSubcollection <- collection$get("location/to/my/directory/")
+```
* Read a table:
- ```{r}
- arvadosFile <- collection$get("myinput.txt")
- arvConnection <- arvadosFile$connection("r")
- mytable <- read.table(arvConnection)
- ```
+```{r}
+arvadosFile <- collection$get("myinput.txt")
+arvConnection <- arvadosFile$connection("r")
+mytable <- read.table(arvConnection)
+```
* Write a table:
- ```{r}
- arvadosFile <- collection$create("myoutput.txt")
- arvConnection <- arvadosFile$connection("w")
- write.table(mytable, arvConnection)
- arvadosFile$flush()
- ```
+```{r}
+arvadosFile <- collection$create("myoutput.txt")
+arvConnection <- arvadosFile$connection("w")
+write.table(mytable, arvConnection)
+arvadosFile$flush()
+```
* Write to existing file (override current content of the file):
- ```{r}
- arvadosFile <- collection$get("location/to/my/file.cpp")
- arvadosFile$write("This is new file content")
- ```
+```{r}
+arvadosFile <- collection$get("location/to/my/file.cpp")
+arvadosFile$write("This is new file content")
+```
* Read whole file or just a portion of it:
- ```{r}
- fileContent <- arvadosFile$read()
- fileContent <- arvadosFile$read("text")
- fileContent <- arvadosFile$read("raw", offset = 1024, length = 512)
- ```
+```{r}
+fileContent <- arvadosFile$read()
+fileContent <- arvadosFile$read("text")
+fileContent <- arvadosFile$read("raw", offset = 1024, length = 512)
+```
* Get ArvadosFile or Subcollection size:
- ```{r}
- size <- arvadosFile$getSizeInBytes()
- ```
+```{r}
+size <- arvadosFile$getSizeInBytes()
+```
or
- ```{r}
- size <- arvadosSubcollection$getSizeInBytes()
- ```
+```{r}
+size <- arvadosSubcollection$getSizeInBytes()
+```
* Create new file in a collection:
- ```{r}
- collection$create(fileNames, optionalRelativePath)
- ```
+```{r}
+collection$create(fileNames, optionalRelativePath)
+```
Example:
- ```{r}
- mainFile <- collection$create("main.cpp", "cpp/src/")
- fileList <- collection$create(c("main.cpp", lib.dll), "cpp/src/")
- ```
+```{r}
+mainFile <- collection$create("main.cpp", "cpp/src/")
+fileList <- collection$create(c("main.cpp", lib.dll), "cpp/src/")
+```
* Add existing ArvadosFile or Subcollection to a collection:
- ```{r}
- folder <- Subcollection$new("src")
- file <- ArvadosFile$new("main.cpp")
- folder$add(file)
- ```
+```{r}
+folder <- Subcollection$new("src")
+file <- ArvadosFile$new("main.cpp")
+folder$add(file)
+```
- ```{r}
- collection$add(folder, "cpp")
- ```
+```{r}
+collection$add(folder, "cpp")
+```
- This examples will add file "main.cpp" in "./cpp/src/" folder if folder exists.
- If subcollection contains more files or folders they will be added recursively.
+This examples will add file "main.cpp" in "./cpp/src/" folder if folder exists.
+If subcollection contains more files or folders they will be added recursively.
* Delete file from a collection:
- ```{r}
- collection$remove("location/to/my/file.cpp")
- ```
+```{r}
+collection$remove("location/to/my/file.cpp")
+```
- You can remove both Subcollection and ArvadosFile.
- If subcollection contains more files or folders they will be removed recursively.
+You can remove both Subcollection and ArvadosFile.
+If subcollection contains more files or folders they will be removed recursively.
- You can also remove multiple files at once:
+You can also remove multiple files at once:
- ```{r}
- collection$remove(c("path/to/my/file.cpp", "path/to/other/file.cpp"))
- ```
+```{r}
+collection$remove(c("path/to/my/file.cpp", "path/to/other/file.cpp"))
+```
* Delete file or folder from a Subcollection:
- ```{r}
- subcollection <- collection$get("mySubcollection/")
- subcollection$remove("fileInsideSubcollection.exe")
- subcollection$remove("folderInsideSubcollection/")
- ```
+```{r}
+subcollection <- collection$get("mySubcollection/")
+subcollection$remove("fileInsideSubcollection.exe")
+subcollection$remove("folderInsideSubcollection/")
+```
* Move file or folder inside collection:
- Directley from collection
+Directley from collection
- ```{r}
- collection$move("folder/file.cpp", "file.cpp")
- ```
+```{r}
+collection$move("folder/file.cpp", "file.cpp")
+```
- Or from file
+Or from file
- ```{r}
- file <- collection$get("location/to/my/file.cpp")
- file$move("newDestination/file.cpp")
- ```
+```{r}
+file <- collection$get("location/to/my/file.cpp")
+file$move("newDestination/file.cpp")
+```
- Or from subcollection
+Or from subcollection
- ```{r}
- subcollection <- collection$get("location/to/folder")
- subcollection$move("newDestination/folder")
- ```
+```{r}
+subcollection <- collection$get("location/to/folder")
+subcollection$move("newDestination/folder")
+```
- Make sure to include new file name in destination.
- In second example file$move("newDestination/") will not work.
+Make sure to include new file name in destination.
+In second example file$move("newDestination/") will not work.
#### Working with Aravdos projects
* Get a project:
- ```{r}
- project <- arv$projects.get("uuid")
- ```
+```{r}
+project <- arv$projects.get("uuid")
+```
* List projects:
- ```{r}
- # list subprojects of a project
- projects <- arv$projects.list(list(list("owner_uuid", "=", "aaaaa-j7d0g-ccccccccccccccc")))
+```{r}
+list subprojects of a project
+projects <- arv$projects.list(list(list("owner_uuid", "=", "aaaaa-j7d0g-ccccccccccccccc")))
- # list projects which have names beginning with Example
- examples <- arv$projects.list(list(list("name","like","Example%")))
- ```
+list projects which have names beginning with Example
+examples <- arv$projects.list(list(list("name","like","Example%")))
+```
* List all projects even if the number of items is greater than maximum API limit:
- ```{r}
- projects <- listAll(arv$projects.list, list(list("name","like","Example%")))
- ```
+```{r}
+projects <- listAll(arv$projects.list, list(list("name","like","Example%")))
+```
* Delete a project:
- ```{r}
- deletedProject <- arv$projects.delete("uuid")
- ```
+```{r}
+deletedProject <- arv$projects.delete("uuid")
+```
* Update project:
- ```{r}
- updatedProject <- arv$projects.update(list(name = "new_name", description = "new description"), "uuid")
- ```
+```{r}
+updatedProject <- arv$projects.update(list(name = "new_name", description = "new description"), "uuid")
+```
* Create project:
- ```{r}
- newProject <- arv$projects.update(list(name = "project_name", description = "project description"))
- ```
+```{r}
+newProject <- arv$projects.update(list(name = "project_name", description = "project description"))
+```
#### Help
* View help page of Arvados classes by puting ? before class name:
- ```{r}
- ?Arvados
- ?Collection
- ?Subcollection
- ?ArvadosFile
- ```
+```{r}
+?Arvados
+?Collection
+?Subcollection
+?ArvadosFile
+```
* View help page of any method defined in Arvados class by puting ? before method name:
- ```{r}
- ?collections.update
- ?jobs.get
- ```
+```{r}
+?collections.update
+?jobs.get
+```
### Building the ArvadosR package
- ```
- cd arvados/sdk && R CMD build R
- ```
+```{bash}
+cd arvados/sdk && R CMD build R
+```
This will create a tarball of the ArvadosR package in the current directory.
self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries)
+ self.fetcher_constructor = partial(CollectionFetcher,
+ api_client=self.api,
+ fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
+ num_retries=self.num_retries)
+
self.work_api = None
expected_api = ["jobs", "containers"]
for api in expected_api:
def arv_make_tool(self, toolpath_object, **kwargs):
kwargs["work_api"] = self.work_api
- kwargs["fetcher_constructor"] = partial(CollectionFetcher,
- api_client=self.api,
- fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
- num_retries=self.num_retries)
+ kwargs["fetcher_constructor"] = self.fetcher_constructor
kwargs["resolver"] = partial(collectionResolver, self.api, num_retries=self.num_retries)
if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
return ArvadosCommandTool(self, toolpath_object, **kwargs)
with self.workflow_eval_lock:
self.processes[container.uuid] = container
- def process_done(self, uuid):
+ def process_done(self, uuid, record):
with self.workflow_eval_lock:
- if uuid in self.processes:
- del self.processes[uuid]
+ j = self.processes[uuid]
+ logger.info("%s %s is %s", self.label(j), uuid, record["state"])
+ self.task_queue.add(partial(j.done, record))
+ del self.processes[uuid]
def wrapped_callback(self, cb, obj, st):
with self.workflow_eval_lock:
j.update_pipeline_component(event["properties"]["new_attributes"])
logger.info("%s %s is Running", self.label(j), uuid)
elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled", "Final"):
- with self.workflow_eval_lock:
- j = self.processes[uuid]
- self.task_queue.add(partial(j.done, event["properties"]["new_attributes"]))
- logger.info("%s %s is %s", self.label(j), uuid, event["properties"]["new_attributes"]["state"])
+ self.process_done(uuid, event["properties"]["new_attributes"])
def label(self, obj):
return "[%s %s]" % (self.work_api[0:-1], obj.name)
if response["state"] == "Final":
logger.info("%s reused container %s", self.arvrunner.label(self), response["container_uuid"])
- self.done(response)
else:
logger.info("%s %s state is %s", self.arvrunner.label(self), response["uuid"], response["state"])
except Exception as e:
processStatus = "permanentFail"
finally:
self.output_callback(outputs, processStatus)
- self.arvrunner.process_done(record["uuid"])
class RunnerContainer(Runner):
logger.info("%s submitted container %s", self.arvrunner.label(self), response["uuid"])
- if response["state"] == "Final":
- self.done(response)
-
def done(self, record):
try:
container = self.arvrunner.api.containers().get(
self.arvrunner.output_callback({}, "permanentFail")
else:
super(RunnerContainer, self).done(container)
- finally:
- self.arvrunner.process_done(record["uuid"])
logger.info("Creating read permission on job %s: %s",
response["uuid"],
e)
-
- with Perf(metrics, "done %s" % self.name):
- self.done(response)
else:
logger.info("%s %s is %s", self.arvrunner.label(self), response["uuid"], response["state"])
except Exception as e:
processStatus = "permanentFail"
finally:
self.output_callback(outputs, processStatus)
- self.arvrunner.process_done(record["uuid"])
class RunnerJob(Runner):
self.uuid = job["uuid"]
self.arvrunner.process_submitted(self)
- if job["state"] in ("Complete", "Failed", "Cancelled"):
- self.done(job)
-
class RunnerTemplate(object):
"""An Arvados pipeline template that invokes a CWL workflow."""
def makePathMapper(self, reffiles, stagedir, **kwargs):
# type: (List[Any], unicode, **Any) -> PathMapper
if self.work_api == "containers":
- return ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+ return ArvPathMapper(self.arvrunner, reffiles+kwargs.get("extra_reffiles", []), kwargs["basedir"],
"/keep/%s",
"/keep/%s/%s",
**kwargs)
**kwargs)
def job(self, joborder, output_callback, **kwargs):
+
+ # Workaround for #13365
+ builderargs = kwargs.copy()
+ builderargs["toplevel"] = True
+ builder = self._init_job(joborder, **builderargs)
+ joborder = builder.job
+
if self.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
if dockerReq and dockerReq.get("dockerOutputDirectory"):
self.wf_pdh = None
self.dynamic_resource_req = []
self.static_resource_req = []
+ self.wf_reffiles = []
def job(self, joborder, output_callback, **kwargs):
kwargs["work_api"] = self.work_api
uri,
False)
+ # Discover files/directories referenced by the
+ # workflow (mainly "default" values)
+ visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
+
+
if self.dynamic_resource_req:
builder = Builder()
builder.job = joborder
joborder_keepmount = copy.deepcopy(joborder)
reffiles = []
- visit_class(joborder_keepmount, ("File", "Directory"), lambda x: reffiles.append(x))
+ visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
- mapper = ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+ mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, kwargs["basedir"],
"/keep/%s",
"/keep/%s/%s",
**kwargs)
+ # For containers API, we need to make sure any extra
+ # referenced files (ie referenced by the workflow but
+ # not in the inputs) are included in the mounts.
+ kwargs["extra_reffiles"] = copy.deepcopy(self.wf_reffiles)
+
def keepmount(obj):
remove_redundant_fields(obj)
with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
import arvados.arvfile
import arvados.errors
+from googleapiclient.errors import HttpError
+
from schema_salad.ref_resolver import DefaultFetcher
logger = logging.getLogger('arvados.cwl-runner')
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
def exists(self, fn):
- collection, rest = self.get_collection(fn)
+ try:
+ collection, rest = self.get_collection(fn)
+ except HttpError as err:
+ if err.resp.status == 404:
+ return False
+ else:
+ raise
if collection is not None:
if rest:
return collection.exists(rest)
elif obj["location"].startswith("_:") and "contents" in obj:
with c.open(path + "/" + obj["basename"], "w") as f:
f.write(obj["contents"].encode("utf-8"))
+ remap.append((obj["location"], path + "/" + obj["basename"]))
else:
raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
import logging
import json
import subprocess
+from collections import namedtuple
from StringIO import StringIO
-from schema_salad.sourceline import SourceLine
+from schema_salad.sourceline import SourceLine, cmap
from cwltool.command_line_tool import CommandLineTool
import cwltool.workflow
if obj.get("location", "").startswith("_:"):
del obj["location"]
+
def remove_redundant_fields(obj):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
+
def find_defaults(d, op):
if isinstance(d, list):
for i in d:
for i in d.itervalues():
find_defaults(i, op)
+def setSecondary(t, fileobj, discovered):
+ if isinstance(fileobj, dict) and fileobj.get("class") == "File":
+ if "secondaryFiles" not in fileobj:
+ fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
+ if discovered is not None:
+ discovered[fileobj["location"]] = fileobj["secondaryFiles"]
+ elif isinstance(fileobj, list):
+ for e in fileobj:
+ setSecondary(t, e, discovered)
+
+def discover_secondary_files(inputs, job_order, discovered=None):
+ for t in inputs:
+ if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
+ setSecondary(t, job_order[shortname(t["id"])], discovered)
+
+
def upload_dependencies(arvrunner, name, document_loader,
- workflowobj, uri, loadref_run, include_primary=True):
+ workflowobj, uri, loadref_run,
+ include_primary=True, discovered_secondaryfiles=None):
"""Upload the dependencies of the workflowobj document to Keep.
Returns a pathmapper object mapping local paths to keep references. Also
for s in workflowobj["$schemas"]:
sc.append({"class": "File", "location": s})
- def capture_default(obj):
+ def visit_default(obj):
remove = [False]
- def add_default(f):
+ def ensure_default_location(f):
if "location" not in f and "path" in f:
f["location"] = f["path"]
del f["path"]
if "location" in f and not arvrunner.fs_access.exists(f["location"]):
- # Remove from sc
+ # Doesn't exist, remove from list of dependencies to upload
sc[:] = [x for x in sc if x["location"] != f["location"]]
# Delete "default" from workflowobj
remove[0] = True
- visit_class(obj["default"], ("File", "Directory"), add_default)
+ visit_class(obj["default"], ("File", "Directory"), ensure_default_location)
if remove[0]:
del obj["default"]
- find_defaults(workflowobj, capture_default)
+ find_defaults(workflowobj, visit_default)
+
+ discovered = {}
+ def discover_default_secondary_files(obj):
+ discover_secondary_files(obj["inputs"],
+ {shortname(t["id"]): t["default"] for t in obj["inputs"] if "default" in t},
+ discovered)
+
+ visit_class(workflowobj, ("CommandLineTool", "Workflow"), discover_default_secondary_files)
+
+ for d in discovered:
+ sc.extend(discovered[d])
mapper = ArvPathMapper(arvrunner, sc, "",
"keep:%s",
def setloc(p):
if "location" in p and (not p["location"].startswith("_:")) and (not p["location"].startswith("keep:")):
p["location"] = mapper.mapper(p["location"]).resolved
- adjustFileObjs(workflowobj, setloc)
- adjustDirObjs(workflowobj, setloc)
+
+ visit_class(workflowobj, ("File", "Directory"), setloc)
+ visit_class(discovered, ("File", "Directory"), setloc)
+
+ if discovered_secondaryfiles is not None:
+ for d in discovered:
+ discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
if "$schemas" in workflowobj:
sch = []
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
+
def packed_workflow(arvrunner, tool, merged_map):
"""Create a packed workflow.
packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
tool.tool["id"], tool.metadata, rewrite_out=rewrites)
- rewrite_to_orig = {}
- for k,v in rewrites.items():
- rewrite_to_orig[v] = k
+ rewrite_to_orig = {v: k for k,v in rewrites.items()}
def visit(v, cur_id):
if isinstance(v, dict):
if v.get("class") in ("CommandLineTool", "Workflow"):
+ if "id" not in v:
+ raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
cur_id = rewrite_to_orig.get(v["id"], v["id"])
if "location" in v and not v["location"].startswith("keep:"):
- v["location"] = merged_map[cur_id][v["location"]]
+ v["location"] = merged_map[cur_id].resolved[v["location"]]
+ if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
+ v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
visit(packed, None)
return packed
+
def tag_git_version(packed):
if tool.tool["id"].startswith("file://"):
path = os.path.dirname(tool.tool["id"][7:])
packed["http://schema.org/version"] = githash
-def discover_secondary_files(inputs, job_order):
- for t in inputs:
- def setSecondary(fileobj):
- if isinstance(fileobj, dict) and fileobj.get("class") == "File":
- if "secondaryFiles" not in fileobj:
- fileobj["secondaryFiles"] = [{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]]
-
- if isinstance(fileobj, list):
- for e in fileobj:
- setSecondary(e)
-
- if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
- setSecondary(job_order[shortname(t["id"])])
-
def upload_job_order(arvrunner, name, tool, job_order):
"""Upload local files referenced in the input object and return updated input
object with 'location' updated to the proper keep references.
return job_order
+FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
+
def upload_workflow_deps(arvrunner, tool):
# Ensure that Docker images needed by this workflow are available
def upload_tool_deps(deptool):
if "id" in deptool:
+ discovered_secondaryfiles = {}
pm = upload_dependencies(arvrunner,
- "%s dependencies" % (shortname(deptool["id"])),
- document_loader,
- deptool,
- deptool["id"],
- False,
- include_primary=False)
+ "%s dependencies" % (shortname(deptool["id"])),
+ document_loader,
+ deptool,
+ deptool["id"],
+ False,
+ include_primary=False,
+ discovered_secondaryfiles=discovered_secondaryfiles)
document_loader.idx[deptool["id"]] = deptool
toolmap = {}
for k,v in pm.items():
toolmap[k] = v.resolved
- merged_map[deptool["id"]] = toolmap
+ merged_map[deptool["id"]] = FileUpdates(toolmap, discovered_secondaryfiles)
tool.visit(upload_tool_deps)
self.arvrunner.output_callback({}, "permanentFail")
else:
self.arvrunner.output_callback(outputs, processStatus)
- finally:
- self.arvrunner.process_done(record["uuid"])
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180403145700',
- 'schema-salad==2.6.20171201034858',
+ 'cwltool==1.0.20180508202931',
+ 'schema-salad==2.7.20180501211602',
'typing==3.5.3.0',
- 'ruamel.yaml==0.13.7',
- 'arvados-python-client>=1.1.4.20180418202329',
+ 'ruamel.yaml >=0.13.11, <0.15',
+ 'arvados-python-client>=1.1.4.20180507184611',
'setuptools',
- 'ciso8601 >=1.0.0, <=1.0.4',
+ 'ciso8601 >=1.0.6'
],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
if ! arv-get f225e6259bdd63bc7240599648dde9f1+97 > /dev/null ; then
arv-put --portable-data-hash hg19/*
fi
+if ! arv-get 4d8a70b1e63b2aad6984e40e338e2373+69 > /dev/null ; then
+ arv-put --portable-data-hash secondaryFiles/hello.txt*
+fi
exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum
}
tool: wf/secret_wf.cwl
doc: "Test secret input parameters"
+ tags: [ secrets ]
- job: null
output:
out: null
tool: wf/runin-reqs-wf4.cwl
doc: "RunInSingleContainer discovers static resource request in subworkflow steps"
+
+- job: secondaryFiles/inp3.yml
+ output: {}
+ tool: secondaryFiles/example1.cwl
+ doc: Discover secondaryFiles at runtime if they are in keep
+
+- job: null
+ output: {}
+ tool: secondaryFiles/example3.cwl
+ doc: Discover secondaryFiles on default values
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf1.cwl
+ doc: "Can have separate default parameters including directory and file inside same directory"
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf2.cwl
+ doc: "Can have a parameter default value that is a directory literal with a file literal"
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf3.cwl
+ doc: "Do not accept a directory literal without a basename"
+ should_fail: true
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf4.cwl
+ doc: default in embedded subworkflow missing 'id' field
+ should_fail: true
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf5.cwl
+ doc: default in embedded subworkflow
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf6.cwl
+ doc: default in RunInSingleContainer step
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf7.cwl
+ doc: workflow level default in RunInSingleContainer
--- /dev/null
+class: Workflow
+cwlVersion: v1.0
+inputs:
+ toplevel_input: File
+outputs: []
+steps:
+ step1:
+ in:
+ step_input: toplevel_input
+ out: []
+ run:
+ id: sub
+ class: CommandLineTool
+ inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ outputs: []
+ baseCommand: echo
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ default:
+ class: File
+ location: hello.txt
+outputs: []
+baseCommand: echo
--- /dev/null
+toplevel_input:
+ class: File
+ location: keep:4d8a70b1e63b2aad6984e40e338e2373+69/hello.txt
\ No newline at end of file
mock.MagicMock(return_value={'status': 403}),
'Permission denied')
j.run(enable_reuse=enable_reuse)
- j.output_callback.assert_called_with({}, 'success')
else:
assert not runner.api.links().create.called
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ inp1:
+ type: File
+ default:
+ class: File
+ location: inp1/hello.txt
+outputs: []
+arguments: [echo, $(inputs.inp1), $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ basename: inp2
+ listing:
+ - class: File
+ basename: "hello.txt"
+ contents: "hello world"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ listing:
+ - class: File
+ location: "inp1/hello.txt"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run:
+ class: CommandLineTool
+ inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ outputs: []
+ arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run:
+ id: stepid
+ class: CommandLineTool
+ inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ outputs: []
+ arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir6a.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in:
+ inp2: inp2
+ out: []
+ run: default-dir7a.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir1.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir2.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir3.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir4.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir5.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ requirements:
+ arv:RunInSingleContainer: {}
+ in: []
+ out: []
+ run: default-dir6.cwl
\ No newline at end of file
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ requirements:
+ arv:RunInSingleContainer: {}
+ in: []
+ out: []
+ run: default-dir7.cwl
\ No newline at end of file
inputs:
- id: '#main/x'
type: File
- default: {class: File, location: 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
+ default: {class: File, location: keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt,
size: 16, basename: blorp.txt, nameroot: blorp, nameext: .txt}
- id: '#main/y'
type: Directory
- default: {class: Directory, location: 'keep:99999999999999999999999999999998+99',
+ default: {class: Directory, location: keep:99999999999999999999999999999998+99,
basename: 99999999999999999999999999999998+99}
- id: '#main/z'
type: Directory
default: {class: Directory, basename: anonymous, listing: [{basename: renamed.txt,
- class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt',
+ class: File, location: keep:99999999999999999999999999999998+99/file1.txt,
nameroot: renamed, nameext: .txt}]}
outputs: []
steps:
import (
"bytes"
+ "context"
"crypto/tls"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
// A Client is an HTTP client with an API endpoint and a set of
KeepServiceURIs []string `json:",omitempty"`
dd *DiscoveryDocument
+
+ ctx context.Context
}
// The default http.Client used by a Client with Insecure==true and
}
}
-// Do adds authentication headers and then calls (*http.Client)Do().
+var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
+
+// Do adds Authorization and X-Request-Id headers and then calls
+// (*http.Client)Do().
func (c *Client) Do(req *http.Request) (*http.Response, error) {
if c.AuthToken != "" {
req.Header.Add("Authorization", "OAuth2 "+c.AuthToken)
}
+
+ if req.Header.Get("X-Request-Id") == "" {
+ reqid, _ := c.context().Value(contextKeyRequestID).(string)
+ if reqid == "" {
+ reqid = reqIDGen.Next()
+ }
+ if req.Header == nil {
+ req.Header = http.Header{"X-Request-Id": {reqid}}
+ } else {
+ req.Header.Set("X-Request-Id", reqid)
+ }
+ }
return c.httpClient().Do(req)
}
return bytes.NewBufferString(v.Encode())
}
+type contextKey string
+
+var contextKeyRequestID contextKey = "X-Request-Id"
+
+func (c *Client) WithRequestID(reqid string) *Client {
+ cc := *c
+ cc.ctx = context.WithValue(cc.context(), contextKeyRequestID, reqid)
+ return &cc
+}
+
+func (c *Client) context() context.Context {
+ if c.ctx == nil {
+ return context.Background()
+ }
+ return c.ctx
+}
+
func (c *Client) httpClient() *http.Client {
switch {
case c.Client != nil:
"net/url"
"sync"
"testing"
+ "testing/iotest"
)
type stubTransport struct {
return nil, fmt.Errorf("something awful happened")
}
+type timeoutTransport struct {
+ response []byte
+}
+
+func (stub *timeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ return &http.Response{
+ Status: "200 OK",
+ StatusCode: 200,
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ Request: req,
+ Body: ioutil.NopCloser(iotest.TimeoutReader(bytes.NewReader(stub.response))),
+ }, nil
+}
+
func TestCurrentUser(t *testing.T) {
t.Parallel()
stub := &stubTransport{
// Collection is an arvados#collection resource.
type Collection struct {
- UUID string `json:"uuid,omitempty"`
- TrashAt *time.Time `json:"trash_at,omitempty"`
- ManifestText string `json:"manifest_text,omitempty"`
- UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
- Name string `json:"name,omitempty"`
- CreatedAt *time.Time `json:"created_at,omitempty"`
- ModifiedAt *time.Time `json:"modified_at,omitempty"`
- PortableDataHash string `json:"portable_data_hash,omitempty"`
- ReplicationConfirmed *int `json:"replication_confirmed,omitempty"`
- ReplicationConfirmedAt *time.Time `json:"replication_confirmed_at,omitempty"`
- ReplicationDesired *int `json:"replication_desired,omitempty"`
- DeleteAt *time.Time `json:"delete_at,omitempty"`
- IsTrashed bool `json:"is_trashed,omitempty"`
+ UUID string `json:"uuid,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+ TrashAt *time.Time `json:"trash_at,omitempty"`
+ ManifestText string `json:"manifest_text,omitempty"`
+ UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
+ Name string `json:"name,omitempty"`
+ CreatedAt *time.Time `json:"created_at,omitempty"`
+ ModifiedAt *time.Time `json:"modified_at,omitempty"`
+ PortableDataHash string `json:"portable_data_hash,omitempty"`
+ ReplicationConfirmed *int `json:"replication_confirmed,omitempty"`
+ ReplicationConfirmedAt *time.Time `json:"replication_confirmed_at,omitempty"`
+ ReplicationDesired *int `json:"replication_desired,omitempty"`
+ StorageClassesDesired []string `json:"storage_classes_desired,omitempty"`
+ StorageClassesConfirmed []string `json:"storage_classes_confirmed,omitempty"`
+ StorageClassesConfirmedAt *time.Time `json:"storage_classes_confirmed_at,omitempty"`
+ DeleteAt *time.Time `json:"delete_at,omitempty"`
+ IsTrashed bool `json:"is_trashed,omitempty"`
}
func (c Collection) resourceName() string {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "io"
+
+type fsBackend interface {
+ keepClient
+ apiClient
+}
+
+// Ideally *Client would do everything; meanwhile keepBackend
+// implements fsBackend by merging the two kinds of arvados client.
+type keepBackend struct {
+ keepClient
+ apiClient
+}
+
+type keepClient interface {
+ ReadAt(locator string, p []byte, off int) (int, error)
+ PutB(p []byte) (string, int, error)
+}
+
+type apiClient interface {
+ RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
+ UpdateBody(rsc resource) io.Reader
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "sync"
+ "time"
+)
+
+var (
+ ErrReadOnlyFile = errors.New("read-only file")
+ ErrNegativeOffset = errors.New("cannot seek to negative offset")
+ ErrFileExists = errors.New("file exists")
+ ErrInvalidOperation = errors.New("invalid operation")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
+ ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrNotADirectory = errors.New("not a directory")
+ ErrPermission = os.ErrPermission
+)
+
+// A File is an *os.File-like interface for reading and writing files
+// in a FileSystem.
+type File interface {
+ io.Reader
+ io.Writer
+ io.Closer
+ io.Seeker
+ Size() int64
+ Readdir(int) ([]os.FileInfo, error)
+ Stat() (os.FileInfo, error)
+ Truncate(int64) error
+ Sync() error
+}
+
+// A FileSystem is an http.Filesystem plus Stat() and support for
+// opening writable files. All methods are safe to call from multiple
+// goroutines.
+type FileSystem interface {
+ http.FileSystem
+ fsBackend
+
+ rootnode() inode
+
+ // filesystem-wide lock: used by Rename() to prevent deadlock
+ // while locking multiple inodes.
+ locker() sync.Locker
+
+ // create a new node with nil parent.
+ newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
+
+ // analogous to os.Stat()
+ Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
+ Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
+ OpenFile(name string, flag int, perm os.FileMode) (File, error)
+
+ Mkdir(name string, perm os.FileMode) error
+ Remove(name string) error
+ RemoveAll(name string) error
+ Rename(oldname, newname string) error
+ Sync() error
+}
+
+type inode interface {
+ SetParent(parent inode, name string)
+ Parent() inode
+ FS() FileSystem
+ Read([]byte, filenodePtr) (int, filenodePtr, error)
+ Write([]byte, filenodePtr) (int, filenodePtr, error)
+ Truncate(int64) error
+ IsDir() bool
+ Readdir() ([]os.FileInfo, error)
+ Size() int64
+ FileInfo() os.FileInfo
+
+ // Child() performs lookups and updates of named child nodes.
+ //
+ // (The term "child" here is used strictly. This means name is
+ // not "." or "..", and name does not contain "/".)
+ //
+ // If replace is non-nil, Child calls replace(x) where x is
+ // the current child inode with the given name. If possible,
+ // the child inode is replaced with the one returned by
+ // replace().
+ //
+ // If replace(x) returns an inode (besides x or nil) that is
+ // subsequently returned by Child(), then Child()'s caller
+ // must ensure the new child's name and parent are set/updated
+ // to Child()'s name argument and its receiver respectively.
+ // This is not necessarily done before replace(x) returns, but
+ // it must be done before Child()'s caller releases the
+ // parent's lock.
+ //
+ // Nil represents "no child". replace(nil) signifies that no
+ // child with this name exists yet. If replace() returns nil,
+ // the existing child should be deleted if possible.
+ //
+ // An implementation of Child() is permitted to ignore
+ // replace() or its return value. For example, a regular file
+ // inode does not have children, so Child() always returns
+ // nil.
+ //
+ // Child() returns the child, if any, with the given name: if
+ // a child was added or changed, the new child is returned.
+ //
+ // Caller must have lock (or rlock if replace is nil).
+ Child(name string, replace func(inode) (inode, error)) (inode, error)
+
+ sync.Locker
+ RLock()
+ RUnlock()
+}
+
+type fileinfo struct {
+ name string
+ mode os.FileMode
+ size int64
+ modTime time.Time
+}
+
+// Name implements os.FileInfo.
+func (fi fileinfo) Name() string {
+ return fi.name
+}
+
+// ModTime implements os.FileInfo.
+func (fi fileinfo) ModTime() time.Time {
+ return fi.modTime
+}
+
+// Mode implements os.FileInfo.
+func (fi fileinfo) Mode() os.FileMode {
+ return fi.mode
+}
+
+// IsDir implements os.FileInfo.
+func (fi fileinfo) IsDir() bool {
+ return fi.mode&os.ModeDir != 0
+}
+
+// Size implements os.FileInfo.
+func (fi fileinfo) Size() int64 {
+ return fi.size
+}
+
+// Sys implements os.FileInfo.
+func (fi fileinfo) Sys() interface{} {
+ return nil
+}
+
+type nullnode struct{}
+
+func (*nullnode) Mkdir(string, os.FileMode) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) Read([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Write([]byte, filenodePtr) (int, filenodePtr, error) {
+ return 0, filenodePtr{}, ErrInvalidOperation
+}
+
+func (*nullnode) Truncate(int64) error {
+ return ErrInvalidOperation
+}
+
+func (*nullnode) FileInfo() os.FileInfo {
+ return fileinfo{}
+}
+
+func (*nullnode) IsDir() bool {
+ return false
+}
+
+func (*nullnode) Readdir() ([]os.FileInfo, error) {
+ return nil, ErrInvalidOperation
+}
+
+func (*nullnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return nil, ErrNotADirectory
+}
+
+type treenode struct {
+ fs FileSystem
+ parent inode
+ inodes map[string]inode
+ fileinfo fileinfo
+ sync.RWMutex
+ nullnode
+}
+
+func (n *treenode) FS() FileSystem {
+ return n.fs
+}
+
+func (n *treenode) SetParent(p inode, name string) {
+ n.Lock()
+ defer n.Unlock()
+ n.parent = p
+ n.fileinfo.name = name
+}
+
+func (n *treenode) Parent() inode {
+ n.RLock()
+ defer n.RUnlock()
+ return n.parent
+}
+
+func (n *treenode) IsDir() bool {
+ return true
+}
+
+func (n *treenode) Child(name string, replace func(inode) (inode, error)) (child inode, err error) {
+ child = n.inodes[name]
+ if name == "" || name == "." || name == ".." {
+ err = ErrInvalidArgument
+ return
+ }
+ if replace == nil {
+ return
+ }
+ newchild, err := replace(child)
+ if err != nil {
+ return
+ }
+ if newchild == nil {
+ delete(n.inodes, name)
+ } else if newchild != child {
+ n.inodes[name] = newchild
+ n.fileinfo.modTime = time.Now()
+ child = newchild
+ }
+ return
+}
+
+func (n *treenode) Size() int64 {
+ return n.FileInfo().Size()
+}
+
+func (n *treenode) FileInfo() os.FileInfo {
+ n.Lock()
+ defer n.Unlock()
+ n.fileinfo.size = int64(len(n.inodes))
+ return n.fileinfo
+}
+
+func (n *treenode) Readdir() (fi []os.FileInfo, err error) {
+ n.RLock()
+ defer n.RUnlock()
+ fi = make([]os.FileInfo, 0, len(n.inodes))
+ for _, inode := range n.inodes {
+ fi = append(fi, inode.FileInfo())
+ }
+ return
+}
+
+type fileSystem struct {
+ root inode
+ fsBackend
+ mutex sync.Mutex
+}
+
+func (fs *fileSystem) rootnode() inode {
+ return fs.root
+}
+
+func (fs *fileSystem) locker() sync.Locker {
+ return &fs.mutex
+}
+
+// OpenFile is analogous to os.OpenFile().
+func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+ return fs.openFile(name, flag, perm)
+}
+
+func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ parent, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return nil, err
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable && parent.IsDir() {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &filehandle{inode: parent}, nil
+ case "..":
+ return &filehandle{inode: parent.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ parent.Lock()
+ defer parent.Unlock()
+ } else {
+ parent.RLock()
+ defer parent.RUnlock()
+ }
+ n, err := parent.Child(name, nil)
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ if !createMode {
+ return nil, os.ErrNotExist
+ }
+ n, err = parent.Child(name, func(inode) (repl inode, err error) {
+ repl, err = parent.FS().newNode(name, perm|0755, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(parent, name)
+ return
+ })
+ if err != nil {
+ return nil, err
+ } else if n == nil {
+ // Parent rejected new child, but returned no error
+ return nil, ErrInvalidArgument
+ }
+ } else if flag&os.O_EXCL != 0 {
+ return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if n.IsDir() {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else if err := n.Truncate(0); err != nil {
+ return nil, err
+ }
+ }
+ return &filehandle{
+ inode: n,
+ append: flag&os.O_APPEND != 0,
+ readable: readable,
+ writable: writable,
+ }, nil
+}
+
+func (fs *fileSystem) Open(name string) (http.File, error) {
+ return fs.OpenFile(name, os.O_RDONLY, 0)
+}
+
+func (fs *fileSystem) Create(name string) (File, error) {
+ return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+}
+
+func (fs *fileSystem) Mkdir(name string, perm os.FileMode) error {
+ dirname, name := path.Split(name)
+ n, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
+ }
+ n.Lock()
+ defer n.Unlock()
+ if child, err := n.Child(name, nil); err != nil {
+ return err
+ } else if child != nil {
+ return os.ErrExist
+ }
+
+ _, err = n.Child(name, func(inode) (repl inode, err error) {
+ repl, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
+ if err != nil {
+ return
+ }
+ repl.SetParent(n, name)
+ return
+ })
+ return err
+}
+
+func (fs *fileSystem) Stat(name string) (os.FileInfo, error) {
+ node, err := rlookup(fs.root, name)
+ if err != nil {
+ return nil, err
+ }
+ return node.FileInfo(), nil
+}
+
+func (fs *fileSystem) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // TODO: If the nearest common ancestor ("nca") of olddirf and
+ // newdirf is on a different filesystem than fs, we should
+ // call nca.FS().Rename() instead of proceeding. Until then
+ // it's awkward for filesystems to implement their own Rename
+ // methods effectively: the only one that runs is the one on
+ // the root FileSystem exposed to the caller (webdav, fuse,
+ // etc).
+
+ // When acquiring locks on multiple inodes, avoid deadlock by
+ // locking the entire containing filesystem first.
+ cfs := olddirf.inode.FS()
+ cfs.locker().Lock()
+ defer cfs.locker().Unlock()
+
+ if cfs != newdirf.inode.FS() {
+ // Moving inodes across filesystems is not (yet)
+ // supported. Locking inodes from different
+ // filesystems could deadlock, so we must error out
+ // now.
+ return ErrInvalidArgument
+ }
+
+ // To ensure we can test reliably whether we're about to move
+ // a directory into itself, lock all potential common
+ // ancestors of olddir and newdir.
+ needLock := []sync.Locker{}
+ for _, node := range []inode{olddirf.inode, newdirf.inode} {
+ needLock = append(needLock, node)
+ for node.Parent() != node && node.Parent().FS() == node.FS() {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ _, err = olddirf.inode.Child(oldname, func(oldinode inode) (inode, error) {
+ if oldinode == nil {
+ return oldinode, os.ErrNotExist
+ }
+ if locked[oldinode] {
+ // oldinode cannot become a descendant of itself.
+ return oldinode, ErrInvalidArgument
+ }
+ if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
+ // moving a mount point to a different parent
+ // is not (yet) supported.
+ return oldinode, ErrInvalidArgument
+ }
+ accepted, err := newdirf.inode.Child(newname, func(existing inode) (inode, error) {
+ if existing != nil && existing.IsDir() {
+ return existing, ErrIsDirectory
+ }
+ return oldinode, nil
+ })
+ if err != nil {
+ // Leave oldinode in olddir.
+ return oldinode, err
+ }
+ accepted.SetParent(newdirf.inode, newname)
+ return nil, nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Remove(name string) error {
+ return fs.remove(strings.TrimRight(name, "/"), false)
+}
+
+func (fs *fileSystem) RemoveAll(name string) error {
+ err := fs.remove(strings.TrimRight(name, "/"), true)
+ if os.IsNotExist(err) {
+ // "If the path does not exist, RemoveAll returns
+ // nil." (see "os" pkg)
+ err = nil
+ }
+ return err
+}
+
+func (fs *fileSystem) remove(name string, recursive bool) error {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dir, err := rlookup(fs.root, dirname)
+ if err != nil {
+ return err
+ }
+ dir.Lock()
+ defer dir.Unlock()
+ _, err = dir.Child(name, func(node inode) (inode, error) {
+ if node == nil {
+ return nil, os.ErrNotExist
+ }
+ if !recursive && node.IsDir() && node.Size() > 0 {
+ return node, ErrDirectoryNotEmpty
+ }
+ return nil, nil
+ })
+ return err
+}
+
+func (fs *fileSystem) Sync() error {
+ log.Printf("TODO: sync fileSystem")
+ return ErrInvalidOperation
+}
+
+// rlookup (recursive lookup) returns the inode for the file/directory
+// with the given name (which may contain "/" separators). If no such
+// file/directory exists, the returned node is nil.
+func rlookup(start inode, path string) (node inode, err error) {
+ node = start
+ for _, name := range strings.Split(path, "/") {
+ if node.IsDir() {
+ if name == "." || name == "" {
+ continue
+ }
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ }
+ node, err = func() (inode, error) {
+ node.RLock()
+ defer node.RUnlock()
+ return node.Child(name, nil)
+ }()
+ if node == nil || err != nil {
+ break
+ }
+ }
+ if node == nil && err == nil {
+ err = os.ErrNotExist
+ }
+ return
+}
+
+func permittedName(name string) bool {
+ return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")
+}
package arvados
import (
- "errors"
+ "encoding/json"
"fmt"
"io"
- "net/http"
+ "log"
"os"
"path"
"regexp"
"time"
)
-var (
- ErrReadOnlyFile = errors.New("read-only file")
- ErrNegativeOffset = errors.New("cannot seek to negative offset")
- ErrFileExists = errors.New("file exists")
- ErrInvalidOperation = errors.New("invalid operation")
- ErrInvalidArgument = errors.New("invalid argument")
- ErrDirectoryNotEmpty = errors.New("directory not empty")
- ErrWriteOnlyMode = errors.New("file is O_WRONLY")
- ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
- ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
- ErrPermission = os.ErrPermission
+var maxBlockSize = 1 << 26
- maxBlockSize = 1 << 26
-)
-
-// A File is an *os.File-like interface for reading and writing files
-// in a CollectionFileSystem.
-type File interface {
- io.Reader
- io.Writer
- io.Closer
- io.Seeker
- Size() int64
- Readdir(int) ([]os.FileInfo, error)
- Stat() (os.FileInfo, error)
- Truncate(int64) error
-}
-
-type keepClient interface {
- ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
-}
-
-type fileinfo struct {
- name string
- mode os.FileMode
- size int64
- modTime time.Time
-}
-
-// Name implements os.FileInfo.
-func (fi fileinfo) Name() string {
- return fi.name
-}
-
-// ModTime implements os.FileInfo.
-func (fi fileinfo) ModTime() time.Time {
- return fi.modTime
-}
-
-// Mode implements os.FileInfo.
-func (fi fileinfo) Mode() os.FileMode {
- return fi.mode
-}
-
-// IsDir implements os.FileInfo.
-func (fi fileinfo) IsDir() bool {
- return fi.mode&os.ModeDir != 0
-}
-
-// Size implements os.FileInfo.
-func (fi fileinfo) Size() int64 {
- return fi.size
-}
-
-// Sys implements os.FileInfo.
-func (fi fileinfo) Sys() interface{} {
- return nil
-}
-
-// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files. All methods are safe to call
-// from multiple goroutines.
+// A CollectionFileSystem is a FileSystem that can be serialized as a
+// manifest and stored as a collection.
type CollectionFileSystem interface {
- http.FileSystem
-
- // analogous to os.Stat()
- Stat(name string) (os.FileInfo, error)
-
- // analogous to os.Create(): create/truncate a file and open it O_RDWR.
- Create(name string) (File, error)
-
- // Like os.OpenFile(): create or open a file or directory.
- //
- // If flag&os.O_EXCL==0, it opens an existing file or
- // directory if one exists. If flag&os.O_CREATE!=0, it creates
- // a new empty file or directory if one does not already
- // exist.
- //
- // When creating a new item, perm&os.ModeDir determines
- // whether it is a file or a directory.
- //
- // A file can be opened multiple times and used concurrently
- // from multiple goroutines. However, each File object should
- // be used by only one goroutine at a time.
- OpenFile(name string, flag int, perm os.FileMode) (File, error)
-
- Mkdir(name string, perm os.FileMode) error
- Remove(name string) error
- RemoveAll(name string) error
- Rename(oldname, newname string) error
+ FileSystem
// Flush all file data to Keep and return a snapshot of the
// filesystem suitable for saving as (Collection)ManifestText.
MarshalManifest(prefix string) (string, error)
}
-type fileSystem struct {
- dirnode
-}
-
-func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
- return fs.dirnode.OpenFile(name, flag, perm)
+type collectionFileSystem struct {
+ fileSystem
+ uuid string
}
-func (fs *fileSystem) Open(name string) (http.File, error) {
- return fs.dirnode.OpenFile(name, os.O_RDONLY, 0)
+// FileSystem returns a CollectionFileSystem for the collection.
+func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFileSystem, error) {
+ var modTime time.Time
+ if c.ModifiedAt == nil {
+ modTime = time.Now()
+ } else {
+ modTime = *c.ModifiedAt
+ }
+ fs := &collectionFileSystem{
+ uuid: c.UUID,
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: client, keepClient: kc},
+ },
+ }
+ root := &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: ".",
+ mode: os.ModeDir | 0755,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }
+ root.SetParent(root, ".")
+ if err := root.loadManifest(c.ManifestText); err != nil {
+ return nil, err
+ }
+ backdateTree(root, modTime)
+ fs.root = root
+ return fs, nil
}
-func (fs *fileSystem) Create(name string) (File, error) {
- return fs.dirnode.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
+func backdateTree(n inode, modTime time.Time) {
+ switch n := n.(type) {
+ case *filenode:
+ n.fileinfo.modTime = modTime
+ case *dirnode:
+ n.fileinfo.modTime = modTime
+ for _, n := range n.inodes {
+ backdateTree(n, modTime)
+ }
+ }
}
-func (fs *fileSystem) Stat(name string) (fi os.FileInfo, err error) {
- node := fs.dirnode.lookupPath(name)
- if node == nil {
- err = os.ErrNotExist
+func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ if name == "" || name == "." || name == ".." {
+ return nil, ErrInvalidArgument
+ }
+ if perm.IsDir() {
+ return &dirnode{
+ fs: fs,
+ treenode: treenode{
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm | os.ModeDir,
+ modTime: modTime,
+ },
+ inodes: make(map[string]inode),
+ },
+ }, nil
} else {
- fi = node.Stat()
+ return &filenode{
+ fs: fs,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
}
- return
}
-type inode interface {
- Parent() inode
- Read([]byte, filenodePtr) (int, filenodePtr, error)
- Write([]byte, filenodePtr) (int, filenodePtr, error)
- Truncate(int64) error
- Readdir() []os.FileInfo
- Size() int64
- Stat() os.FileInfo
- sync.Locker
- RLock()
- RUnlock()
+func (fs *collectionFileSystem) Sync() error {
+ log.Printf("cfs.Sync()")
+ if fs.uuid == "" {
+ return nil
+ }
+ txt, err := fs.MarshalManifest(".")
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ return err
+ }
+ coll := &Collection{
+ UUID: fs.uuid,
+ ManifestText: txt,
+ }
+ err = fs.RequestAndDecode(nil, "PUT", "arvados/v1/collections/"+fs.uuid, fs.UpdateBody(coll), map[string]interface{}{"select": []string{"uuid"}})
+ if err != nil {
+ log.Printf("WARNING: (collectionFileSystem)Sync() failed: %s", err)
+ }
+ return err
}
-// filenode implements inode.
-type filenode struct {
- fileinfo fileinfo
- parent *dirnode
- segments []segment
- // number of times `segments` has changed in a
- // way that might invalidate a filenodePtr
- repacked int64
- memsize int64 // bytes in memSegments
- sync.RWMutex
+func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
+ fs.fileSystem.root.Lock()
+ defer fs.fileSystem.root.Unlock()
+ return fs.fileSystem.root.(*dirnode).marshalManifest(prefix)
}
// filenodePtr is an offset into a file that is (usually) efficient to
return
}
+// filenode implements inode.
+type filenode struct {
+ parent inode
+ fs FileSystem
+ fileinfo fileinfo
+ segments []segment
+ // number of times `segments` has changed in a
+ // way that might invalidate a filenodePtr
+ repacked int64
+ memsize int64 // bytes in memSegments
+ sync.RWMutex
+ nullnode
+}
+
// caller must have lock
func (fn *filenode) appendSegment(e segment) {
fn.segments = append(fn.segments, e)
fn.fileinfo.size += int64(e.Len())
}
+func (fn *filenode) SetParent(p inode, name string) {
+ fn.Lock()
+ defer fn.Unlock()
+ fn.parent = p
+ fn.fileinfo.name = name
+}
+
func (fn *filenode) Parent() inode {
fn.RLock()
defer fn.RUnlock()
return fn.parent
}
-func (fn *filenode) Readdir() []os.FileInfo {
- return nil
+func (fn *filenode) FS() FileSystem {
+ return fn.fs
}
// Read reads file data from a single segment, starting at startPtr,
return fn.fileinfo.Size()
}
-func (fn *filenode) Stat() os.FileInfo {
+func (fn *filenode) FileInfo() os.FileInfo {
fn.RLock()
defer fn.RUnlock()
return fn.fileinfo
if !ok || seg.Len() < maxBlockSize {
continue
}
- locator, _, err := fn.parent.kc.PutB(seg.buf)
+ locator, _, err := fn.FS().PutB(seg.buf)
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
}
fn.memsize -= int64(seg.Len())
fn.segments[idx] = storedSegment{
- kc: fn.parent.kc,
+ kc: fn.FS(),
locator: locator,
size: seg.Len(),
offset: 0,
}
}
-// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
- var modTime time.Time
- if c.ModifiedAt == nil {
- modTime = time.Now()
- } else {
- modTime = *c.ModifiedAt
- }
- fs := &fileSystem{dirnode: dirnode{
- client: client,
- kc: kc,
- fileinfo: fileinfo{
- name: ".",
- mode: os.ModeDir | 0755,
- modTime: modTime,
- },
- parent: nil,
- inodes: make(map[string]inode),
- }}
- fs.dirnode.parent = &fs.dirnode
- if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
- return nil, err
- }
- return fs, nil
-}
-
-type filehandle struct {
- inode
- ptr filenodePtr
- append bool
- readable bool
- writable bool
- unreaddirs []os.FileInfo
-}
-
-func (f *filehandle) Read(p []byte) (n int, err error) {
- if !f.readable {
- return 0, ErrWriteOnlyMode
- }
- f.inode.RLock()
- defer f.inode.RUnlock()
- n, f.ptr, err = f.inode.Read(p, f.ptr)
- return
-}
-
-func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
- size := f.inode.Size()
- ptr := f.ptr
- switch whence {
- case io.SeekStart:
- ptr.off = off
- case io.SeekCurrent:
- ptr.off += off
- case io.SeekEnd:
- ptr.off = size + off
- }
- if ptr.off < 0 {
- return f.ptr.off, ErrNegativeOffset
- }
- if ptr.off != f.ptr.off {
- f.ptr = ptr
- // force filenode to recompute f.ptr fields on next
- // use
- f.ptr.repacked = -1
- }
- return f.ptr.off, nil
-}
-
-func (f *filehandle) Truncate(size int64) error {
- return f.inode.Truncate(size)
+type dirnode struct {
+ fs *collectionFileSystem
+ treenode
}
-func (f *filehandle) Write(p []byte) (n int, err error) {
- if !f.writable {
- return 0, ErrReadOnlyFile
- }
- f.inode.Lock()
- defer f.inode.Unlock()
- if fn, ok := f.inode.(*filenode); ok && f.append {
- f.ptr = filenodePtr{
- off: fn.fileinfo.size,
- segmentIdx: len(fn.segments),
- segmentOff: 0,
- repacked: fn.repacked,
- }
- }
- n, f.ptr, err = f.inode.Write(p, f.ptr)
- return
+func (dn *dirnode) FS() FileSystem {
+ return dn.fs
}
-func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.Stat().IsDir() {
- return nil, ErrInvalidOperation
- }
- if count <= 0 {
- return f.inode.Readdir(), nil
- }
- if f.unreaddirs == nil {
- f.unreaddirs = f.inode.Readdir()
- }
- if len(f.unreaddirs) == 0 {
- return nil, io.EOF
- }
- if count > len(f.unreaddirs) {
- count = len(f.unreaddirs)
+func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ if dn == dn.fs.rootnode() && name == ".arvados#collection" {
+ gn := &getternode{Getter: func() ([]byte, error) {
+ var coll Collection
+ var err error
+ coll.ManifestText, err = dn.fs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ data, err := json.Marshal(&coll)
+ if err == nil {
+ data = append(data, '\n')
+ }
+ return data, err
+ }}
+ gn.SetParent(dn, name)
+ return gn, nil
}
- ret := f.unreaddirs[:count]
- f.unreaddirs = f.unreaddirs[count:]
- return ret, nil
-}
-
-func (f *filehandle) Stat() (os.FileInfo, error) {
- return f.inode.Stat(), nil
-}
-
-func (f *filehandle) Close() error {
- return nil
-}
-
-type dirnode struct {
- fileinfo fileinfo
- parent *dirnode
- client *Client
- kc keepClient
- inodes map[string]inode
- sync.RWMutex
+ return dn.treenode.Child(name, replace)
}
// sync flushes in-memory data (for all files in the tree rooted at
for _, sb := range sbs {
block = append(block, sb.fn.segments[sb.idx].(*memSegment).buf...)
}
- locator, _, err := dn.kc.PutB(block)
+ locator, _, err := dn.fs.PutB(block)
if err != nil {
return err
}
for _, sb := range sbs {
data := sb.fn.segments[sb.idx].(*memSegment).buf
sb.fn.segments[sb.idx] = storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: locator,
size: len(block),
offset: off,
return flush(pending)
}
-func (dn *dirnode) MarshalManifest(prefix string) (string, error) {
- dn.Lock()
- defer dn.Unlock()
- return dn.marshalManifest(prefix)
-}
-
// caller must have read lock.
func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var streamLen int64
blkLen = int(offset + length - pos - int64(blkOff))
}
fnode.appendSegment(storedSegment{
- kc: dn.kc,
+ kc: dn.fs,
locator: seg.locator,
size: seg.size,
offset: blkOff,
// only safe to call from loadManifest -- no locking
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
+ var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
- if basename == "" || basename == "." || basename == ".." {
- err = fmt.Errorf("invalid filename")
+ if !permittedName(basename) {
+ err = fmt.Errorf("invalid file part %q in path %q", basename, path)
return
}
for _, name := range names[:len(names)-1] {
switch name {
case "", ".":
+ continue
case "..":
- dn = dn.parent
- default:
- switch node := dn.inodes[name].(type) {
- case nil:
- dn = dn.newDirnode(name, 0755, dn.fileinfo.modTime)
- case *dirnode:
- dn = node
- case *filenode:
- err = ErrFileExists
- return
+ if node == dn {
+ // can't be sure parent will be a *dirnode
+ return nil, ErrInvalidArgument
}
- }
- }
- switch node := dn.inodes[basename].(type) {
- case nil:
- fn = dn.newFilenode(basename, 0755, dn.fileinfo.modTime)
- case *filenode:
- fn = node
- case *dirnode:
- err = ErrIsDirectory
- }
- return
-}
-
-func (dn *dirnode) mkdir(name string) (*filehandle, error) {
- return dn.OpenFile(name, os.O_CREATE|os.O_EXCL, os.ModeDir|0755)
-}
-
-func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
- f, err := dn.mkdir(name)
- if err == nil {
- err = f.Close()
- }
- return err
-}
-
-func (dn *dirnode) Remove(name string) error {
- return dn.remove(strings.TrimRight(name, "/"), false)
-}
-
-func (dn *dirnode) RemoveAll(name string) error {
- err := dn.remove(strings.TrimRight(name, "/"), true)
- if os.IsNotExist(err) {
- // "If the path does not exist, RemoveAll returns
- // nil." (see "os" pkg)
- err = nil
- }
- return err
-}
-
-func (dn *dirnode) remove(name string, recursive bool) error {
- dirname, name := path.Split(name)
- if name == "" || name == "." || name == ".." {
- return ErrInvalidArgument
- }
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return os.ErrNotExist
- }
- dn.Lock()
- defer dn.Unlock()
- switch node := dn.inodes[name].(type) {
- case nil:
- return os.ErrNotExist
- case *dirnode:
- node.RLock()
- defer node.RUnlock()
- if !recursive && len(node.inodes) > 0 {
- return ErrDirectoryNotEmpty
- }
- }
- delete(dn.inodes, name)
- return nil
-}
-
-func (dn *dirnode) Rename(oldname, newname string) error {
- olddir, oldname := path.Split(oldname)
- if oldname == "" || oldname == "." || oldname == ".." {
- return ErrInvalidArgument
- }
- olddirf, err := dn.OpenFile(olddir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", olddir, err)
- }
- defer olddirf.Close()
- newdir, newname := path.Split(newname)
- if newname == "." || newname == ".." {
- return ErrInvalidArgument
- } else if newname == "" {
- // Rename("a/b", "c/") means Rename("a/b", "c/b")
- newname = oldname
- }
- newdirf, err := dn.OpenFile(newdir+".", os.O_RDONLY, 0)
- if err != nil {
- return fmt.Errorf("%q: %s", newdir, err)
- }
- defer newdirf.Close()
-
- // When acquiring locks on multiple nodes, all common
- // ancestors must be locked first in order to avoid
- // deadlock. This is assured by locking the path from root to
- // newdir, then locking the path from root to olddir, skipping
- // any already-locked nodes.
- needLock := []sync.Locker{}
- for _, f := range []*filehandle{olddirf, newdirf} {
- node := f.inode
- needLock = append(needLock, node)
- for node.Parent() != node {
node = node.Parent()
- needLock = append(needLock, node)
- }
- }
- locked := map[sync.Locker]bool{}
- for i := len(needLock) - 1; i >= 0; i-- {
- if n := needLock[i]; !locked[n] {
- n.Lock()
- defer n.Unlock()
- locked[n] = true
+ continue
}
- }
-
- olddn := olddirf.inode.(*dirnode)
- newdn := newdirf.inode.(*dirnode)
- oldinode, ok := olddn.inodes[oldname]
- if !ok {
- return os.ErrNotExist
- }
- if locked[oldinode] {
- // oldinode cannot become a descendant of itself.
- return ErrInvalidArgument
- }
- if existing, ok := newdn.inodes[newname]; ok {
- // overwriting an existing file or dir
- if dn, ok := existing.(*dirnode); ok {
- if !oldinode.Stat().IsDir() {
- return ErrIsDirectory
- }
- dn.RLock()
- defer dn.RUnlock()
- if len(dn.inodes) > 0 {
- return ErrDirectoryNotEmpty
+ node, err = node.Child(name, func(child inode) (inode, error) {
+ if child == nil {
+ child, err := node.FS().newNode(name, 0755|os.ModeDir, node.Parent().FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
+ child.SetParent(node, name)
+ return child, nil
+ } else if !child.IsDir() {
+ return child, ErrFileExists
+ } else {
+ return child, nil
}
+ })
+ if err != nil {
+ return
}
- } else {
- if newdn.inodes == nil {
- newdn.inodes = make(map[string]inode)
- }
- newdn.fileinfo.size++
}
- newdn.inodes[newname] = oldinode
- switch n := oldinode.(type) {
- case *dirnode:
- n.parent = newdn
- case *filenode:
- n.parent = newdn
- default:
- panic(fmt.Sprintf("bad inode type %T", n))
- }
- delete(olddn.inodes, oldname)
- olddn.fileinfo.size--
- return nil
-}
-
-func (dn *dirnode) Parent() inode {
- dn.RLock()
- defer dn.RUnlock()
- return dn.parent
-}
-
-func (dn *dirnode) Readdir() (fi []os.FileInfo) {
- dn.RLock()
- defer dn.RUnlock()
- fi = make([]os.FileInfo, 0, len(dn.inodes))
- for _, inode := range dn.inodes {
- fi = append(fi, inode.Stat())
- }
- return
-}
-
-func (dn *dirnode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Write(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
- return 0, ptr, ErrInvalidOperation
-}
-
-func (dn *dirnode) Size() int64 {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo.Size()
-}
-
-func (dn *dirnode) Stat() os.FileInfo {
- dn.RLock()
- defer dn.RUnlock()
- return dn.fileinfo
-}
-
-func (dn *dirnode) Truncate(int64) error {
- return ErrInvalidOperation
-}
-
-// lookupPath returns the inode for the file/directory with the given
-// name (which may contain "/" separators), along with its parent
-// node. If no such file/directory exists, the returned node is nil.
-func (dn *dirnode) lookupPath(path string) (node inode) {
- node = dn
- for _, name := range strings.Split(path, "/") {
- dn, ok := node.(*dirnode)
- if !ok {
- return nil
- }
- if name == "." || name == "" {
- continue
- }
- if name == ".." {
- node = node.Parent()
- continue
+ _, err = node.Child(basename, func(child inode) (inode, error) {
+ switch child := child.(type) {
+ case nil:
+ child, err = node.FS().newNode(basename, 0755, node.FileInfo().ModTime())
+ if err != nil {
+ return nil, err
+ }
+ child.SetParent(node, basename)
+ fn = child.(*filenode)
+ return child, nil
+ case *filenode:
+ fn = child
+ return child, nil
+ case *dirnode:
+ return child, ErrIsDirectory
+ default:
+ return child, ErrInvalidArgument
}
- dn.RLock()
- node = dn.inodes[name]
- dn.RUnlock()
- }
+ })
return
}
-func (dn *dirnode) newDirnode(name string, perm os.FileMode, modTime time.Time) *dirnode {
- child := &dirnode{
- parent: dn,
- client: dn.client,
- kc: dn.kc,
- fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | perm,
- modTime: modTime,
- },
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
-}
-
-func (dn *dirnode) newFilenode(name string, perm os.FileMode, modTime time.Time) *filenode {
- child := &filenode{
- parent: dn,
- fileinfo: fileinfo{
- name: name,
- mode: perm,
- modTime: modTime,
- },
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
- }
- dn.inodes[name] = child
- dn.fileinfo.size++
- return child
-}
-
-// OpenFile is analogous to os.OpenFile().
-func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
- if flag&os.O_SYNC != 0 {
- return nil, ErrSyncNotSupported
- }
- dirname, name := path.Split(name)
- dn, ok := dn.lookupPath(dirname).(*dirnode)
- if !ok {
- return nil, os.ErrNotExist
- }
- var readable, writable bool
- switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
- case os.O_RDWR:
- readable = true
- writable = true
- case os.O_RDONLY:
- readable = true
- case os.O_WRONLY:
- writable = true
- default:
- return nil, fmt.Errorf("invalid flags 0x%x", flag)
- }
- if !writable {
- // A directory can be opened via "foo/", "foo/.", or
- // "foo/..".
- switch name {
- case ".", "":
- return &filehandle{inode: dn}, nil
- case "..":
- return &filehandle{inode: dn.Parent()}, nil
- }
- }
- createMode := flag&os.O_CREATE != 0
- if createMode {
- dn.Lock()
- defer dn.Unlock()
- } else {
- dn.RLock()
- defer dn.RUnlock()
- }
- n, ok := dn.inodes[name]
- if !ok {
- if !createMode {
- return nil, os.ErrNotExist
- }
- if perm.IsDir() {
- n = dn.newDirnode(name, 0755, time.Now())
- } else {
- n = dn.newFilenode(name, 0755, time.Now())
- }
- } else if flag&os.O_EXCL != 0 {
- return nil, ErrFileExists
- } else if flag&os.O_TRUNC != 0 {
- if !writable {
- return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
- } else if fn, ok := n.(*filenode); !ok {
- return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
- } else {
- fn.Truncate(0)
- }
- }
- return &filehandle{
- inode: n,
- append: flag&os.O_APPEND != 0,
- readable: readable,
- writable: writable,
- }, nil
-}
-
type segment interface {
io.ReaderAt
Len() int
}
type storedSegment struct {
- kc keepClient
+ kc fsBackend
locator string
size int // size of stored block (also encoded in locator)
offset int // position of segment within the stored block
}
func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
maxBlockSize = 8
defer func() { maxBlockSize = 2 << 26 }()
err = fs.Rename(
fmt.Sprintf("dir%d/file%d/patherror", i, j),
fmt.Sprintf("dir%d/irrelevant", i))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
// newname parent dir is a file
err = fs.Rename(
fmt.Sprintf("dir%d/dir%d/file%d", i, j, j),
fmt.Sprintf("dir%d/file%d/patherror", i, inner-j-1))
- c.Check(err, check.ErrorMatches, `.*does not exist`)
+ c.Check(err, check.ErrorMatches, `.*not a directory`)
}(i, j)
}
}
// expect ~2 seconds to load a manifest with 256K files
func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
+ if testing.Short() {
+ c.Skip("slow")
+ }
+
const (
dirCount = 512
fileCount = 512
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+ "time"
+)
+
+func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+ var modTime time.Time
+ if coll.ModifiedAt != nil {
+ modTime = *coll.ModifiedAt
+ } else {
+ modTime = time.Now()
+ }
+ placeholder := &treenode{
+ fs: fs,
+ parent: parent,
+ inodes: nil,
+ fileinfo: fileinfo{
+ name: coll.Name,
+ modTime: modTime,
+ mode: 0755 | os.ModeDir,
+ },
+ }
+ return &deferrednode{wrapped: placeholder, create: func() inode {
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ cfs, err := coll.FileSystem(fs, fs)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ root := cfs.rootnode()
+ root.SetParent(parent, coll.Name)
+ return root
+ }}
+}
+
+// A deferrednode wraps an inode that's expensive to build. Initially,
+// it responds to basic directory functions by proxying to the given
+// placeholder. If a caller uses a read/write/lock operation,
+// deferrednode calls the create() func to create the real inode, and
+// proxies to the real inode from then on.
+//
+// In practice, this means a deferrednode's parent's directory listing
+// can be generated using only the placeholder, instead of waiting for
+// create().
+type deferrednode struct {
+ wrapped inode
+ create func() inode
+ mtx sync.Mutex
+ created bool
+}
+
+func (dn *deferrednode) realinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ if !dn.created {
+ dn.wrapped = dn.create()
+ dn.created = true
+ }
+ return dn.wrapped
+}
+
+func (dn *deferrednode) currentinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ return dn.wrapped
+}
+
+func (dn *deferrednode) Read(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Read(p, pos)
+}
+
+func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Write(p, pos)
+}
+
+func (dn *deferrednode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return dn.realinode().Child(name, replace)
+}
+
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() ([]os.FileInfo, error) { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
--- /dev/null
+package arvados
+
+import (
+ "io"
+ "os"
+)
+
+type filehandle struct {
+ inode
+ ptr filenodePtr
+ append bool
+ readable bool
+ writable bool
+ unreaddirs []os.FileInfo
+}
+
+func (f *filehandle) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
+ n, f.ptr, err = f.inode.Read(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Seek(off int64, whence int) (pos int64, err error) {
+ size := f.inode.Size()
+ ptr := f.ptr
+ switch whence {
+ case io.SeekStart:
+ ptr.off = off
+ case io.SeekCurrent:
+ ptr.off += off
+ case io.SeekEnd:
+ ptr.off = size + off
+ }
+ if ptr.off < 0 {
+ return f.ptr.off, ErrNegativeOffset
+ }
+ if ptr.off != f.ptr.off {
+ f.ptr = ptr
+ // force filenode to recompute f.ptr fields on next
+ // use
+ f.ptr.repacked = -1
+ }
+ return f.ptr.off, nil
+}
+
+func (f *filehandle) Truncate(size int64) error {
+ return f.inode.Truncate(size)
+}
+
+func (f *filehandle) Write(p []byte) (n int, err error) {
+ if !f.writable {
+ return 0, ErrReadOnlyFile
+ }
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ segmentIdx: len(fn.segments),
+ segmentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
+ n, f.ptr, err = f.inode.Write(p, f.ptr)
+ return
+}
+
+func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
+ if !f.inode.IsDir() {
+ return nil, ErrInvalidOperation
+ }
+ if count <= 0 {
+ return f.inode.Readdir()
+ }
+ if f.unreaddirs == nil {
+ var err error
+ f.unreaddirs, err = f.inode.Readdir()
+ if err != nil {
+ return nil, err
+ }
+ }
+ if len(f.unreaddirs) == 0 {
+ return nil, io.EOF
+ }
+ if count > len(f.unreaddirs) {
+ count = len(f.unreaddirs)
+ }
+ ret := f.unreaddirs[:count]
+ f.unreaddirs = f.unreaddirs[count:]
+ return ret, nil
+}
+
+func (f *filehandle) Stat() (os.FileInfo, error) {
+ return f.inode.FileInfo(), nil
+}
+
+func (f *filehandle) Close() error {
+ return nil
+}
+
+func (f *filehandle) Sync() error {
+ // Sync the containing filesystem.
+ return f.FS().Sync()
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "os"
+ "time"
+)
+
+// A getternode is a read-only character device that returns whatever
+// data is returned by the supplied function.
+type getternode struct {
+ Getter func() ([]byte, error)
+
+ treenode
+ data *bytes.Reader
+}
+
+func (*getternode) IsDir() bool {
+ return false
+}
+
+func (*getternode) Child(string, func(inode) (inode, error)) (inode, error) {
+ return nil, ErrInvalidArgument
+}
+
+func (gn *getternode) get() error {
+ if gn.data != nil {
+ return nil
+ }
+ data, err := gn.Getter()
+ if err != nil {
+ return err
+ }
+ gn.data = bytes.NewReader(data)
+ return nil
+}
+
+func (gn *getternode) Size() int64 {
+ return gn.FileInfo().Size()
+}
+
+func (gn *getternode) FileInfo() os.FileInfo {
+ gn.Lock()
+ defer gn.Unlock()
+ var size int64
+ if gn.get() == nil {
+ size = gn.data.Size()
+ }
+ return fileinfo{
+ modTime: time.Now(),
+ mode: 0444,
+ size: size,
+ }
+}
+
+func (gn *getternode) Read(p []byte, ptr filenodePtr) (int, filenodePtr, error) {
+ if err := gn.get(); err != nil {
+ return 0, ptr, err
+ }
+ n, err := gn.data.ReadAt(p, ptr.off)
+ return n, filenodePtr{off: ptr.off + int64(n)}, err
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "sync"
+ "time"
+)
+
+// lookupnode is a caching tree node that is initially empty and calls
+// loadOne and loadAll to load/update child nodes as needed.
+//
+// See (*customFileSystem)MountUsers for example usage.
+type lookupnode struct {
+ inode
+ loadOne func(parent inode, name string) (inode, error)
+ loadAll func(parent inode) ([]inode, error)
+ stale func(time.Time) bool
+
+ // internal fields
+ staleLock sync.Mutex
+ staleAll time.Time
+ staleOne map[string]time.Time
+}
+
+func (ln *lookupnode) Readdir() ([]os.FileInfo, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) {
+ all, err := ln.loadAll(ln)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range all {
+ _, err = ln.inode.Child(child.FileInfo().Name(), func(inode) (inode, error) {
+ return child, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ ln.staleAll = checkTime
+ // No value in ln.staleOne can make a difference to an
+ // "entry is stale?" test now, because no value is
+ // newer than ln.staleAll. Reclaim memory.
+ ln.staleOne = nil
+ }
+ return ln.inode.Readdir()
+}
+
+func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ ln.staleLock.Lock()
+ defer ln.staleLock.Unlock()
+ checkTime := time.Now()
+ if ln.stale(ln.staleAll) && ln.stale(ln.staleOne[name]) {
+ _, err := ln.inode.Child(name, func(inode) (inode, error) {
+ return ln.loadOne(ln, name)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if ln.staleOne == nil {
+ ln.staleOne = map[string]time.Time{name: checkTime}
+ } else {
+ ln.staleOne[name] = checkTime
+ }
+ }
+ return ln.inode.Child(name, replace)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "strings"
+)
+
+func (fs *customFileSystem) defaultUUID(uuid string) (string, error) {
+ if uuid != "" {
+ return uuid, nil
+ }
+ var resp User
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return "", err
+ }
+ return resp.UUID, nil
+}
+
+// loadOneChild loads only the named child, if it exists.
+func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
+ }
+
+ var contents CollectionList
+ err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{
+ {"name", "=", name},
+ {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}},
+ {"groups.group_class", "=", "project"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if len(contents.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ coll := contents.Items[0]
+
+ if strings.Contains(coll.UUID, "-j7d0g-") {
+ // Group item was loaded into a Collection var -- but
+ // we only need the Name and UUID anyway, so it's OK.
+ return fs.newProjectNode(parent, coll.Name, coll.UUID), nil
+ } else if strings.Contains(coll.UUID, "-4zz18-") {
+ return deferredCollectionFS(fs, parent, coll), nil
+ } else {
+ log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID)
+ return nil, ErrInvalidArgument
+ }
+}
+
+func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, error) {
+ uuid, err := fs.defaultUUID(uuid)
+ if err != nil {
+ return nil, err
+ }
+
+ var inodes []inode
+
+ // Note: the "filters" slice's backing array might be reused
+ // by append(filters,...) below. This isn't goroutine safe,
+ // but all accesses are in the same goroutine, so it's OK.
+ filters := []Filter{{"owner_uuid", "=", uuid}}
+ params := ResourceListParams{
+ Count: "none",
+ Filters: filters,
+ Order: "uuid",
+ }
+ for {
+ var resp CollectionList
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return nil, err
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, i := range resp.Items {
+ coll := i
+ if !permittedName(coll.Name) {
+ continue
+ }
+ inodes = append(inodes, deferredCollectionFS(fs, parent, coll))
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+
+ filters = append(filters, Filter{"group_class", "=", "project"})
+ params.Filters = filters
+ for {
+ var resp GroupList
+ err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if err != nil {
+ return nil, err
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, group := range resp.Items {
+ if !permittedName(group.Name) {
+ continue
+ }
+ inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID))
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+ return inodes, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+type spiedRequest struct {
+ method string
+ path string
+ params map[string]interface{}
+}
+
+type spyingClient struct {
+ *Client
+ calls []spiedRequest
+}
+
+func (sc *spyingClient) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
+ var paramsCopy map[string]interface{}
+ var buf bytes.Buffer
+ json.NewEncoder(&buf).Encode(params)
+ json.NewDecoder(&buf).Decode(¶msCopy)
+ sc.calls = append(sc.calls, spiedRequest{
+ method: method,
+ path: path,
+ params: paramsCopy,
+ })
+ return sc.Client.RequestAndDecode(dst, method, path, body, params)
+}
+
+func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
+ s.fs.MountProject("home", "")
+ s.testHomeProject(c, "/home")
+}
+
+func (s *SiteFSSuite) TestUsersDir(c *check.C) {
+ s.testHomeProject(c, "/users/active")
+}
+
+func (s *SiteFSSuite) testHomeProject(c *check.C, path string) {
+ f, err := s.fs.Open(path)
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Not(check.Equals), 0)
+
+ ok := false
+ for _, fi := range fis {
+ c.Check(fi.Name(), check.Not(check.Equals), "")
+ if fi.Name() == "A Project" {
+ ok = true
+ }
+ }
+ c.Check(ok, check.Equals, true)
+
+ f, err = s.fs.Open(path + "/A Project/..")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+ _, basename := filepath.Split(path)
+ c.Check(fi.Name(), check.Equals, basename)
+
+ f, err = s.fs.Open(path + "/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ fi, err = f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+
+ for _, nx := range []string{
+ path + "/Unrestricted public data",
+ path + "/Unrestricted public data/does not exist",
+ path + "/A Project/does not exist",
+ } {
+ c.Log(nx)
+ f, err = s.fs.Open(nx)
+ c.Check(err, check.NotNil)
+ c.Check(os.IsNotExist(err), check.Equals, true)
+ }
+}
+
+func (s *SiteFSSuite) TestProjectReaddirAfterLoadOne(c *check.C) {
+ f, err := s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project/Project does not exist")
+ c.Assert(err, check.NotNil)
+ f, err = s.fs.Open("/users/active/A Project/A Subproject")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ f, err = s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ defer f.Close()
+ fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fis)
+ var foundSubproject, foundCollection bool
+ for _, fi := range fis {
+ switch fi.Name() {
+ case "A Subproject":
+ foundSubproject = true
+ case "collection_to_move_around":
+ foundCollection = true
+ }
+ }
+ c.Check(foundSubproject, check.Equals, true)
+ c.Check(foundCollection, check.Equals, true)
+}
+
+func (s *SiteFSSuite) TestSlashInName(c *check.C) {
+ badCollection := Collection{
+ Name: "bad/collection",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err := s.client.RequestAndDecode(&badCollection, "POST", "arvados/v1/collections", s.client.UpdateBody(&badCollection), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+badCollection.UUID, nil, nil)
+
+ badProject := Group{
+ Name: "bad/project",
+ GroupClass: "project",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&badProject, "POST", "arvados/v1/groups", s.client.UpdateBody(&badProject), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/"+badProject.UUID, nil, nil)
+
+ dir, err := s.fs.Open("/users/active/A Project")
+ c.Assert(err, check.IsNil)
+ fis, err := dir.Readdir(-1)
+ c.Check(err, check.IsNil)
+ for _, fi := range fis {
+ c.Logf("fi.Name() == %q", fi.Name())
+ c.Check(strings.Contains(fi.Name(), "/"), check.Equals, false)
+ }
+}
+
+func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) {
+ s.fs.MountProject("home", "")
+
+ project, err := s.fs.OpenFile("/home/A Project", 0, 0)
+ c.Assert(err, check.IsNil)
+
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+
+ oob := Collection{
+ Name: "oob",
+ OwnerUUID: arvadostest.AProjectUUID,
+ }
+ err = s.client.RequestAndDecode(&oob, "POST", "arvados/v1/collections", s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ f, err := s.fs.Open("/home/A Project/oob")
+ c.Assert(err, check.IsNil)
+ fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
+ c.Check(fi.IsDir(), check.Equals, true)
+ f.Close()
+
+ wf, err := s.fs.OpenFile("/home/A Project/oob/test.txt", os.O_CREATE|os.O_RDWR, 0700)
+ c.Assert(err, check.IsNil)
+ _, err = wf.Write([]byte("hello oob\n"))
+ c.Check(err, check.IsNil)
+ err = wf.Close()
+ c.Check(err, check.IsNil)
+
+ // Delete test.txt behind s.fs's back by updating the
+ // collection record with the old (empty) ManifestText.
+ err = s.client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+oob.UUID, s.client.UpdateBody(&oob), nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob/test.txt")
+ c.Check(err, check.NotNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.IsNil)
+
+ err = s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+oob.UUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ err = project.Sync()
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open("/home/A Project/oob")
+ c.Check(err, check.NotNil)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "strings"
+ "sync"
+ "time"
+)
+
+type CustomFileSystem interface {
+ FileSystem
+ MountByID(mount string)
+ MountProject(mount, uuid string)
+ MountUsers(mount string)
+}
+
+type customFileSystem struct {
+ fileSystem
+ root *vdirnode
+
+ staleThreshold time.Time
+ staleLock sync.Mutex
+}
+
+func (c *Client) CustomFileSystem(kc keepClient) CustomFileSystem {
+ root := &vdirnode{}
+ fs := &customFileSystem{
+ root: root,
+ fileSystem: fileSystem{
+ fsBackend: keepBackend{apiClient: c, keepClient: kc},
+ root: root,
+ },
+ }
+ root.inode = &treenode{
+ fs: fs,
+ parent: root,
+ fileinfo: fileinfo{
+ name: "/",
+ mode: os.ModeDir | 0755,
+ modTime: time.Now(),
+ },
+ inodes: make(map[string]inode),
+ }
+ return fs
+}
+
+func (fs *customFileSystem) MountByID(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return &vdirnode{
+ inode: &treenode{
+ fs: fs,
+ parent: fs.root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: mount,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ create: fs.mountByID,
+ }, nil
+ })
+}
+
+func (fs *customFileSystem) MountProject(mount, uuid string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return fs.newProjectNode(fs.root, mount, uuid), nil
+ })
+}
+
+func (fs *customFileSystem) MountUsers(mount string) {
+ fs.root.inode.Child(mount, func(inode) (inode, error) {
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: fs.usersLoadOne,
+ loadAll: fs.usersLoadAll,
+ inode: &treenode{
+ fs: fs,
+ parent: fs.root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: mount,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }, nil
+ })
+}
+
+// SiteFileSystem returns a FileSystem that maps collections and other
+// Arvados objects onto a filesystem layout.
+//
+// This is experimental: the filesystem layout is not stable, and
+// there are significant known bugs and shortcomings. For example,
+// writes are not persisted until Sync() is called.
+func (c *Client) SiteFileSystem(kc keepClient) CustomFileSystem {
+ fs := c.CustomFileSystem(kc)
+ fs.MountByID("by_id")
+ fs.MountUsers("users")
+ return fs
+}
+
+func (fs *customFileSystem) Sync() error {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ fs.staleThreshold = time.Now()
+ return nil
+}
+
+// Stale returns true if information obtained at time t should be
+// considered stale.
+func (fs *customFileSystem) Stale(t time.Time) bool {
+ fs.staleLock.Lock()
+ defer fs.staleLock.Unlock()
+ return !fs.staleThreshold.Before(t)
+}
+
+func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
+ return nil, ErrInvalidOperation
+}
+
+func (fs *customFileSystem) mountByID(parent inode, id string) inode {
+ if strings.Contains(id, "-4zz18-") || pdhRegexp.MatchString(id) {
+ return fs.mountCollection(parent, id)
+ } else if strings.Contains(id, "-j7d0g-") {
+ return fs.newProjectNode(fs.root, id, id)
+ } else {
+ return nil
+ }
+}
+
+func (fs *customFileSystem) mountCollection(parent inode, id string) inode {
+ var coll Collection
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+id, nil, nil)
+ if err != nil {
+ return nil
+ }
+ cfs, err := coll.FileSystem(fs, fs)
+ if err != nil {
+ return nil
+ }
+ root := cfs.rootnode()
+ root.SetParent(parent, id)
+ return root
+}
+
+func (fs *customFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &lookupnode{
+ stale: fs.Stale,
+ loadOne: func(parent inode, name string) (inode, error) { return fs.projectsLoadOne(parent, uuid, name) },
+ loadAll: func(parent inode) ([]inode, error) { return fs.projectsLoadAll(parent, uuid) },
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
+// vdirnode wraps an inode by ignoring any requests to add/replace
+// children, and calling a create() func when a non-existing child is
+// looked up.
+//
+// create() can return either a new node, which will be added to the
+// treenode, or nil for ENOENT.
+type vdirnode struct {
+ inode
+ create func(parent inode, name string) inode
+}
+
+func (vn *vdirnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
+ return vn.inode.Child(name, func(existing inode) (inode, error) {
+ if existing == nil && vn.create != nil {
+ existing = vn.create(vn, name)
+ if existing != nil {
+ existing.SetParent(vn, name)
+ vn.inode.(*treenode).fileinfo.modTime = time.Now()
+ }
+ }
+ if replace == nil {
+ return existing, nil
+ } else if tryRepl, err := replace(existing); err != nil {
+ return existing, err
+ } else if tryRepl != existing {
+ return existing, ErrInvalidArgument
+ } else {
+ return existing, nil
+ }
+ })
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "net/http"
+ "os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&SiteFSSuite{})
+
+type SiteFSSuite struct {
+ client *Client
+ fs CustomFileSystem
+ kc keepClient
+}
+
+func (s *SiteFSSuite) SetUpTest(c *check.C) {
+ s.client = &Client{
+ APIHost: os.Getenv("ARVADOS_API_HOST"),
+ AuthToken: arvadostest.ActiveToken,
+ Insecure: true,
+ }
+ s.kc = &keepClientStub{
+ blocks: map[string][]byte{
+ "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
+ }}
+ s.fs = s.client.SiteFileSystem(s.kc)
+}
+
+func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
+ _, ok := s.fs.(http.FileSystem)
+ c.Check(ok, check.Equals, true)
+}
+
+func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(len(fis), check.Equals, 0)
+}
+
+func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
+ f, err := s.fs.Open("/by_id")
+ c.Assert(err, check.IsNil)
+ fis, err := f.Readdir(-1)
+ c.Check(err, check.IsNil)
+ c.Check(len(fis), check.Equals, 0)
+
+ err = s.fs.Mkdir("/by_id/"+arvadostest.FooCollection, 0755)
+ c.Check(err, check.Equals, os.ErrExist)
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.NonexistentCollection)
+ c.Assert(err, check.Equals, os.ErrNotExist)
+
+ for _, path := range []string{
+ arvadostest.FooCollection,
+ arvadostest.FooPdh,
+ arvadostest.AProjectUUID + "/" + arvadostest.FooCollectionName,
+ } {
+ f, err = s.fs.Open("/by_id/" + path)
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(-1)
+ var names []string
+ for _, fi := range fis {
+ names = append(names, fi.Name())
+ }
+ c.Check(names, check.DeepEquals, []string{"foo"})
+ }
+
+ f, err = s.fs.Open("/by_id/" + arvadostest.AProjectUUID + "/A Subproject/baz_file")
+ c.Assert(err, check.IsNil)
+ fis, err = f.Readdir(-1)
+ var names []string
+ for _, fi := range fis {
+ names = append(names, fi.Name())
+ }
+ c.Check(names, check.DeepEquals, []string{"baz"})
+
+ _, err = s.fs.OpenFile("/by_id/"+arvadostest.NonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
+ c.Check(err, check.Equals, ErrInvalidOperation)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection, "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ _, err = s.fs.Stat("/by_id/beep")
+ c.Check(err, check.Equals, os.ErrNotExist)
+ err = s.fs.Rename("/by_id/"+arvadostest.FooCollection+"/foo", "/by_id/"+arvadostest.FooCollection+"/bar")
+ c.Check(err, check.IsNil)
+
+ err = s.fs.Rename("/by_id", "/beep")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+)
+
+func (fs *customFileSystem) usersLoadOne(parent inode, name string) (inode, error) {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, ResourceListParams{
+ Count: "none",
+ Filters: []Filter{{"username", "=", name}},
+ })
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return nil, os.ErrNotExist
+ }
+ user := resp.Items[0]
+ return fs.newProjectNode(parent, user.Username, user.UUID), nil
+}
+
+func (fs *customFileSystem) usersLoadAll(parent inode) ([]inode, error) {
+ params := ResourceListParams{
+ Count: "none",
+ Order: "uuid",
+ }
+ var inodes []inode
+ for {
+ var resp UserList
+ err := fs.RequestAndDecode(&resp, "GET", "arvados/v1/users", nil, params)
+ if err != nil {
+ return nil, err
+ } else if len(resp.Items) == 0 {
+ return inodes, nil
+ }
+ for _, user := range resp.Items {
+ if user.Username == "" {
+ continue
+ }
+ inodes = append(inodes, fs.newProjectNode(parent, user.Username, user.UUID))
+ }
+ params.Filters = []Filter{{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}}
+ }
+}
// Group is an arvados#group record
type Group struct {
- UUID string `json:"uuid,omitempty"`
- Name string `json:"name,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
+ UUID string `json:"uuid,omitempty"`
+ Name string `json:"name,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+ GroupClass string `json:"group_class"`
}
// GroupList is an arvados#groupList resource.
Offset int `json:"offset"`
Limit int `json:"limit"`
}
+
+func (g Group) resourceName() string {
+ return "group"
+}
scanner := bufio.NewScanner(resp.Body)
sawEOF := false
for scanner.Scan() {
+ if scanner.Err() != nil {
+ // If we encounter a read error (timeout,
+ // connection failure), stop now and return it
+ // below, so it doesn't get masked by the
+ // ensuing "badly formatted response" error.
+ break
+ }
if sawEOF {
return nil, fmt.Errorf("Index response contained non-terminal blank line")
}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "net/http"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&KeepServiceSuite{})
+
+type KeepServiceSuite struct{}
+
+func (*KeepServiceSuite) TestIndexTimeout(c *check.C) {
+ client := &Client{
+ Client: &http.Client{
+ Transport: &timeoutTransport{response: []byte("\n")},
+ },
+ APIHost: "zzzzz.arvadosapi.com",
+ AuthToken: "xyzzy",
+ }
+ _, err := (&KeepService{}).IndexMount(client, "fake", "")
+ c.Check(err, check.ErrorMatches, `.*timeout.*`)
+}
// Number of retries
Retries int
+
+ // X-Request-Id for outgoing requests
+ RequestID string
}
var CertFiles = []string{
// Add api token header
req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
+ if c.RequestID != "" {
+ req.Header.Add("X-Request-Id", c.RequestID)
+ }
if c.External {
req.Header.Add("X-External-Client", "1")
}
FederatedActiveUserUUID = "zbbbb-tpzed-xurymjxw79nv3jz"
SpectatorUserUUID = "zzzzz-tpzed-l1s2piq4t4mps8r"
UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
+ FooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
FooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
NonexistentCollection = "zzzzz-4zz18-totallynotexist"
FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
+ AProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
+
FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
defer os.Chdir(cwd)
chdirToPythonTests()
- bgRun(exec.Command("python", "run_test_server.py", "stop"))
+ cmd := exec.Command("python", "run_test_server.py", "stop")
+ bgRun(cmd)
+ // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
+ cmd.Wait()
}
// StartKeep starts the given number of keep servers,
chdirToPythonTests()
cmd := exec.Command("python", "run_test_server.py", "stop_keep", "--num-keep-servers", strconv.Itoa(numKeepServers))
- cmd.Stdin = nil
- cmd.Stderr = os.Stderr
- cmd.Stdout = os.Stderr
- if err := cmd.Run(); err != nil {
- log.Fatalf("%+v: %s", cmd.Args, err)
- }
+ bgRun(cmd)
+ // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
+ cmd.Wait()
}
// Start cmd, with stderr and stdout redirected to our own
gen := &IDGenerator{Prefix: "req-"}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Header.Get("X-Request-Id") == "" {
+ if req.Header == nil {
+ req.Header = http.Header{}
+ }
req.Header.Set("X-Request-Id", gen.Next())
}
h.ServeHTTP(w, req)
"remoteAddr": req.RemoteAddr,
"reqForwardedFor": req.Header.Get("X-Forwarded-For"),
"reqMethod": req.Method,
+ "reqHost": req.Host,
"reqPath": req.URL.Path[1:],
+ "reqQuery": req.URL.RawQuery,
"reqBytes": req.ContentLength,
})
logRequest(w, req, lgr)
}
func (w *responseWriter) Write(data []byte) (n int, err error) {
+ if w.wroteStatus == 0 {
+ w.WriteHeader(http.StatusOK)
+ }
n, err = w.ResponseWriter.Write(data)
w.wroteBodyBytes += n
w.err = err
func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
- hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ data := []byte("TestOverrideDiscovery")
+ hash := fmt.Sprintf("%x+%d", md5.Sum(data), len(data))
st := StubGetHandler{
c,
hash,
arvadostest.ActiveToken,
http.StatusOK,
- []byte("TestOverrideDiscovery")}
+ data}
ks := RunSomeFakeKeepServers(st, 2)
os.Setenv("ARVADOS_KEEP_SERVICES", "")
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/asyncbuf"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
// A Keep "block" is 64MB.
HTTPClient HTTPClient
Retries int
BlockCache *BlockCache
+ RequestID string
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ reqid := kc.getRequestID()
+
+ var expectLength int64
+ if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
+ expectLength = -1
+ } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil {
+ expectLength = -1
+ } else {
+ expectLength = n
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
continue
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+ req.Header.Add("X-Request-Id", reqid)
resp, err := kc.httpClient().Do(req)
if err != nil {
// Probably a network error, may be transient,
// can try again.
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
retryList = append(retryList, host)
- } else if resp.StatusCode != http.StatusOK {
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
var respbody []byte
respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
resp.Body.Close()
} else if resp.StatusCode == 404 {
count404++
}
- } else if resp.ContentLength < 0 {
- // Missing Content-Length
- resp.Body.Close()
- return nil, 0, "", fmt.Errorf("Missing Content-Length of block")
- } else {
- // Success.
- if method == "GET" {
- return HashCheckingReader{
- Reader: resp.Body,
- Hash: md5.New(),
- Check: locator[0:32],
- }, resp.ContentLength, url, nil
- } else {
+ continue
+ }
+ if expectLength < 0 {
+ if resp.ContentLength < 0 {
resp.Body.Close()
- return nil, resp.ContentLength, url, nil
+ return nil, 0, "", fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator)
}
+ expectLength = resp.ContentLength
+ } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength {
+ resp.Body.Close()
+ return nil, 0, "", fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength)
+ }
+ // Success
+ if method == "GET" {
+ return HashCheckingReader{
+ Reader: resp.Body,
+ Hash: md5.New(),
+ Check: locator[0:32],
+ }, expectLength, url, nil
+ } else {
+ resp.Body.Close()
+ return nil, expectLength, url, nil
}
-
}
serversToTry = retryList
}
return nil, err
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+ req.Header.Set("X-Request-Id", kc.getRequestID())
resp, err := kc.httpClient().Do(req)
if err != nil {
return nil, err
return c
}
+var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
+
+func (kc *KeepClient) getRequestID() string {
+ if kc.RequestID != "" {
+ return kc.RequestID
+ } else {
+ return reqIDGen.Next()
+ }
+}
+
type Locator struct {
Hash string
Size int // -1 if data size is not known
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
<-st.handled
type FailThenSucceedHandler struct {
handled chan string
count int
- successhandler StubGetHandler
+ successhandler http.Handler
+ reqIDs []string
}
func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
if fh.count == 0 {
resp.WriteHeader(500)
fh.count += 1
func(kc *KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
func (s *StandaloneSuite) TestGetFailRetry(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := &FailThenSucceedHandler{make(chan string, 1), 0,
- StubGetHandler{
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubGetHandler{
c,
hash,
"abc123",
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
c.Check(content, DeepEquals, []byte("foo"))
+
+ c.Logf("%q", st.reqIDs)
+ c.Assert(len(st.reqIDs) > 1, Equals, true)
+ for _, reqid := range st.reqIDs {
+ c.Check(reqid, Not(Equals), "")
+ c.Check(reqid, Equals, st.reqIDs[0])
+ }
}
func (s *StandaloneSuite) TestGetNetError(c *C) {
c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
}
-type FailThenSucceedPutHandler struct {
- handled chan string
- count int
- successhandler StubPutHandler
-}
-
-func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- if h.count == 0 {
- resp.WriteHeader(500)
- h.count += 1
- h.handled <- fmt.Sprintf("http://%s", req.Host)
- } else {
- h.successhandler.ServeHTTP(resp, req)
- }
-}
-
func (s *StandaloneSuite) TestPutBRetry(c *C) {
- st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
- StubPutHandler{
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubPutHandler{
c,
Md5String("foo"),
"abc123",
"io"
"io/ioutil"
"log"
- "math/rand"
"net/http"
"os"
"strings"
}
func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
- upload_status chan<- uploadStatus, expectedLength int64, requestID int32) {
+ upload_status chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
- DebugPrintf("DEBUG: [%08x] Error creating request PUT %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%s] Error creating request PUT %v error: %v", reqid, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
// to be empty, so don't set req.Body.
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
+ req.Header.Add("X-Request-Id", reqid)
+ req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
var resp *http.Response
if resp, err = this.httpClient().Do(req); err != nil {
- DebugPrintf("DEBUG: [%08x] Upload failed %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
- DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, err2.Error(), response)
+ DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, err2.Error(), response)
upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
- DebugPrintf("DEBUG: [%08x] Upload %v success", requestID, url)
+ DebugPrintf("DEBUG: [%s] Upload %v success", reqid, url)
upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
if resp.StatusCode >= 300 && response == "" {
response = resp.Status
}
- DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
+ DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, resp.StatusCode, response)
upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- // Generate an arbitrary ID to identify this specific
- // transaction in debug logs.
- requestID := rand.Int31()
+ reqid := this.getRequestID()
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
for active*replicasPerThread < replicasTodo {
// Start some upload requests
if next_server < len(sv) {
- DebugPrintf("DEBUG: [%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
- go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, requestID)
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[next_server])
+ go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, reqid)
next_server += 1
active += 1
} else {
}
}
}
- DebugPrintf("DEBUG: [%08x] Replicas remaining to write: %v active uploads: %v",
- requestID, replicasTodo, active)
+ DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v",
+ reqid, replicasTodo, active)
// Now wait for something to happen.
if active > 0 {
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
- 'ciso8601 >=1.0.0, <=1.0.4',
+ 'ciso8601 >=1.0.6',
'future',
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.13.7',
+ 'ruamel.yaml >=0.13.11, <0.15',
'setuptools',
'ws4py <0.4',
],
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-cli (0.1.20171211220040)
+ arvados-cli (1.1.4.20180412190507)
activesupport (>= 3.2.13, < 5)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1, >= 0.1.20150128223554)
execjs
coffee-script-source (1.12.2)
concurrent-ruby (1.0.5)
- crass (1.0.3)
+ crass (1.0.4)
curb (0.9.4)
- database_cleaner (1.6.2)
+ database_cleaner (1.7.0)
erubis (2.7.0)
- eventmachine (1.2.5)
+ eventmachine (1.2.6)
execjs (2.7.0)
extlib (0.9.16)
factory_girl (4.9.0)
httpclient (2.8.3)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
- jquery-rails (4.3.1)
+ jquery-rails (4.3.3)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
logging (2.2.2)
little-plugger (~> 1.1)
multi_json (~> 1.10)
- lograge (0.9.0)
+ lograge (0.10.0)
actionpack (>= 4)
activesupport (>= 4)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.2.0)
+ loofah (2.2.2)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.0)
mini_mime (1.0.0)
mini_portile2 (2.3.0)
minitest (5.11.3)
- mocha (1.3.0)
+ mocha (1.5.0)
metaclass (~> 0.0.1)
multi_json (1.13.1)
multi_xml (0.6.0)
oauth2 (~> 1.1)
omniauth (~> 1.2)
os (0.9.6)
- passenger (5.2.1)
+ passenger (5.3.0)
rack
rake (>= 0.8.1)
pg (0.21.0)
protected_attributes (1.1.4)
activemodel (>= 4.0.1, < 5.0)
public_suffix (3.0.2)
- rack (1.6.9)
+ rack (1.6.10)
rack-test (0.6.3)
rack (>= 1.0)
rails (4.2.10)
activesupport (>= 4.2.0, < 5.0)
nokogiri (~> 1.6)
rails-deprecated_sanitizer (>= 1.0.1)
- rails-html-sanitizer (1.0.3)
- loofah (~> 2.0)
+ rails-html-sanitizer (1.0.4)
+ loofah (~> 2.2, >= 2.2.2)
rails-observers (0.1.5)
activemodel (>= 4.0)
railties (4.2.10)
activesupport (= 4.2.10)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
- rake (12.3.0)
+ rake (12.3.1)
ref (2.0.0)
- request_store (1.4.0)
+ request_store (1.4.1)
rack (>= 1.4)
responders (2.4.0)
actionpack (>= 4.2.0, < 5.3)
def find_object_by_uuid
if loc = Keep::Locator.parse(params[:id])
loc.strip_hints!
- if c = Collection.readable_by(*@read_users).where({ portable_data_hash: loc.to_s }).limit(1).first
+
+ # It matters which Collection object we pick because we use it to get signed_manifest_text,
+ # the value of which is affected by the value of trash_at.
+ #
+ # From postgres doc: "By default, null values sort as if larger than any non-null
+ # value; that is, NULLS FIRST is the default for DESC order, and
+ # NULLS LAST otherwise."
+ #
+ # "trash_at desc" sorts null first, then latest to earliest, so
+ # it will select the Collection object with the longest
+ # available lifetime.
+
+ if c = Collection.readable_by(*@read_users).where({ portable_data_hash: loc.to_s }).order("trash_at desc").limit(1).first
@object = {
uuid: c.portable_data_hash,
portable_data_hash: c.portable_data_hash,
accept_attribute_as_json :prefs, Hash
skip_before_filter :find_object_by_uuid, only:
- [:activate, :current, :system, :setup]
+ [:activate, :current, :system, :setup, :merge]
skip_before_filter :render_404_if_no_object, only:
- [:activate, :current, :system, :setup]
+ [:activate, :current, :system, :setup, :merge]
before_filter :admin_required, only: [:setup, :unsetup, :update_uuid]
def current
show
end
+ def merge
+ if !Thread.current[:api_client].andand.is_trusted
+ return send_error("supplied API token is not from a trusted client", status: 403)
+ elsif Thread.current[:api_client_authorization].scopes != ['all']
+ return send_error("cannot merge with a scoped token", status: 403)
+ end
+
+ new_auth = ApiClientAuthorization.validate(token: params[:new_user_token])
+ if !new_auth
+ return send_error("invalid new_user_token", status: 401)
+ end
+ if !new_auth.api_client.andand.is_trusted
+ return send_error("supplied new_user_token is not from a trusted client", status: 403)
+ elsif new_auth.scopes != ['all']
+ return send_error("supplied new_user_token has restricted scope", status: 403)
+ end
+ new_user = new_auth.user
+
+ if current_user.uuid == new_user.uuid
+ return send_error("cannot merge user to self", status: 422)
+ end
+
+ if !new_user.can?(write: params[:new_owner_uuid])
+ return send_error("cannot move objects into supplied new_owner_uuid: new user does not have write permission", status: 403)
+ end
+
+ redirect = params[:redirect_to_new_user]
+ if !redirect
+ return send_error("merge with redirect_to_new_user=false is not yet supported", status: 422)
+ end
+
+ @object = current_user
+ act_as_system_user do
+ @object.merge(new_owner_uuid: params[:new_owner_uuid], redirect_to_user_uuid: redirect && new_user.uuid)
+ end
+ show
+ end
+
protected
+ def self._merge_requires_parameters
+ {
+ new_owner_uuid: {
+ type: 'string', required: true,
+ },
+ new_user_token: {
+ type: 'string', required: true,
+ },
+ redirect_to_new_user: {
+ type: 'boolean', required: false,
+ },
+ }
+ end
+
def self._setup_requires_parameters
{
user: {
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name"]
+ safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username"]
if @select
@select = @select & safe_attrs
else
# Only local users can create sessions, hence uuid_like_pattern
# here.
- user = User.where('identity_url = ? and uuid like ?',
- omniauth['info']['identity_url'],
- User.uuid_like_pattern).first
+ user = User.unscoped.where('identity_url = ? and uuid like ?',
+ omniauth['info']['identity_url'],
+ User.uuid_like_pattern).first
if not user
# Check for permission to log in to an existing User record with
# a different identity_url
end
end
end
+
if not user
# New user registration
user = User.new(:email => omniauth['info']['email'],
# First login to a pre-activated account
user.identity_url = omniauth['info']['identity_url']
end
+
+ while (uuid = user.redirect_to_user_uuid)
+ user = User.where(uuid: uuid).first
+ if !user
+ raise Exception.new("identity_url #{omniauth['info']['identity_url']} redirects to nonexistent uuid #{uuid}")
+ end
+ end
end
# For the benefit of functional and integration tests:
uuid_prefix+".arvadosapi.com")
end
- def self.validate(token:, remote:)
+ def self.validate(token:, remote: nil)
return nil if !token
remote ||= Rails.configuration.uuid_prefix
has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
has_many :repositories, foreign_key: :owner_uuid, primary_key: :uuid
+ default_scope { where('redirect_to_user_uuid is null') }
+
api_accessible :user, extend: :common do |t|
t.add :email
t.add :username
old_uuid = self.uuid
self.uuid = new_uuid
save!(validate: false)
+ change_all_uuid_refs(old_uuid: old_uuid, new_uuid: new_uuid)
+ end
+ end
+
+ # Move this user's (i.e., self's) owned items into new_owner_uuid.
+ # Also redirect future uses of this account to
+ # redirect_to_user_uuid, i.e., when a caller authenticates to this
+ # account in the future, the account redirect_to_user_uuid account
+ # will be used instead.
+ #
+ # current_user must have admin privileges, i.e., the caller is
+ # responsible for checking permission to do this.
+ def merge(new_owner_uuid:, redirect_to_user_uuid:)
+ raise PermissionDeniedError if !current_user.andand.is_admin
+ raise "not implemented" if !redirect_to_user_uuid
+ transaction(requires_new: true) do
+ reload
+ raise "cannot merge an already merged user" if self.redirect_to_user_uuid
+
+ new_user = User.where(uuid: redirect_to_user_uuid).first
+ raise "user does not exist" if !new_user
+ raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
+
+ # Existing API tokens are updated to authenticate to the new
+ # user.
+ ApiClientAuthorization.
+ where(user_id: id).
+ update_all(user_id: new_user.id)
+
+ # References to the old user UUID in the context of a user ID
+ # (rather than a "home project" in the project hierarchy) are
+ # updated to point to the new user.
+ [
+ [AuthorizedKey, :owner_uuid],
+ [AuthorizedKey, :authorized_user_uuid],
+ [Repository, :owner_uuid],
+ [Link, :owner_uuid],
+ [Link, :tail_uuid],
+ [Link, :head_uuid],
+ ].each do |klass, column|
+ klass.where(column => uuid).update_all(column => new_user.uuid)
+ end
+
+ # References to the merged user's "home project" are updated to
+ # point to new_owner_uuid.
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
- klass.columns.each do |col|
- if col.name.end_with?('_uuid')
- column = col.name.to_sym
- klass.where(column => old_uuid).update_all(column => new_uuid)
- end
- end
+ next if [ApiClientAuthorization,
+ AuthorizedKey,
+ Link,
+ Log,
+ Repository].include?(klass)
+ next if !klass.columns.collect(&:name).include?('owner_uuid')
+ klass.where(owner_uuid: uuid).update_all(owner_uuid: new_owner_uuid)
end
+
+ update_attributes!(redirect_to_user_uuid: new_user.uuid)
+ invalidate_permissions_cache
end
end
protected
+ def change_all_uuid_refs(old_uuid:, new_uuid:)
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+ klass.columns.each do |col|
+ if col.name.end_with?('_uuid')
+ column = col.name.to_sym
+ klass.where(column => old_uuid).update_all(column => new_uuid)
+ end
+ end
+ end
+ end
+
def ensure_ownership_path_leads_to_user
true
end
def permission_to_update
- if username_changed?
+ if username_changed? || redirect_to_user_uuid_changed?
current_user.andand.is_admin
else
# users must be able to update themselves (even if they are
def permission_to_create
current_user.andand.is_admin or
- (self == current_user and
+ (self == current_user &&
+ self.redirect_to_user_uuid.nil? &&
self.is_active == Rails.configuration.new_users_are_active)
end
end
end
+ # Redact new_user_token param in /arvados/v1/users/merge
+ # request. Log the auth UUID instead, if the token exists.
+ if params['new_user_token'].is_a? String
+ params['new_user_token_uuid'] =
+ ApiClientAuthorization.
+ where('api_token = ?', params['new_user_token']).
+ first.andand.uuid
+ params['new_user_token'] = '[...]'
+ end
+
params_s = SafeJSON.dump(params)
if params_s.length > Rails.configuration.max_request_log_params_size
payload[:params_truncated] = params_s[0..Rails.configuration.max_request_log_params_size] + "[...]"
post 'setup', on: :collection
post 'unsetup', on: :member
post 'update_uuid', on: :member
+ post 'merge', on: :collection
end
resources :virtual_machines do
get 'logins', on: :member
--- /dev/null
+class AddRedirectToUserUuidToUsers < ActiveRecord::Migration
+ def up
+ add_column :users, :redirect_to_user_uuid, :string
+ User.reset_column_information
+ remove_index :users, name: 'users_search_index'
+ add_index :users, User.searchable_columns('ilike') - ['prefs'], name: 'users_search_index'
+ end
+
+ def down
+ remove_index :users, name: 'users_search_index'
+ remove_column :users, :redirect_to_user_uuid
+ User.reset_column_information
+ add_index :users, User.searchable_columns('ilike') - ['prefs'], name: 'users_search_index'
+ end
+end
updated_at timestamp without time zone NOT NULL,
default_owner_uuid character varying(255),
is_active boolean DEFAULT false,
- username character varying(255)
+ username character varying(255),
+ redirect_to_user_uuid character varying
);
-- Name: users_search_index; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX users_search_index ON users USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, email, first_name, last_name, identity_url, default_owner_uuid, username);
+CREATE INDEX users_search_index ON users USING btree (uuid, owner_uuid, modified_by_client_uuid, modified_by_user_uuid, email, first_name, last_name, identity_url, default_owner_uuid, username, redirect_to_user_uuid);
--
INSERT INTO schema_migrations (version) VALUES ('20180313180114');
+INSERT INTO schema_migrations (version) VALUES ('20180501182859');
+
replication_confirmed: 2
updated_at: 2015-02-07 00:24:52.983381227 Z
uuid: zzzzz-4zz18-434zv1tnnf2rygp
- manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:3:foo 3:6:bar\n"
+ manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:3:foo 3:3:bar\n"
name: replication want=2 have=2
storage_classes_desired_default_unconfirmed:
end
end
+ test "refuse to merge with redirect_to_user_uuid=false (not yet supported)" do
+ authorize_with :project_viewer_trustedclient
+ post :merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: false,
+ }
+ assert_response(422)
+ end
+
+ test "refuse to merge user into self" do
+ authorize_with(:active_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ end
+
+ [[:active, :project_viewer_trustedclient],
+ [:active_trustedclient, :project_viewer]].each do |src, dst|
+ test "refuse to merge with untrusted token (#{src} -> #{dst})" do
+ authorize_with(src)
+ post(:merge, {
+ new_user_token: api_client_authorizations(dst).api_token,
+ new_owner_uuid: api_client_authorizations(dst).user.uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(403)
+ end
+ end
+
+ [[:expired_trustedclient, :project_viewer_trustedclient],
+ [:project_viewer_trustedclient, :expired_trustedclient]].each do |src, dst|
+ test "refuse to merge with expired token (#{src} -> #{dst})" do
+ authorize_with(src)
+ post(:merge, {
+ new_user_token: api_client_authorizations(dst).api_token,
+ new_owner_uuid: api_client_authorizations(dst).user.uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(401)
+ end
+ end
+
+ [['src', :active_trustedclient],
+ ['dst', :project_viewer_trustedclient]].each do |which_scoped, auth|
+ test "refuse to merge with scoped #{which_scoped} token" do
+ act_as_system_user do
+ api_client_authorizations(auth).update_attributes(scopes: ["GET /", "POST /", "PUT /"])
+ end
+ authorize_with(:active_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:project_viewer_trustedclient).api_token,
+ new_owner_uuid: users(:project_viewer).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(403)
+ end
+ end
+
+ test "refuse to merge if new_owner_uuid is not writable" do
+ authorize_with(:project_viewer_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ new_owner_uuid: groups(:anonymously_accessible_project).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(403)
+ end
+
+ test "refuse to merge if new_owner_uuid is empty" do
+ authorize_with(:project_viewer_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ new_owner_uuid: "",
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ end
+
+ test "refuse to merge if new_owner_uuid is not provided" do
+ authorize_with(:project_viewer_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ end
+
+ test "refuse to update redirect_to_user_uuid directly" do
+ authorize_with(:active_trustedclient)
+ patch(:update, {
+ id: users(:active).uuid,
+ user: {
+ redirect_to_user_uuid: users(:active).uuid,
+ },
+ })
+ assert_response(403)
+ end
+
+ test "merge 'project_viewer' account into 'active' account" do
+ authorize_with(:project_viewer_trustedclient)
+ post(:merge, {
+ new_user_token: api_client_authorizations(:active_trustedclient).api_token,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(:success)
+ assert_equal(users(:project_viewer).redirect_to_user_uuid, users(:active).uuid)
+
+ auth = ApiClientAuthorization.validate(token: api_client_authorizations(:project_viewer).api_token)
+ assert_not_nil(auth)
+ assert_not_nil(auth.user)
+ assert_equal(users(:active).uuid, auth.user.uuid)
+ end
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
- "last_name"].sort
+ "last_name", "username"].sort
def check_non_admin_index
assert_response :success
end
nil
end
+
+ test 'merge active into project_viewer account' do
+ post('/arvados/v1/groups', {
+ group: {
+ group_class: 'project',
+ name: "active user's stuff",
+ },
+ }, auth(:project_viewer))
+ assert_response(:success)
+ project_uuid = json_response['uuid']
+
+ post('/arvados/v1/users/merge', {
+ new_user_token: api_client_authorizations(:project_viewer_trustedclient).api_token,
+ new_owner_uuid: project_uuid,
+ redirect_to_new_user: true,
+ }, auth(:active_trustedclient))
+ assert_response(:success)
+
+ get('/arvados/v1/users/current', {}, auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['uuid'])
+
+ get('/arvados/v1/authorized_keys/' + authorized_keys(:active).uuid, {}, auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal(users(:project_viewer).uuid, json_response['authorized_user_uuid'])
+
+ get('/arvados/v1/repositories/' + repositories(:foo).uuid, {}, auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+
+ get('/arvados/v1/groups/' + groups(:aproject).uuid, {}, auth(:active))
+ assert_response(:success)
+ assert_equal(project_uuid, json_response['owner_uuid'])
+ end
end
replacing.nice = n
newq[uuid] = replacing
- if state == "PENDING" && reason == "BadConstraints" && p == 0 && replacing.wantPriority > 0 {
+ if state == "PENDING" && ((reason == "BadConstraints" && p == 0) || reason == "launch failed requeued held") && replacing.wantPriority > 0 {
// When using SLURM 14.x or 15.x, our queued
// jobs land in this state when "scontrol
// reconfigure" invalidates their feature
// reappeared, so rather than second-guessing
// whether SLURM is ready, we just keep trying
// this until it works.
+ //
+ // "launch failed requeued held" seems to be
+ // another manifestation of this problem,
+ // resolved the same way.
+ log.Printf("releasing held job %q", uuid)
sqc.Slurm.Release(uuid)
+ } else if p < 1<<20 && replacing.wantPriority > 0 {
+ log.Printf("warning: job %q has low priority %d, nice %d, state %q, reason %q", uuid, p, n, state, reason)
}
}
sqc.queue = newq
return 0, errors.New("not implemented")
}
+func (fw FileWrapper) Sync() error {
+ return errors.New("not implemented")
+}
+
func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
if filename == hwImageId+".tar" {
rdr := ioutil.NopCloser(&bytes.Buffer{})
],
install_requires=[
'arvados-python-client >= 0.1.20151118035730',
- 'llfuse>=1.2',
+ # llfuse 1.3.4 fails to install via pip
+ 'llfuse >=1.2, <1.3.4',
'python-daemon',
- 'ciso8601',
+ 'ciso8601 >=1.0.6',
'setuptools'
],
test_suite='tests',
package main
import (
+ "bytes"
+ "crypto/md5"
"fmt"
"log"
"math"
Dumper *log.Logger
MinMtime int64
- collScanned int
- serviceRoots map[string]string
- errors []error
- mutex sync.Mutex
+ classes []string
+ mounts int
+ mountsByClass map[string]map[*KeepMount]bool
+ collScanned int
+ serviceRoots map[string]string
+ errors []error
+ stats balancerStats
+ mutex sync.Mutex
}
// Run performs a balance operation using the given config and
if err != nil {
return
}
+
for _, srv := range bal.KeepServices {
err = srv.discoverMounts(&config.Client)
if err != nil {
repl = *coll.ReplicationDesired
}
debugf("%v: %d block x%d", coll.UUID, len(blkids), repl)
- bal.BlockStateMap.IncreaseDesired(repl, blkids)
+ bal.BlockStateMap.IncreaseDesired(coll.StorageClassesDesired, repl, blkids)
return nil
}
// This just calls balanceBlock() once for each block, using a
// pool of worker goroutines.
defer timeMe(bal.Logger, "ComputeChangeSets")()
- bal.setupServiceRoots()
+ bal.setupLookupTables()
type balanceTask struct {
blkid arvados.SizedDigest
}
nWorkers := 1 + runtime.NumCPU()
todo := make(chan balanceTask, nWorkers)
+ results := make(chan balanceResult, 16)
var wg sync.WaitGroup
for i := 0; i < nWorkers; i++ {
wg.Add(1)
go func() {
for work := range todo {
- bal.balanceBlock(work.blkid, work.blk)
+ results <- bal.balanceBlock(work.blkid, work.blk)
}
wg.Done()
}()
}
})
close(todo)
- wg.Wait()
+ go func() {
+ wg.Wait()
+ close(results)
+ }()
+ bal.collectStatistics(results)
}
-func (bal *Balancer) setupServiceRoots() {
+func (bal *Balancer) setupLookupTables() {
bal.serviceRoots = make(map[string]string)
+ bal.classes = []string{"default"}
+ bal.mountsByClass = map[string]map[*KeepMount]bool{"default": {}}
+ bal.mounts = 0
for _, srv := range bal.KeepServices {
bal.serviceRoots[srv.UUID] = srv.UUID
+ for _, mnt := range srv.mounts {
+ bal.mounts++
+
+ // All mounts on a read-only service are
+ // effectively read-only.
+ mnt.ReadOnly = mnt.ReadOnly || srv.ReadOnly
+
+ if len(mnt.StorageClasses) == 0 {
+ bal.mountsByClass["default"][mnt] = true
+ continue
+ }
+ for _, class := range mnt.StorageClasses {
+ if mbc := bal.mountsByClass[class]; mbc == nil {
+ bal.classes = append(bal.classes, class)
+ bal.mountsByClass[class] = map[*KeepMount]bool{mnt: true}
+ } else {
+ mbc[mnt] = true
+ }
+ }
+ }
}
+ // Consider classes in lexicographic order to avoid flapping
+ // between balancing runs. The outcome of the "prefer a mount
+ // we're already planning to use for a different storage
+ // class" case in balanceBlock depends on the order classes
+ // are considered.
+ sort.Strings(bal.classes)
}
const (
changeNone: "none",
}
+type balanceResult struct {
+ blk *BlockState
+ blkid arvados.SizedDigest
+ have int
+ want int
+}
+
// balanceBlock compares current state to desired state for a single
// block, and makes the appropriate ChangeSet calls.
-func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) {
+func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) balanceResult {
debugf("balanceBlock: %v %+v", blkid, blk)
- // A slot is somewhere a replica could potentially be trashed
- // from, pulled from, or pulled to. Each KeepService gets
- // either one empty slot, or one or more non-empty slots.
type slot struct {
- srv *KeepService // never nil
- repl *Replica // nil if none found
+ mnt *KeepMount // never nil
+ repl *Replica // replica already stored here (or nil)
+ want bool // we should pull/leave a replica here
}
- // First, we build an ordered list of all slots worth
- // considering (including all slots where replicas have been
- // found, as well as all of the optimal slots for this block).
- // Then, when we consider each slot in that order, we will
- // have all of the information we need to make a decision
- // about that slot.
+ // Build a list of all slots (one per mounted volume).
+ slots := make([]slot, 0, bal.mounts)
+ for _, srv := range bal.KeepServices {
+ for _, mnt := range srv.mounts {
+ var repl *Replica
+ for r := range blk.Replicas {
+ if blk.Replicas[r].KeepMount == mnt {
+ repl = &blk.Replicas[r]
+ }
+ }
+ // Initial value of "want" is "have, and can't
+ // delete". These untrashable replicas get
+ // prioritized when sorting slots: otherwise,
+ // non-optimal readonly copies would cause us
+ // to overreplicate.
+ slots = append(slots, slot{
+ mnt: mnt,
+ repl: repl,
+ want: repl != nil && (mnt.ReadOnly || repl.Mtime >= bal.MinMtime),
+ })
+ }
+ }
uuids := keepclient.NewRootSorter(bal.serviceRoots, string(blkid[:32])).GetSortedRoots()
- rendezvousOrder := make(map[*KeepService]int, len(uuids))
- slots := make([]slot, len(uuids))
+ srvRendezvous := make(map[*KeepService]int, len(uuids))
for i, uuid := range uuids {
srv := bal.KeepServices[uuid]
- rendezvousOrder[srv] = i
- slots[i].srv = srv
- }
-
- // Sort readonly replicas ahead of trashable ones. This way,
- // if a single service has excessive replicas, the ones we
- // encounter last (and therefore choose to delete) will be on
- // the writable volumes, where possible.
- //
- // TODO: within the trashable set, prefer the oldest replica
- // that doesn't have a timestamp collision with others.
- sort.Slice(blk.Replicas, func(i, j int) bool {
- mnt := blk.Replicas[i].KeepMount
- return mnt.ReadOnly || mnt.KeepService.ReadOnly
- })
+ srvRendezvous[srv] = i
+ }
- // Assign existing replicas to slots.
- for ri := range blk.Replicas {
- repl := &blk.Replicas[ri]
- srv := repl.KeepService
- slotIdx := rendezvousOrder[srv]
- if slots[slotIdx].repl != nil {
- // Additional replicas on a single server are
- // considered non-optimal. Within this
- // category, we don't try to optimize layout:
- // we just say the optimal order is the order
- // we encounter them.
- slotIdx = len(slots)
- slots = append(slots, slot{srv: srv})
+ // Below we set underreplicated=true if we find any storage
+ // class that's currently underreplicated -- in that case we
+ // won't want to trash any replicas.
+ underreplicated := false
+
+ unsafeToDelete := make(map[int64]bool, len(slots))
+ for _, class := range bal.classes {
+ desired := blk.Desired[class]
+ if desired == 0 {
+ continue
+ }
+ // Sort the slots by desirability.
+ sort.Slice(slots, func(i, j int) bool {
+ si, sj := slots[i], slots[j]
+ if classi, classj := bal.mountsByClass[class][si.mnt], bal.mountsByClass[class][sj.mnt]; classi != classj {
+ // Prefer a mount that satisfies the
+ // desired class.
+ return bal.mountsByClass[class][si.mnt]
+ } else if wanti, wantj := si.want, si.want; wanti != wantj {
+ // Prefer a mount that will have a
+ // replica no matter what we do here
+ // -- either because it already has an
+ // untrashable replica, or because we
+ // already need it to satisfy a
+ // different storage class.
+ return slots[i].want
+ } else if orderi, orderj := srvRendezvous[si.mnt.KeepService], srvRendezvous[sj.mnt.KeepService]; orderi != orderj {
+ // Prefer a better rendezvous
+ // position.
+ return orderi < orderj
+ } else if repli, replj := si.repl != nil, sj.repl != nil; repli != replj {
+ // Prefer a mount that already has a
+ // replica.
+ return repli
+ } else {
+ // If pull/trash turns out to be
+ // needed, distribute the
+ // new/remaining replicas uniformly
+ // across qualifying mounts on a given
+ // server.
+ return rendezvousLess(si.mnt.DeviceID, sj.mnt.DeviceID, blkid)
+ }
+ })
+
+ // Servers and mounts (with or without existing
+ // replicas) that are part of the best achievable
+ // layout for this storage class.
+ wantSrv := map[*KeepService]bool{}
+ wantMnt := map[*KeepMount]bool{}
+ // Positions (with existing replicas) that have been
+ // protected (via unsafeToDelete) to ensure we don't
+ // reduce replication below desired level when
+ // trashing replicas that aren't optimal positions for
+ // any storage class.
+ protMnt := map[*KeepMount]bool{}
+
+ // trySlot tries using a slot to meet requirements,
+ // and returns true if all requirements are met.
+ trySlot := func(i int) bool {
+ slot := slots[i]
+ if len(protMnt) < desired && slot.repl != nil {
+ unsafeToDelete[slot.repl.Mtime] = true
+ protMnt[slot.mnt] = true
+ }
+ if len(wantMnt) < desired && (slot.repl != nil || !slot.mnt.ReadOnly) {
+ slots[i].want = true
+ wantSrv[slot.mnt.KeepService] = true
+ wantMnt[slot.mnt] = true
+ }
+ return len(protMnt) >= desired && len(wantMnt) >= desired
+ }
+
+ // First try to achieve desired replication without
+ // using the same server twice.
+ done := false
+ for i := 0; i < len(slots) && !done; i++ {
+ if !wantSrv[slots[i].mnt.KeepService] {
+ done = trySlot(i)
+ }
}
- slots[slotIdx].repl = repl
- }
-
- // number of replicas already found in positions better than
- // the position we're contemplating now.
- reportedBestRepl := 0
- // To be safe we assume two replicas with the same Mtime are
- // in fact the same replica being reported more than
- // once. len(uniqueBestRepl) is the number of distinct
- // replicas in the best rendezvous positions we've considered
- // so far.
- uniqueBestRepl := make(map[int64]bool, len(bal.serviceRoots))
- // pulls is the number of Pull changes we have already
- // requested. (For purposes of deciding whether to Pull to
- // rendezvous position N, we should assume all pulls we have
- // requested on rendezvous positions M<N will be successful.)
- pulls := 0
+
+ // If that didn't suffice, do another pass without the
+ // "distinct services" restriction. (Achieving the
+ // desired volume replication on fewer than the
+ // desired number of services is better than
+ // underreplicating.)
+ for i := 0; i < len(slots) && !done; i++ {
+ done = trySlot(i)
+ }
+
+ if !underreplicated {
+ safe := 0
+ for _, slot := range slots {
+ if slot.repl == nil || !bal.mountsByClass[class][slot.mnt] {
+ continue
+ }
+ if safe++; safe >= desired {
+ break
+ }
+ }
+ underreplicated = safe < desired
+ }
+ }
+
+ // TODO: If multiple replicas are trashable, prefer the oldest
+ // replica that doesn't have a timestamp collision with
+ // others.
+
+ var have, want int
+ for _, slot := range slots {
+ if slot.want {
+ want++
+ }
+ if slot.repl != nil {
+ have++
+ }
+ }
+
var changes []string
for _, slot := range slots {
- change := changeNone
- srv, repl := slot.srv, slot.repl
// TODO: request a Touch if Mtime is duplicated.
- if repl != nil {
- // This service has a replica. We should
- // delete it if [1] we already have enough
- // distinct replicas in better rendezvous
- // positions and [2] this replica's Mtime is
- // distinct from all of the better replicas'
- // Mtimes.
- if !srv.ReadOnly &&
- !repl.KeepMount.ReadOnly &&
- repl.Mtime < bal.MinMtime &&
- len(uniqueBestRepl) >= blk.Desired &&
- !uniqueBestRepl[repl.Mtime] {
- srv.AddTrash(Trash{
- SizedDigest: blkid,
- Mtime: repl.Mtime,
- })
- change = changeTrash
- } else {
- change = changeStay
- }
- uniqueBestRepl[repl.Mtime] = true
- reportedBestRepl++
- } else if pulls+reportedBestRepl < blk.Desired &&
- len(blk.Replicas) > 0 &&
- !srv.ReadOnly {
- // This service doesn't have a replica. We
- // should pull one to this server if we don't
- // already have enough (existing+requested)
- // replicas in better rendezvous positions.
- srv.AddPull(Pull{
+ var change int
+ switch {
+ case !underreplicated && slot.repl != nil && !slot.want && !unsafeToDelete[slot.repl.Mtime]:
+ slot.mnt.KeepService.AddTrash(Trash{
SizedDigest: blkid,
- Source: blk.Replicas[0].KeepService,
+ Mtime: slot.repl.Mtime,
+ From: slot.mnt,
+ })
+ change = changeTrash
+ case len(blk.Replicas) == 0:
+ change = changeNone
+ case slot.repl == nil && slot.want && !slot.mnt.ReadOnly:
+ slot.mnt.KeepService.AddPull(Pull{
+ SizedDigest: blkid,
+ From: blk.Replicas[0].KeepMount.KeepService,
+ To: slot.mnt,
})
- pulls++
change = changePull
+ default:
+ change = changeStay
}
if bal.Dumper != nil {
var mtime int64
- if repl != nil {
- mtime = repl.Mtime
+ if slot.repl != nil {
+ mtime = slot.repl.Mtime
}
- changes = append(changes, fmt.Sprintf("%s:%d=%s,%d", srv.ServiceHost, srv.ServicePort, changeName[change], mtime))
+ srv := slot.mnt.KeepService
+ changes = append(changes, fmt.Sprintf("%s:%d/%s=%s,%d", srv.ServiceHost, srv.ServicePort, slot.mnt.UUID, changeName[change], mtime))
}
}
if bal.Dumper != nil {
- bal.Dumper.Printf("%s have=%d want=%d %s", blkid, len(blk.Replicas), blk.Desired, strings.Join(changes, " "))
+ bal.Dumper.Printf("%s have=%d want=%v %s", blkid, have, want, strings.Join(changes, " "))
+ }
+ return balanceResult{
+ blk: blk,
+ blkid: blkid,
+ have: have,
+ want: want,
}
}
replHistogram []int
}
-func (bal *Balancer) getStatistics() (s balancerStats) {
+func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
+ var s balancerStats
s.replHistogram = make([]int, 2)
- bal.BlockStateMap.Apply(func(blkid arvados.SizedDigest, blk *BlockState) {
- surplus := len(blk.Replicas) - blk.Desired
- bytes := blkid.Size()
+ for result := range results {
+ surplus := result.have - result.want
+ bytes := result.blkid.Size()
switch {
- case len(blk.Replicas) == 0 && blk.Desired > 0:
+ case result.have == 0 && result.want > 0:
s.lost.replicas -= surplus
s.lost.blocks++
s.lost.bytes += bytes * int64(-surplus)
- case len(blk.Replicas) < blk.Desired:
+ case surplus < 0:
s.underrep.replicas -= surplus
s.underrep.blocks++
s.underrep.bytes += bytes * int64(-surplus)
- case len(blk.Replicas) > 0 && blk.Desired == 0:
+ case surplus > 0 && result.want == 0:
counter := &s.garbage
- for _, r := range blk.Replicas {
+ for _, r := range result.blk.Replicas {
if r.Mtime >= bal.MinMtime {
counter = &s.unref
break
counter.replicas += surplus
counter.blocks++
counter.bytes += bytes * int64(surplus)
- case len(blk.Replicas) > blk.Desired:
+ case surplus > 0:
s.overrep.replicas += surplus
s.overrep.blocks++
- s.overrep.bytes += bytes * int64(len(blk.Replicas)-blk.Desired)
+ s.overrep.bytes += bytes * int64(len(result.blk.Replicas)-result.want)
default:
- s.justright.replicas += blk.Desired
+ s.justright.replicas += result.want
s.justright.blocks++
- s.justright.bytes += bytes * int64(blk.Desired)
+ s.justright.bytes += bytes * int64(result.want)
}
- if blk.Desired > 0 {
- s.desired.replicas += blk.Desired
+ if result.want > 0 {
+ s.desired.replicas += result.want
s.desired.blocks++
- s.desired.bytes += bytes * int64(blk.Desired)
+ s.desired.bytes += bytes * int64(result.want)
}
- if len(blk.Replicas) > 0 {
- s.current.replicas += len(blk.Replicas)
+ if len(result.blk.Replicas) > 0 {
+ s.current.replicas += len(result.blk.Replicas)
s.current.blocks++
- s.current.bytes += bytes * int64(len(blk.Replicas))
+ s.current.bytes += bytes * int64(len(result.blk.Replicas))
}
- for len(s.replHistogram) <= len(blk.Replicas) {
+ for len(s.replHistogram) <= len(result.blk.Replicas) {
s.replHistogram = append(s.replHistogram, 0)
}
- s.replHistogram[len(blk.Replicas)]++
- })
+ s.replHistogram[len(result.blk.Replicas)]++
+ }
for _, srv := range bal.KeepServices {
s.pulls += len(srv.ChangeSet.Pulls)
s.trashes += len(srv.ChangeSet.Trashes)
}
- return
+ bal.stats = s
}
// PrintStatistics writes statistics about the computed changes to
// bal.Logger. It should not be called until ComputeChangeSets has
// finished.
func (bal *Balancer) PrintStatistics() {
- s := bal.getStatistics()
bal.logf("===")
- bal.logf("%s lost (0=have<want)", s.lost)
- bal.logf("%s underreplicated (0<have<want)", s.underrep)
- bal.logf("%s just right (have=want)", s.justright)
- bal.logf("%s overreplicated (have>want>0)", s.overrep)
- bal.logf("%s unreferenced (have>want=0, new)", s.unref)
- bal.logf("%s garbage (have>want=0, old)", s.garbage)
+ bal.logf("%s lost (0=have<want)", bal.stats.lost)
+ bal.logf("%s underreplicated (0<have<want)", bal.stats.underrep)
+ bal.logf("%s just right (have=want)", bal.stats.justright)
+ bal.logf("%s overreplicated (have>want>0)", bal.stats.overrep)
+ bal.logf("%s unreferenced (have>want=0, new)", bal.stats.unref)
+ bal.logf("%s garbage (have>want=0, old)", bal.stats.garbage)
bal.logf("===")
- bal.logf("%s total commitment (excluding unreferenced)", s.desired)
- bal.logf("%s total usage", s.current)
+ bal.logf("%s total commitment (excluding unreferenced)", bal.stats.desired)
+ bal.logf("%s total usage", bal.stats.current)
bal.logf("===")
for _, srv := range bal.KeepServices {
bal.logf("%s: %v\n", srv, srv.ChangeSet)
}
bal.logf("===")
- bal.printHistogram(s, 60)
+ bal.printHistogram(60)
bal.logf("===")
}
-func (bal *Balancer) printHistogram(s balancerStats, hashColumns int) {
+func (bal *Balancer) printHistogram(hashColumns int) {
bal.logf("Replication level distribution (counting N replicas on a single server as N):")
maxCount := 0
- for _, count := range s.replHistogram {
+ for _, count := range bal.stats.replHistogram {
if maxCount < count {
maxCount = count
}
hashes := strings.Repeat("#", hashColumns)
countWidth := 1 + int(math.Log10(float64(maxCount+1)))
scaleCount := 10 * float64(hashColumns) / math.Floor(1+10*math.Log10(float64(maxCount+1)))
- for repl, count := range s.replHistogram {
+ for repl, count := range bal.stats.replHistogram {
nHashes := int(scaleCount * math.Log10(float64(count+1)))
bal.logf("%2d: %*d %s", repl, countWidth, count, hashes[:nHashes])
}
anyDesired := false
bal.BlockStateMap.Apply(func(_ arvados.SizedDigest, blk *BlockState) {
- if blk.Desired > 0 {
- anyDesired = true
+ for _, desired := range blk.Desired {
+ if desired > 0 {
+ anyDesired = true
+ break
+ }
}
})
if !anyDesired {
bal.Logger.Printf(f, args...)
}
}
+
+// Rendezvous hash sort function. Less efficient than sorting on
+// precomputed rendezvous hashes, but also rarely used.
+func rendezvousLess(i, j string, blkid arvados.SizedDigest) bool {
+ a := md5.Sum([]byte(string(blkid[:32]) + i))
+ b := md5.Sum([]byte(string(blkid[:32]) + j))
+ return bytes.Compare(a[:], b[:]) < 0
+}
}
c.Check(trashReqs.Count(), check.Equals, 0)
c.Check(pullReqs.Count(), check.Equals, 0)
- stats := bal.getStatistics()
- c.Check(stats.pulls, check.Not(check.Equals), 0)
- c.Check(stats.underrep.replicas, check.Not(check.Equals), 0)
- c.Check(stats.overrep.replicas, check.Not(check.Equals), 0)
+ c.Check(bal.stats.pulls, check.Not(check.Equals), 0)
+ c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0)
+ c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0)
}
func (s *runSuite) TestCommit(c *check.C) {
c.Check(err, check.IsNil)
c.Check(trashReqs.Count(), check.Equals, 8)
c.Check(pullReqs.Count(), check.Equals, 4)
- stats := bal.getStatistics()
// "foo" block is overreplicated by 2
- c.Check(stats.trashes, check.Equals, 2)
+ c.Check(bal.stats.trashes, check.Equals, 2)
// "bar" block is underreplicated by 1, and its only copy is
// in a poor rendezvous position
- c.Check(stats.pulls, check.Equals, 2)
+ c.Check(bal.stats.pulls, check.Equals, 2)
}
func (s *runSuite) TestRunForever(c *check.C) {
type tester struct {
known int
- desired int
+ desired map[string]int
current slots
timestamps []int64
shouldPull slots
shouldTrash slots
+
+ shouldPullMounts []string
+ shouldTrashMounts []string
}
func (bal *balancerSuite) SetUpSuite(c *check.C) {
UUID: fmt.Sprintf("zzzzz-bi6l4-%015x", i),
},
}
- srv.mounts = []*KeepMount{{KeepMount: arvados.KeepMount{UUID: fmt.Sprintf("mount-%015x", i)}, KeepService: srv}}
+ srv.mounts = []*KeepMount{{
+ KeepMount: arvados.KeepMount{
+ UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+ },
+ KeepService: srv,
+ }}
bal.srvs[i] = srv
bal.KeepServices[srv.UUID] = srv
}
func (bal *balancerSuite) TestPerfect(c *check.C) {
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 1},
shouldPull: nil,
shouldTrash: nil})
func (bal *balancerSuite) TestDecreaseRepl(c *check.C) {
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 2, 1},
shouldTrash: slots{2}})
}
func (bal *balancerSuite) TestDecreaseReplToZero(c *check.C) {
bal.try(c, tester{
- desired: 0,
+ desired: map[string]int{"default": 0},
current: slots{0, 1, 3},
shouldTrash: slots{0, 1, 3}})
}
func (bal *balancerSuite) TestIncreaseRepl(c *check.C) {
bal.try(c, tester{
- desired: 4,
+ desired: map[string]int{"default": 4},
current: slots{0, 1},
shouldPull: slots{2, 3}})
}
func (bal *balancerSuite) TestSkipReadonly(c *check.C) {
bal.srvList(0, slots{3})[0].ReadOnly = true
bal.try(c, tester{
- desired: 4,
+ desired: map[string]int{"default": 4},
current: slots{0, 1},
shouldPull: slots{2, 4}})
}
func (bal *balancerSuite) TestFixUnbalanced(c *check.C) {
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 0},
shouldPull: slots{1}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 7},
shouldPull: slots{0, 1}})
// if only one of the pulls succeeds, we'll see this next:
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 1, 7},
shouldPull: slots{0},
shouldTrash: slots{7}})
// if both pulls succeed, we'll see this next:
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 0, 1, 7},
shouldTrash: slots{2, 7}})
// unbalanced + excessive replication => pull + trash
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 5, 7},
shouldPull: slots{0, 1},
shouldTrash: slots{7}})
}
func (bal *balancerSuite) TestMultipleReplicasPerService(c *check.C) {
+ for _, srv := range bal.srvs {
+ for i := 0; i < 3; i++ {
+ m := *(srv.mounts[0])
+ srv.mounts = append(srv.mounts, &m)
+ }
+ }
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 0},
shouldPull: slots{1}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{2, 2},
shouldPull: slots{0, 1}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 0, 1},
shouldTrash: slots{0}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 1, 0},
shouldTrash: slots{1}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 0, 1, 0, 2},
shouldTrash: slots{0, 1, 2}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 1, 1, 0, 2},
shouldTrash: slots{1, 1, 2}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 1, 2},
shouldPull: slots{0},
shouldTrash: slots{1}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 1, 0},
timestamps: []int64{12345678, 12345678, 12345679},
shouldTrash: nil})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{1, 1},
shouldPull: slots{0}})
}
// For purposes of increasing replication, we assume identical
// replicas are distinct.
bal.try(c, tester{
- desired: 4,
+ desired: map[string]int{"default": 4},
current: slots{0, 1},
timestamps: []int64{12345678, 12345678},
shouldPull: slots{2, 3}})
// For purposes of decreasing replication, we assume identical
// replicas are NOT distinct.
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{12345678, 12345678, 12345678}})
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{12345678, 10000000, 10000000}})
}
newTime := bal.MinMtime + 3600
// The excess replica is too new to delete.
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{oldTime, newTime, newTime + 1}})
// The best replicas are too new to delete, but the excess
// replica is old enough.
bal.try(c, tester{
- desired: 2,
+ desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{newTime, newTime + 1, oldTime},
shouldTrash: slots{2}})
}
+func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
+ // For known blocks 0/1/2/3, server 9 is slot 9/1/14/0 in
+ // probe order. For these tests we give it two mounts, one
+ // with classes=[special], one with
+ // classes=[special,special2].
+ bal.srvs[9].mounts = []*KeepMount{{
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special"},
+ UUID: "zzzzz-mount-special00000009",
+ DeviceID: "9-special",
+ },
+ KeepService: bal.srvs[9],
+ }, {
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special", "special2"},
+ UUID: "zzzzz-mount-special20000009",
+ DeviceID: "9-special-and-special2",
+ },
+ KeepService: bal.srvs[9],
+ }}
+ // For known blocks 0/1/2/3, server 13 (d) is slot 5/3/11/1 in
+ // probe order. We give it two mounts, one with
+ // classes=[special3], one with classes=[default].
+ bal.srvs[13].mounts = []*KeepMount{{
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special2"},
+ UUID: "zzzzz-mount-special2000000d",
+ DeviceID: "13-special2",
+ },
+ KeepService: bal.srvs[13],
+ }, {
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"default"},
+ UUID: "zzzzz-mount-00000000000000d",
+ DeviceID: "13-default",
+ },
+ KeepService: bal.srvs[13],
+ }}
+ // Pull to slot 9 because that's the only server with the
+ // desired class "special".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2, "special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{9},
+ shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ // If some storage classes are not satisfied, don't trash any
+ // excess replicas. (E.g., if someone desires repl=1 on
+ // class=durable, and we have two copies on class=volatile, we
+ // should wait for pull to succeed before trashing anything).
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{9},
+ shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ // Once storage classes are satisfied, trash excess replicas
+ // that appear earlier in probe order but aren't needed to
+ // satisfy the desired classes.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1, 9},
+ shouldTrash: slots{0, 1}})
+ // Pull to slot 5, the best server with class "special2".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special2": 1},
+ current: slots{0, 1},
+ shouldPull: slots{5},
+ shouldPullMounts: []string{"zzzzz-mount-special2000000d"}})
+ // Pull to slot 5 and 9 to get replication 2 in desired class
+ // "special2".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special2": 2},
+ current: slots{0, 1},
+ shouldPull: slots{5, 9},
+ shouldPullMounts: []string{"zzzzz-mount-special20000009", "zzzzz-mount-special2000000d"}})
+ // Slot 0 has a replica in "default", slot 1 has a replica
+ // in "special"; we need another replica in "default", i.e.,
+ // on slot 2.
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2, "special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{2}})
+ // Pull to best probe position 0 (despite wrong storage class)
+ // if it's impossible to achieve desired replication in the
+ // desired class (only slots 1 and 3 have special2).
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"special2": 3},
+ current: slots{3},
+ shouldPull: slots{0, 1}})
+ // Trash excess replica.
+ bal.try(c, tester{
+ known: 3,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1},
+ shouldTrash: slots{1}})
+ // Leave one copy on slot 1 because slot 0 (server 9) only
+ // gives us repl=1.
+ bal.try(c, tester{
+ known: 3,
+ desired: map[string]int{"special": 2},
+ current: slots{0, 1}})
+}
+
// Clear all servers' changesets, balance a single block, and verify
// the appropriate changes for that block have been added to the
// changesets.
func (bal *balancerSuite) try(c *check.C, t tester) {
- bal.setupServiceRoots()
+ bal.setupLookupTables()
blk := &BlockState{
+ Replicas: bal.replList(t.known, t.current),
Desired: t.desired,
- Replicas: bal.replList(t.known, t.current)}
+ }
for i, t := range t.timestamps {
blk.Replicas[i].Mtime = t
}
bal.balanceBlock(knownBlkid(t.known), blk)
var didPull, didTrash slots
+ var didPullMounts, didTrashMounts []string
for i, srv := range bal.srvs {
var slot int
for probeOrder, srvNum := range bal.knownRendezvous[t.known] {
}
for _, pull := range srv.Pulls {
didPull = append(didPull, slot)
+ didPullMounts = append(didPullMounts, pull.To.UUID)
c.Check(pull.SizedDigest, check.Equals, knownBlkid(t.known))
}
for _, trash := range srv.Trashes {
didTrash = append(didTrash, slot)
+ didTrashMounts = append(didTrashMounts, trash.From.UUID)
c.Check(trash.SizedDigest, check.Equals, knownBlkid(t.known))
}
}
}
c.Check(didPull, check.DeepEquals, t.shouldPull)
c.Check(didTrash, check.DeepEquals, t.shouldTrash)
+ if t.shouldPullMounts != nil {
+ sort.Strings(didPullMounts)
+ c.Check(didPullMounts, check.DeepEquals, t.shouldPullMounts)
+ }
+ if t.shouldTrashMounts != nil {
+ sort.Strings(didTrashMounts)
+ c.Check(didTrashMounts, check.DeepEquals, t.shouldTrashMounts)
+ }
}
// srvList returns the KeepServices, sorted in rendezvous order and
// replList is like srvList but returns an "existing replicas" slice,
// suitable for a BlockState test fixture.
func (bal *balancerSuite) replList(knownBlockID int, order slots) (repls []Replica) {
+ nextMnt := map[*KeepService]int{}
mtime := time.Now().UnixNano() - (bal.signatureTTL+86400)*1e9
for _, srv := range bal.srvList(knownBlockID, order) {
- repls = append(repls, Replica{srv.mounts[0], mtime})
+ // round-robin repls onto each srv's mounts
+ n := nextMnt[srv]
+ nextMnt[srv] = (n + 1) % len(srv.mounts)
+
+ repls = append(repls, Replica{srv.mounts[n], mtime})
mtime++
}
return
Mtime int64
}
-// BlockState indicates the number of desired replicas (according to
-// the collections we know about) and the replicas actually stored
-// (according to the keepstore indexes we know about).
+// BlockState indicates the desired storage class and number of
+// replicas (according to the collections we know about) and the
+// replicas actually stored (according to the keepstore indexes we
+// know about).
type BlockState struct {
Replicas []Replica
- Desired int
+ Desired map[string]int
+ // TODO: Support combinations of classes ("private + durable")
+ // by replacing the map[string]int with a map[*[]string]int
+ // here, where the map keys come from a pool of semantically
+ // distinct class combinations.
+ //
+ // TODO: Use a pool of semantically distinct Desired maps to
+ // conserve memory (typically there are far more BlockState
+ // objects in memory than distinct Desired profiles).
}
+var defaultClasses = []string{"default"}
+
func (bs *BlockState) addReplica(r Replica) {
bs.Replicas = append(bs.Replicas, r)
}
-func (bs *BlockState) increaseDesired(n int) {
- if bs.Desired < n {
- bs.Desired = n
+func (bs *BlockState) increaseDesired(classes []string, n int) {
+ if len(classes) == 0 {
+ classes = defaultClasses
+ }
+ for _, class := range classes {
+ if bs.Desired == nil {
+ bs.Desired = map[string]int{class: n}
+ } else if d, ok := bs.Desired[class]; !ok || d < n {
+ bs.Desired[class] = n
+ }
}
}
}
// IncreaseDesired updates the map to indicate the desired replication
-// for the given blocks is at least n.
-func (bsm *BlockStateMap) IncreaseDesired(n int, blocks []arvados.SizedDigest) {
+// for the given blocks in the given storage class is at least n.
+func (bsm *BlockStateMap) IncreaseDesired(classes []string, n int, blocks []arvados.SizedDigest) {
bsm.mutex.Lock()
defer bsm.mutex.Unlock()
for _, blkid := range blocks {
- bsm.get(blkid).increaseDesired(n)
+ bsm.get(blkid).increaseDesired(classes, n)
}
}
// store it locally.
type Pull struct {
arvados.SizedDigest
- Source *KeepService
+ From *KeepService
+ To *KeepMount
}
// MarshalJSON formats a pull request the way keepstore wants to see
// it.
func (p Pull) MarshalJSON() ([]byte, error) {
type KeepstorePullRequest struct {
- Locator string `json:"locator"`
- Servers []string `json:"servers"`
+ Locator string `json:"locator"`
+ Servers []string `json:"servers"`
+ MountUUID string `json:"mount_uuid"`
}
return json.Marshal(KeepstorePullRequest{
- Locator: string(p.SizedDigest[:32]),
- Servers: []string{p.Source.URLBase()}})
+ Locator: string(p.SizedDigest[:32]),
+ Servers: []string{p.From.URLBase()},
+ MountUUID: p.To.KeepMount.UUID,
+ })
}
// Trash is a request to delete a block.
type Trash struct {
arvados.SizedDigest
Mtime int64
+ From *KeepMount
}
// MarshalJSON formats a trash request the way keepstore wants to see
type KeepstoreTrashRequest struct {
Locator string `json:"locator"`
BlockMtime int64 `json:"block_mtime"`
+ MountUUID string `json:"mount_uuid"`
}
return json.Marshal(KeepstoreTrashRequest{
Locator: string(t.SizedDigest[:32]),
- BlockMtime: t.Mtime})
+ BlockMtime: t.Mtime,
+ MountUUID: t.From.KeepMount.UUID,
+ })
}
// ChangeSet is a set of change requests that will be sent to a
type changeSetSuite struct{}
func (s *changeSetSuite) TestJSONFormat(c *check.C) {
+ mnt := &KeepMount{
+ KeepMount: arvados.KeepMount{
+ UUID: "zzzzz-mount-abcdefghijklmno"}}
srv := &KeepService{
KeepService: arvados.KeepService{
UUID: "zzzzz-bi6l4-000000000000001",
buf, err := json.Marshal([]Pull{{
SizedDigest: arvados.SizedDigest("acbd18db4cc2f85cedef654fccc4a4d8+3"),
- Source: srv}})
+ To: mnt,
+ From: srv}})
c.Check(err, check.IsNil)
- c.Check(string(buf), check.Equals, `[{"locator":"acbd18db4cc2f85cedef654fccc4a4d8","servers":["http://keep1.zzzzz.arvadosapi.com:25107"]}]`)
+ c.Check(string(buf), check.Equals, `[{"locator":"acbd18db4cc2f85cedef654fccc4a4d8","servers":["http://keep1.zzzzz.arvadosapi.com:25107"],"mount_uuid":"zzzzz-mount-abcdefghijklmno"}]`)
buf, err = json.Marshal([]Trash{{
SizedDigest: arvados.SizedDigest("acbd18db4cc2f85cedef654fccc4a4d8+3"),
+ From: mnt,
Mtime: 123456789}})
c.Check(err, check.IsNil)
- c.Check(string(buf), check.Equals, `[{"locator":"acbd18db4cc2f85cedef654fccc4a4d8","block_mtime":123456789}]`)
+ c.Check(string(buf), check.Equals, `[{"locator":"acbd18db4cc2f85cedef654fccc4a4d8","block_mtime":123456789,"mount_uuid":"zzzzz-mount-abcdefghijklmno"}]`)
}
import (
"bytes"
+ "fmt"
"io"
"io/ioutil"
"net/url"
"os"
"os/exec"
+ "path/filepath"
"strings"
"time"
check "gopkg.in/check.v1"
)
-func (s *IntegrationSuite) TestWebdavWithCadaver(c *check.C) {
+func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
+ w := "/c=" + newCollection.UUID + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return r, w, pdh
+ }, nil)
+}
+
+func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
+ s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
+ r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
+ w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
+ return r, w, pdh
+ }, nil)
+}
+
+func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+ rpath := "/users/active/foo_file_in_dir/"
+ s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
+ wpath := "/users/active/" + newCollection.Name
+ pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
+ return rpath, wpath, pdh
+ }, func(path string) bool {
+ // Skip tests that rely on writes, because /users/
+ // tree is read-only.
+ return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
+ })
+}
+
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
+
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
- localfile, err := ioutil.TempFile("", "localfile")
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ localfile, err := ioutil.TempFile(tempdir, "localfile")
c.Assert(err, check.IsNil)
- defer os.Remove(localfile.Name())
localfile.Write(testdata)
- emptyfile, err := ioutil.TempFile("", "emptyfile")
+ emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
c.Assert(err, check.IsNil)
- defer os.Remove(emptyfile.Name())
- checkfile, err := ioutil.TempFile("", "checkfile")
+ checkfile, err := ioutil.TempFile(tempdir, "checkfile")
c.Assert(err, check.IsNil)
- defer os.Remove(checkfile.Name())
var newCollection arvados.Collection
arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
err = arv.RequestAndDecode(&newCollection, "POST", "/arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
c.Assert(err, check.IsNil)
- writePath := "/c=" + newCollection.UUID + "/t=" + arv.AuthToken + "/"
- pdhPath := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arv.AuthToken + "/"
+ readPath, writePath, pdhPath := pathFunc(newCollection)
matchToday := time.Now().Format("Jan +2")
- readPath := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
type testcase struct {
path string
cmd string
},
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ if skip != nil && skip(trial.path) {
+ c.Log("(skip)")
+ continue
+ }
os.Remove(checkfile.Name())
- cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+trial.path)
- cmd.Stdin = bytes.NewBufferString(trial.cmd)
- stdout, err := cmd.StdoutPipe()
- c.Assert(err, check.Equals, nil)
- cmd.Stderr = cmd.Stdout
- go cmd.Start()
-
- var buf bytes.Buffer
- _, err = io.Copy(&buf, stdout)
- c.Check(err, check.Equals, nil)
- err = cmd.Wait()
- c.Check(err, check.Equals, nil)
- c.Check(buf.String(), check.Matches, trial.match)
+ stdout := s.runCadaver(c, password, trial.path, trial.cmd)
+ c.Check(stdout, check.Matches, trial.match)
if trial.data == nil {
continue
c.Check(err, check.IsNil)
}
}
+
+func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
+ for _, path := range []string{"/by_id", "/by_id/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
+ }
+ for _, path := range []string{
+ "/by_id/" + arvadostest.FooPdh,
+ "/by_id/" + arvadostest.FooPdh + "/",
+ "/by_id/" + arvadostest.FooCollection,
+ "/by_id/" + arvadostest.FooCollection + "/",
+ } {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
+ }
+}
+
+func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
+ for _, path := range []string{"/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
+ }
+ for _, path := range []string{"/users", "/users/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
+ }
+ for _, path := range []string{"/users/active", "/users/active/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
+ c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
+ }
+ for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
+ stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
+ c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
+ }
+}
+
+func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
+ tempdir, err := ioutil.TempDir("", "keep-web-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
+ if password != "" {
+ // cadaver won't try username/password authentication
+ // unless the server responds 401 to an
+ // unauthenticated request, which it only does in
+ // AttachmentOnlyHost, TrustAllContent, and
+ // per-collection vhost cases.
+ s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
+
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ cmd.Stdin = bytes.NewBufferString(stdin)
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ return buf.String()
+}
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
+//
+// The following URLs are read-only, but otherwise interchangeable
+// with the above:
+//
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
+// http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt
+// http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt
+//
+// If the collection is named "MyCollection" and located in a project
+// called "MyProject" which is in the home project of a user with
+// username is "bob", the following read-only URL is also available
+// when authenticating as bob:
+//
+// http://collections.example.com/users/bob/MyProject/MyCollection/foo/bar.txt
//
// An additional form is supported specifically to make it more
// convenient to maintain support for existing Workbench download
//
// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
//
+// Collections can also be accessed (read-only) via "/by_id/X" where X
+// is a UUID or portable data hash.
+//
// Authorization mechanisms
//
// A token can be provided in an Authorization header:
//
// Indexes
//
-// Currently, keep-web does not generate HTML index listings, nor does
-// it serve a default file like "index.html" when a directory is
-// requested. These features are likely to be added in future
-// versions. Until then, keep-web responds with 404 if a directory
-// name (or any path ending with "/") is requested.
+// Keep-web returns a generic HTML index listing when a directory is
+// requested with the GET method. It does not serve a default file
+// like "index.html". Directory listings are also returned for WebDAV
+// PROPFIND requests.
//
// Compatibility
//
"html"
"html/template"
"io"
- "log"
"net/http"
"net/url"
"os"
+ "path/filepath"
"sort"
"strconv"
"strings"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ log "github.com/Sirupsen/logrus"
"golang.org/x/net/webdav"
)
}
func (uos *updateOnSuccess) Write(p []byte) (int, error) {
- if uos.err != nil {
- return 0, uos.err
- }
if !uos.sentHeader {
uos.WriteHeader(http.StatusOK)
}
+ if uos.err != nil {
+ return 0, uos.err
+ }
return uos.ResponseWriter.Write(p)
}
"HEAD": true,
"POST": true,
}
+ // top-level dirs to serve with siteFS
+ siteFSDir = map[string]bool{
+ "": true, // root directory
+ "by_id": true,
+ "users": true,
+ }
)
// ServeHTTP implements http.Handler.
} else if w.WroteStatus() == 0 {
w.WriteHeader(statusCode)
} else if w.WroteStatus() != statusCode {
- httpserver.Log(r.RemoteAddr, "WARNING",
+ log.WithField("RequestID", r.Header.Get("X-Request-Id")).Warn(
fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
}
if statusText == "" {
statusText = http.StatusText(statusCode)
}
- httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
}()
if strings.HasPrefix(r.URL.Path, "/_health/") && r.Method == "GET" {
w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
}
- arv := h.clientPool.Get()
- if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
- return
- }
- defer h.clientPool.Put(arv)
-
pathParts := strings.Split(r.URL.Path[1:], "/")
var stripParts int
- var targetID string
+ var collectionID string
var tokens []string
var reqTokens []string
var pathToken bool
var attachment bool
+ var useSiteFS bool
credentialsOK := h.Config.TrustAllContent
if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
attachment = true
}
- if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
+ if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
+ } else if siteFSDir[pathParts[0]] {
+ useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID[/PATH...]
- targetID = parseCollectionIDFromURL(pathParts[0][2:])
+ collectionID = parseCollectionIDFromURL(pathParts[0][2:])
stripParts = 1
} else if len(pathParts) >= 2 && pathParts[0] == "collections" {
if len(pathParts) >= 4 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetID = parseCollectionIDFromURL(pathParts[2])
+ collectionID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
stripParts = 4
pathToken = true
} else {
// /collections/ID/PATH...
- targetID = parseCollectionIDFromURL(pathParts[1])
+ collectionID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
stripParts = 2
}
}
- if targetID == "" {
+ if collectionID == "" && !useSiteFS {
statusCode = http.StatusNotFound
return
}
+ forceReload := false
+ if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
+ forceReload = true
+ }
+
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
return
}
+ if useSiteFS {
+ if tokens == nil {
+ tokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
+ }
+ h.serveSiteFS(w, r, tokens, credentialsOK, attachment)
+ return
+ }
+
targetPath := pathParts[stripParts:]
if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
// http://ID.example/t=TOKEN/PATH...
stripParts++
}
- forceReload := false
- if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
- forceReload = true
+ arv := h.clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
+ return
}
+ defer h.clientPool.Put(arv)
var collection *arvados.Collection
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
+ collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
+ kc.RequestID = r.Header.Get("X-Request-Id")
var basename string
if len(targetPath) > 0 {
}
applyContentDispositionHdr(w, r, basename, attachment)
- client := &arvados.Client{
+ client := (&arvados.Client{
APIHost: arv.ApiServer,
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
- }
+ }).WithRequestID(r.Header.Get("X-Request-Id"))
+
fs, err := collection.FileSystem(client, kc)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- targetIsPDH := arvadosclient.PDHMatch(targetID)
- if targetIsPDH && writeMethod[r.Method] {
+ writefs, writeOK := fs.(arvados.CollectionFileSystem)
+ targetIsPDH := arvadosclient.PDHMatch(collectionID)
+ if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
statusCode, statusText = http.StatusMethodNotAllowed, errReadOnly.Error()
return
}
w = &updateOnSuccess{
ResponseWriter: w,
update: func() error {
- return h.Config.Cache.Update(client, *collection, fs)
+ return h.Config.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
// "dirname/fnm".
h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
} else if stat.IsDir() {
- h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
+ h.serveDirectory(w, r, collection.Name, fs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
}
}
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
+ if len(tokens) == 0 {
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+ http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ return
+ }
+ if writeMethod[r.Method] {
+ http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
+ return
+ }
+ arv := h.clientPool.Get()
+ if arv == nil {
+ http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ return
+ }
+ defer h.clientPool.Put(arv)
+ arv.ApiToken = tokens[0]
+
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ kc.RequestID = r.Header.Get("X-Request-Id")
+ client := (&arvados.Client{
+ APIHost: arv.ApiServer,
+ AuthToken: arv.ApiToken,
+ Insecure: arv.ApiInsecure,
+ }).WithRequestID(r.Header.Get("X-Request-Id"))
+ fs := client.SiteFileSystem(kc)
+ f, err := fs.Open(r.URL.Path)
+ if os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ } else if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer f.Close()
+ if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+ if !strings.HasSuffix(r.URL.Path, "/") {
+ h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
+ } else {
+ h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+ }
+ return
+ }
+ if r.Method == "GET" {
+ _, basename := filepath.Split(r.URL.Path)
+ applyContentDispositionHdr(w, r, basename, attachment)
+ }
+ wh := webdav.Handler{
+ Prefix: "/",
+ FileSystem: &webdavFS{
+ collfs: fs,
+ writing: writeMethod[r.Method],
+ alwaysReadEOF: r.Method == "PROPFIND",
+ },
+ LockSystem: h.webdavLS,
+ Logger: func(_ *http.Request, err error) {
+ if err != nil {
+ log.Printf("error from webdav handler: %q", err)
+ }
+ },
+ }
+ wh.ServeHTTP(w, r)
+}
+
var dirListingTemplate = `<!DOCTYPE HTML>
<HTML><HEAD>
<META name="robots" content="NOINDEX">
- <TITLE>{{ .Collection.Name }}</TITLE>
+ <TITLE>{{ .CollectionName }}</TITLE>
<STYLE type="text/css">
body {
margin: 1.5em;
</STYLE>
</HEAD>
<BODY>
+
<H1>{{ .CollectionName }}</H1>
<P>This collection of data files is being shared with you through
Arvados. You can download individual files listed below. To download
-the entire collection with wget, try:</P>
+the entire directory tree with wget, try:</P>
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
<H2>File Listing</H2>
{{if .Files}}
<UL>
-{{range .Files}} <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
+{{range .Files}}
+{{if .IsDir }}
+ <LI>{{" " | printf "%15s " | nbsp}}<A href="{{print "./" .Name}}/">{{.Name}}/</A></LI>
+{{else}}
+ <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{print "./" .Name}}">{{.Name}}</A></LI>
+{{end}}
+{{end}}
</UL>
{{else}}
<P>(No files; this collection is empty.)</P>
`
type fileListEnt struct {
- Name string
- Size int64
+ Name string
+ Size int64
+ IsDir bool
}
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
var files []fileListEnt
var walk func(string) error
if !strings.HasSuffix(base, "/") {
return err
}
for _, ent := range ents {
- if ent.IsDir() {
+ if recurse && ent.IsDir() {
err = walk(path + ent.Name() + "/")
if err != nil {
return err
}
} else {
files = append(files, fileListEnt{
- Name: path + ent.Name(),
- Size: ent.Size(),
+ Name: path + ent.Name(),
+ Size: ent.Size(),
+ IsDir: ent.IsDir(),
})
}
}
"CollectionName": collectionName,
"Files": files,
"Request": r,
- "StripParts": stripParts,
+ "StripParts": strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
})
}
"net/http"
"net/http/httptest"
"net/url"
+ "os"
"path/filepath"
"regexp"
"strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/auth"
check "gopkg.in/check.v1"
http.StatusOK,
"foo",
)
- c.Check(strings.Split(resp.Header().Get("Content-Disposition"), ";")[0], check.Equals, "attachment")
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
+ "",
+ "",
+ http.StatusOK,
+ "foo",
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
)
}
+func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll := arvados.Collection{ManifestText: mtxt}
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
+ c.Assert(err, check.IsNil)
+
+ u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
+}
+
// XHRs can't follow redirect-with-cookie so they rely on method=POST
// and disposition=attachment (telling us it's acceptable to respond
// with content instead of a redirect) and an Origin header that gets
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
for _, trial := range []struct {
- uri string
- header http.Header
- expect []string
- cutDirs int
+ uri string
+ header http.Header
+ expect []string
+ redirect string
+ cutDirs int
}{
{
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 0,
+ cutDirs: 1,
},
{
uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
expect: []string{"dir1/foo", "dir1/bar"},
cutDirs: 2,
},
+ {
+ uri: "download.example.com/users/active/foo_file_in_dir/",
+ header: authHeader,
+ expect: []string{"dir1/"},
+ cutDirs: 3,
+ },
+ {
+ uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
+ header: authHeader,
+ expect: []string{"bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/",
+ header: authHeader,
+ expect: []string{"users/"},
+ cutDirs: 0,
+ },
+ {
+ uri: "download.example.com/users",
+ header: authHeader,
+ redirect: "/users/",
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/",
+ header: authHeader,
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/active",
+ header: authHeader,
+ redirect: "/users/active/",
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
+ uri: "download.example.com/users/active/",
+ header: authHeader,
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
{
uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
header: nil,
cutDirs: 1,
},
{
- uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 1,
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
+ header: authHeader,
+ redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 2,
},
{
uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
header: authHeader,
expect: []string{"foo", "bar"},
- cutDirs: 2,
+ cutDirs: 3,
},
{
- uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
- header: authHeader,
- expect: []string{"foo", "bar"},
- cutDirs: 0,
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
+ redirect: "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
},
{
uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
s.testServer.Handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
for _, c := range cookies {
resp = httptest.NewRecorder()
s.testServer.Handler.ServeHTTP(resp, req)
}
+ if trial.redirect != "" {
+ c.Check(req.URL.Path, check.Equals, trial.redirect)
+ }
if trial.expect == nil {
c.Check(resp.Code, check.Equals, http.StatusNotFound)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK)
for _, e := range trial.expect {
- c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
}
c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
}
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
}
+
+func copyHeader(h http.Header) http.Header {
+ hc := http.Header{}
+ for k, v := range h {
+ hc[k] = append([]string(nil), v...)
+ }
+ return hc
+}
import (
"flag"
"fmt"
- "log"
"os"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/config"
+ log "github.com/Sirupsen/logrus"
"github.com/coreos/go-systemd/daemon"
)
if os.Getenv("ARVADOS_API_TOKEN") == "" {
os.Setenv("ARVADOS_API_TOKEN", "xxx")
}
+
+ log.SetFormatter(&log.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ })
}
func main() {
}
func (srv *server) Start() error {
- srv.Handler = &handler{Config: srv.Config}
+ srv.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(&handler{Config: srv.Config}))
srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
func (s *IntegrationSuite) Test404(c *check.C) {
for _, uri := range []string{
// Routing errors (always 404 regardless of what's stored in Keep)
- "/",
"/foo",
"/download",
"/collections",
// existence automatically so sequences like "mkcol foo; put foo/bar"
// work as expected.
type webdavFS struct {
- collfs arvados.CollectionFileSystem
+ collfs arvados.FileSystem
writing bool
// webdav PROPFIND reads the first few bytes of each file
// whose filename extension isn't recognized, which is
}
func (fs *webdavFS) makeparents(name string) {
+ if !fs.writing {
+ return
+ }
dir, _ := path.Split(name)
if dir == "" || dir == "/" {
return
}
func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
- writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+ writing := flag&(os.O_WRONLY|os.O_RDWR|os.O_TRUNC) != 0
if writing {
fs.makeparents(name)
}
// webdav module returns 404 on all OpenFile errors,
// but returns 405 Method Not Allowed if OpenFile()
// succeeds but Write() or Close() fails. We'd rather
- // have 405.
- f = writeFailer{File: f, err: errReadOnly}
+ // have 405. writeFailer ensures Close() fails if the
+ // file is opened for writing *or* Write() is called.
+ var err error
+ if writing {
+ err = errReadOnly
+ }
+ f = writeFailer{File: f, err: err}
}
if fs.alwaysReadEOF {
f = readEOF{File: f}
}
func (wf writeFailer) Write([]byte) (int, error) {
+ wf.err = errReadOnly
return 0, wf.err
}
func (wf writeFailer) Close() error {
+ err := wf.File.Close()
+ if err != nil {
+ wf.err = err
+ }
return wf.err
}
var err error
arv := *kc.Arvados
arv.ApiToken = tok
+ arv.RequestID = req.Header.Get("X-Request-Id")
if op == "read" {
err = arv.Call("HEAD", "keep_services", "", "accessible", nil, nil)
} else {
return true, tok
}
+// We need to make a private copy of the default http transport early
+// in initialization, then make copies of our private copy later. It
+// won't be safe to copy http.DefaultTransport itself later, because
+// its private mutexes might have already been used. (Without this,
+// the test suite sometimes panics "concurrent map writes" in
+// net/http.(*Transport).removeIdleConnLocked().)
+var defaultTransport = *(http.DefaultTransport.(*http.Transport))
+
type proxyHandler struct {
http.Handler
*keepclient.KeepClient
func MakeRESTRouter(enable_get bool, enable_put bool, kc *keepclient.KeepClient, timeout time.Duration, mgmtToken string) http.Handler {
rest := mux.NewRouter()
- transport := *(http.DefaultTransport.(*http.Transport))
+ transport := defaultTransport
transport.DialContext = (&net.Dialer{
Timeout: keepclient.DefaultConnectTimeout,
KeepAlive: keepclient.DefaultKeepAlive,
func (h *proxyHandler) makeKeepClient(req *http.Request) *keepclient.KeepClient {
kc := *h.KeepClient
+ kc.RequestID = req.Header.Get("X-Request-Id")
kc.HTTPClient = &proxyClient{
client: &http.Client{
Timeout: h.timeout,
Transport: h.transport,
},
- proto: req.Proto,
- requestID: req.Header.Get("X-Request-Id"),
+ proto: req.Proto,
}
return &kc
}
var viaAlias = "keepproxy"
type proxyClient struct {
- client keepclient.HTTPClient
- proto string
- requestID string
+ client keepclient.HTTPClient
+ proto string
}
func (pc *proxyClient) Do(req *http.Request) (*http.Response, error) {
req.Header.Add("Via", pc.proto+" "+viaAlias)
- req.Header.Add("X-Request-Id", pc.requestID)
return pc.client.Do(req)
}
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
var bytesDeleted, bytesInTrash int64
- var blocksDeleted, blocksInTrash int
- params := storage.ListBlobsParameters{Include: &storage.IncludeBlobDataset{Metadata: true}}
+ var blocksDeleted, blocksInTrash int64
- for {
- resp, err := v.container.ListBlobs(params)
+ doBlob := func(b storage.Blob) {
+ // Check whether the block is flagged as trash
+ if b.Metadata["expires_at"] == "" {
+ return
+ }
+
+ atomic.AddInt64(&blocksInTrash, 1)
+ atomic.AddInt64(&bytesInTrash, b.Properties.ContentLength)
+
+ expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
if err != nil {
- log.Printf("EmptyTrash: ListBlobs: %v", err)
- break
+ log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
+ return
}
- for _, b := range resp.Blobs {
- // Check if the block is expired
- if b.Metadata["expires_at"] == "" {
- continue
- }
- blocksInTrash++
- bytesInTrash += b.Properties.ContentLength
+ if expiresAt > time.Now().Unix() {
+ return
+ }
- expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
- if err != nil {
- log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
- continue
- }
+ err = v.container.DeleteBlob(b.Name, &storage.DeleteBlobOptions{
+ IfMatch: b.Properties.Etag,
+ })
+ if err != nil {
+ log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+ return
+ }
+ atomic.AddInt64(&blocksDeleted, 1)
+ atomic.AddInt64(&bytesDeleted, b.Properties.ContentLength)
+ }
- if expiresAt > time.Now().Unix() {
- continue
+ var wg sync.WaitGroup
+ todo := make(chan storage.Blob, theConfig.EmptyTrashWorkers)
+ for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for b := range todo {
+ doBlob(b)
}
+ }()
+ }
- err = v.container.DeleteBlob(b.Name, &storage.DeleteBlobOptions{
- IfMatch: b.Properties.Etag,
- })
- if err != nil {
- log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
- continue
- }
- blocksDeleted++
- bytesDeleted += b.Properties.ContentLength
+ params := storage.ListBlobsParameters{Include: &storage.IncludeBlobDataset{Metadata: true}}
+ for {
+ resp, err := v.container.ListBlobs(params)
+ if err != nil {
+ log.Printf("EmptyTrash: ListBlobs: %v", err)
+ break
+ }
+ for _, b := range resp.Blobs {
+ todo <- b
}
if resp.NextMarker == "" {
break
}
params.Marker = resp.NextMarker
}
+ close(todo)
+ wg.Wait()
log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
}
azureWriteRaceInterval = time.Second
azureWriteRacePollTime = time.Millisecond
- allDone := make(chan struct{})
+ var wg sync.WaitGroup
+
v.azHandler.race = make(chan chan struct{})
+
+ wg.Add(1)
go func() {
+ defer wg.Done()
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
t.Error(err)
continuePut := make(chan struct{})
// Wait for the stub's Put to create the empty blob
v.azHandler.race <- continuePut
+ wg.Add(1)
go func() {
+ defer wg.Done()
buf := make([]byte, len(TestBlock))
_, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
t.Error(err)
}
- close(allDone)
}()
// Wait for the stub's Get to get the empty blob
close(v.azHandler.race)
// Allow stub's Put to continue, so the real data is ready
// when the volume's Get retries
<-continuePut
- // Wait for volume's Get to return the real data
- <-allDone
+ // Wait for Get() and Put() to finish
+ wg.Wait()
}
func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
EnableDelete bool
TrashLifetime arvados.Duration
TrashCheckInterval arvados.Duration
+ PullWorkers int
+ TrashWorkers int
+ EmptyTrashWorkers int
+ TLSCertificateFile string
+ TLSKeyFile string
Volumes VolumeList
pullq.ReplaceQueue(plist)
}
-// TrashRequest consists of a block locator and it's Mtime
+// TrashRequest consists of a block locator and its Mtime
type TrashRequest struct {
Locator string `json:"locator"`
BlockMtime int64 `json:"block_mtime"`
"flag"
"fmt"
"net"
- "net/http"
"os"
"os/signal"
"syscall"
log.Fatal(err)
}
- // Initialize Pull queue and worker
+ // Initialize keepclient for pull workers
keepClient := &keepclient.KeepClient{
Arvados: &arvadosclient.ArvadosClient{},
Want_replicas: 1,
}
- // Initialize the pullq and worker
+ // Initialize the pullq and workers
pullq = NewWorkQueue()
- go RunPullWorker(pullq, keepClient)
+ for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
+ go RunPullWorker(pullq, keepClient)
+ }
- // Initialize the trashq and worker
+ // Initialize the trashq and workers
trashq = NewWorkQueue()
- go RunTrashWorker(trashq)
+ for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
+ go RunTrashWorker(trashq)
+ }
// Start emptyTrash goroutine
doneEmptyingTrash := make(chan bool)
log.Printf("Error notifying init daemon: %v", err)
}
log.Println("listening at", listener.Addr())
- srv := &http.Server{Handler: router}
+ srv := &server{}
+ srv.Handler = router
srv.Serve(listener)
}
"regexp"
"strings"
"sync"
+ "sync/atomic"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
func (v *S3Volume) EmptyTrash() {
var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
- // Use a merge sort to find matching sets of trash/X and recent/X.
- trashL := s3Lister{
- Bucket: v.bucket.Bucket,
- Prefix: "trash/",
- PageSize: v.IndexPageSize,
- }
// Define "ready to delete" as "...when EmptyTrash started".
startT := time.Now()
- for trash := trashL.First(); trash != nil; trash = trashL.Next() {
+
+ emptyOneKey := func(trash *s3.Key) {
loc := trash.Key[6:]
if !v.isKeepBlock(loc) {
- continue
+ return
}
- bytesInTrash += trash.Size
- blocksInTrash++
+ atomic.AddInt64(&bytesInTrash, trash.Size)
+ atomic.AddInt64(&blocksInTrash, 1)
trashT, err := time.Parse(time.RFC3339, trash.LastModified)
if err != nil {
log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err)
- continue
+ return
}
recent, err := v.bucket.Head("recent/"+loc, nil)
if err != nil && os.IsNotExist(v.translateError(err)) {
if err != nil {
log.Printf("error: %s: EmptyTrash: Untrash(%q): %s", v, loc, err)
}
- continue
+ return
} else if err != nil {
log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err)
- continue
+ return
}
recentT, err := v.lastModified(recent)
if err != nil {
log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
- continue
+ return
}
if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc)
v.fixRace(loc)
v.Touch(loc)
- continue
+ return
}
_, err := v.bucket.Head(loc, nil)
if os.IsNotExist(err) {
log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc)
v.fixRace(loc)
- continue
+ return
} else if err != nil {
log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err)
- continue
+ return
}
}
if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
- continue
+ return
}
err = v.bucket.Del(trash.Key)
if err != nil {
log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err)
- continue
+ return
}
- bytesDeleted += trash.Size
- blocksDeleted++
+ atomic.AddInt64(&bytesDeleted, trash.Size)
+ atomic.AddInt64(&blocksDeleted, 1)
_, err = v.bucket.Head(loc, nil)
- if os.IsNotExist(err) {
- err = v.bucket.Del("recent/" + loc)
- if err != nil {
- log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err)
- }
- } else if err != nil {
- log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err)
+ if err == nil {
+ log.Printf("warning: %s: EmptyTrash: HEAD %q succeeded immediately after deleting %q", v, loc, loc)
+ return
+ }
+ if !os.IsNotExist(v.translateError(err)) {
+ log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err)
+ return
+ }
+ err = v.bucket.Del("recent/" + loc)
+ if err != nil {
+ log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err)
}
}
+
+ var wg sync.WaitGroup
+ todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers)
+ for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for key := range todo {
+ emptyOneKey(key)
+ }
+ }()
+ }
+
+ trashL := s3Lister{
+ Bucket: v.bucket.Bucket,
+ Prefix: "trash/",
+ PageSize: v.IndexPageSize,
+ }
+ for trash := trashL.First(); trash != nil; trash = trashL.Next() {
+ todo <- trash
+ }
+ close(todo)
+ wg.Wait()
+
if err := trashL.Error(); err != nil {
log.Printf("error: %s: EmptyTrash: lister: %s", v, err)
}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "crypto/tls"
+ "net"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+)
+
+type server struct {
+ http.Server
+
+ // channel (size=1) with the current keypair
+ currentCert chan *tls.Certificate
+}
+
+func (srv *server) Serve(l net.Listener) error {
+ if theConfig.TLSCertificateFile == "" && theConfig.TLSKeyFile == "" {
+ return srv.Server.Serve(l)
+ }
+ // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
+ srv.TLSConfig = &tls.Config{
+ GetCertificate: srv.getCertificate,
+ PreferServerCipherSuites: true,
+ CurvePreferences: []tls.CurveID{
+ tls.CurveP256,
+ tls.X25519,
+ },
+ MinVersion: tls.VersionTLS12,
+ CipherSuites: []uint16{
+ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ },
+ }
+ srv.currentCert = make(chan *tls.Certificate, 1)
+ go srv.refreshCertificate(theConfig.TLSCertificateFile, theConfig.TLSKeyFile)
+ return srv.Server.ServeTLS(l, "", "")
+}
+
+func (srv *server) refreshCertificate(certfile, keyfile string) {
+ cert, err := tls.LoadX509KeyPair(certfile, keyfile)
+ if err != nil {
+ log.WithError(err).Fatal("error loading X509 key pair")
+ }
+ srv.currentCert <- &cert
+
+ reload := make(chan os.Signal, 1)
+ signal.Notify(reload, syscall.SIGHUP)
+ for range reload {
+ cert, err := tls.LoadX509KeyPair(certfile, keyfile)
+ if err != nil {
+ log.WithError(err).Warn("error loading X509 key pair")
+ continue
+ }
+ // Throw away old cert and start using new one
+ <-srv.currentCert
+ srv.currentCert <- &cert
+ }
+}
+
+func (srv *server) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ if srv.currentCert == nil {
+ panic("srv.currentCert not initialized")
+ }
+ cert := <-srv.currentCert
+ srv.currentCert <- cert
+ return cert, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "testing"
+)
+
+func TestTLS(t *testing.T) {
+ defer func() {
+ theConfig.TLSKeyFile = ""
+ theConfig.TLSCertificateFile = ""
+ }()
+ theConfig.TLSKeyFile = "../api/tmp/self-signed.key"
+ theConfig.TLSCertificateFile = "../api/tmp/self-signed.pem"
+ srv := &server{}
+ srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("OK"))
+ })
+ l, err := net.Listen("tcp", ":")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ go srv.Serve(l)
+ defer srv.Shutdown(context.Background())
+ c := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+ resp, err := c.Get("https://" + l.Addr().String() + "/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Error(err)
+ }
+ if !bytes.Equal(body, []byte("OK")) {
+ t.Errorf("expected OK, got %q", body)
+ }
+}
How often to check for (and delete) trashed blocks whose
TrashLifetime has expired.
+TrashWorkers:
+
+ Maximum number of concurrent trash operations. Default is 1, i.e.,
+ trash lists are processed serially.
+
+EmptyTrashWorkers:
+
+ Maximum number of concurrent block deletion operations (per
+ volume) when emptying trash. Default is 1.
+
+PullWorkers:
+
+ Maximum number of concurrent pull operations. Default is 1, i.e.,
+ pull lists are processed serially.
+
+TLSCertificateFile:
+
+ Path to server certificate file in X509 format. Enables TLS mode.
+
+ Example: /var/lib/acme/live/keep0.example.com/fullchain
+
+TLSKeyFile:
+
+ Path to server key file in X509 format. Enables TLS mode.
+
+ The key pair is read from disk during startup, and whenever SIGHUP
+ is received.
+
+ Example: /var/lib/acme/live/keep0.example.com/privkey
+
Volumes:
List of storage volumes. If omitted or empty, the default is to
"strconv"
"strings"
"sync"
+ "sync/atomic"
"syscall"
"time"
)
// and deletes those with deadline < now.
func (v *UnixVolume) EmptyTrash() {
var bytesDeleted, bytesInTrash int64
- var blocksDeleted, blocksInTrash int
+ var blocksDeleted, blocksInTrash int64
- err := filepath.Walk(v.Root, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- log.Printf("EmptyTrash: filepath.Walk: %v: %v", path, err)
- return nil
- }
+ doFile := func(path string, info os.FileInfo) {
if info.Mode().IsDir() {
- return nil
+ return
}
matches := unixTrashLocRegexp.FindStringSubmatch(path)
if len(matches) != 3 {
- return nil
+ return
}
deadline, err := strconv.ParseInt(matches[2], 10, 64)
if err != nil {
log.Printf("EmptyTrash: %v: ParseInt(%v): %v", path, matches[2], err)
- return nil
+ return
}
- bytesInTrash += info.Size()
- blocksInTrash++
+ atomic.AddInt64(&bytesInTrash, info.Size())
+ atomic.AddInt64(&blocksInTrash, 1)
if deadline > time.Now().Unix() {
- return nil
+ return
}
err = v.os.Remove(path)
if err != nil {
log.Printf("EmptyTrash: Remove %v: %v", path, err)
+ return
+ }
+ atomic.AddInt64(&bytesDeleted, info.Size())
+ atomic.AddInt64(&blocksDeleted, 1)
+ }
+
+ type dirent struct {
+ path string
+ info os.FileInfo
+ }
+ var wg sync.WaitGroup
+ todo := make(chan dirent, theConfig.EmptyTrashWorkers)
+ for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for e := range todo {
+ doFile(e.path, e.info)
+ }
+ }()
+ }
+
+ err := filepath.Walk(v.Root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ log.Printf("EmptyTrash: filepath.Walk: %v: %v", path, err)
return nil
}
- bytesDeleted += info.Size()
- blocksDeleted++
+ todo <- dirent{path, info}
return nil
})
+ close(todo)
+ wg.Wait()
if err != nil {
log.Printf("EmptyTrash error for %v: %v", v.String(), err)
File.open(@tmpdir+'/succeed', 'w') do |f| end
invoke_sync binstubs: ['new_user']
spied = File.read(@tmpdir+'/spy')
- assert_match %r{useradd -m -c active -s /bin/bash -G fuse active}, spied
+ assert_match %r{useradd -m -c active -s /bin/bash -G (fuse)? active}, spied
# BUG(TC): This assertion succeeds only if docker and fuse groups
# exist on the host, but is insensitive to the admin group (groups
# are quietly ignored by login-sync if they don't exist on the
# current host).
- assert_match %r{useradd -m -c adminroot -s /bin/bash -G docker(,admin)?,fuse adminroot}, spied
+ assert_match %r{useradd -m -c adminroot -s /bin/bash -G (docker)?(,admin)?(,fuse)? adminroot}, spied
end
def test_useradd_success
# binstub_new_user/useradd will succeed.
File.open(@tmpdir+'/succeed', 'w') do |f|
f.puts 'useradd -m -c active -s /bin/bash -G fuse active'
+ f.puts 'useradd -m -c active -s /bin/bash -G active'
# Accept either form; see note about groups in test_useradd_error.
f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,fuse adminroot'
f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,admin,fuse adminroot'
+ f.puts 'useradd -m -c adminroot -s /bin/bash -G docker adminroot'
+ f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,admin adminroot'
end
$stderr.puts "*** Expect crash after getpwnam() fails:"
invoke_sync binstubs: ['new_user']
import stat
import tempfile
import shutil
+import errno
from functools import partial
import arvados
import StringIO
logger.info("%s passed", name)
else:
if isinstance(detail_content, StringIO.StringIO):
- sys.stderr.write(detail_content.getvalue())
+ detail_content.seek(0)
+ chunk = detail_content.read(4096)
+ while chunk:
+ try:
+ sys.stderr.write(chunk)
+ chunk = detail_content.read(4096)
+ except IOError as e:
+ if e.errno == errno.EAGAIN:
+ # try again (probably pipe buffer full)
+ pass
+ else:
+ raise
logger.info("%s failed", name)
return code
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
check_output.assert_called_with(['scontrol', 'update', 'NodeName=compute99', 'Weight=1000', 'Features=instancetype=z1.test'])
+
+ @mock.patch('subprocess.check_output')
+ def test_failed_arvados_calls_retried(self, check_output):
+ super(SLURMComputeNodeSetupActorTestCase, self).test_failed_arvados_calls_retried()
+
+ @mock.patch('subprocess.check_output')
+ def test_subscribe(self, check_output):
+ super(SLURMComputeNodeSetupActorTestCase, self).test_subscribe()
+
+ @mock.patch('subprocess.check_output')
+ def test_creation_with_arvados_node(self, check_output):
+ super(SLURMComputeNodeSetupActorTestCase, self).test_creation_with_arvados_node()
ENV GEM_PATH /var/lib/gems
ENV PATH $PATH:/var/lib/gems/bin
-ENV GOVERSION 1.8.3
+ENV GOVERSION 1.10.1
# Install golang binary
RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
EOF = None
def __init__(self, job_uuid):
- logger.debug('load stderr events for job %s', job_uuid)
self.job_uuid = job_uuid
+ self.event_types = (['stderr'] if '-8i9sb-' in job_uuid else ['crunchstat', 'arv-mount'])
+ logger.debug('load %s events for job %s', self.event_types, self.job_uuid)
def __str__(self):
return self.job_uuid
last_id = 0
filters = [
['object_uuid', '=', self.job_uuid],
- ['event_type', '=', 'stderr']]
+ ['event_type', 'in', self.event_types]]
try:
while True:
page = arvados.api().logs().index(
projects / arvados.git / commitdiff
