sdk/python/tests/fed-migrate/*.cwlex
doc/install/*.xlsx
sdk/cwl/tests/wf/hello.txt
-sdk/cwl/tests/wf/indir1/hello2.txt
\ No newline at end of file
+sdk/cwl/tests/wf/indir1/hello2.txt
+sdk/cwl/tests/chipseq/data/Genomes/*
\ No newline at end of file
remote: https://rubygems.org/
specs:
RedCloth (4.3.2)
- actioncable (5.2.6)
- actionpack (= 5.2.6)
+ actioncable (5.2.6.3)
+ actionpack (= 5.2.6.3)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
+ actionmailer (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.6)
- actionview (= 5.2.6)
- activesupport (= 5.2.6)
+ actionpack (5.2.6.3)
+ actionview (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.6)
- activesupport (= 5.2.6)
+ actionview (5.2.6.3)
+ activesupport (= 5.2.6.3)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.6)
- activesupport (= 5.2.6)
+ activejob (5.2.6.3)
+ activesupport (= 5.2.6.3)
globalid (>= 0.3.6)
- activemodel (5.2.6)
- activesupport (= 5.2.6)
- activerecord (5.2.6)
- activemodel (= 5.2.6)
- activesupport (= 5.2.6)
+ activemodel (5.2.6.3)
+ activesupport (= 5.2.6.3)
+ activerecord (5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
arel (>= 9.0)
- activestorage (5.2.6)
- actionpack (= 5.2.6)
- activerecord (= 5.2.6)
+ activestorage (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
marcel (~> 1.0.0)
- activesupport (5.2.6)
+ activesupport (5.2.6.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
multipart-post (>= 1.2, < 3)
ffi (1.10.0)
flamegraph (0.9.5)
- globalid (0.4.2)
- activesupport (>= 4.2.0)
+ globalid (1.0.0)
+ activesupport (>= 5.0)
googleauth (0.9.0)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.10.0)
+ loofah (2.14.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (1.0.1)
+ marcel (1.0.2)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
mime-types (3.2.2)
mime-types-data (~> 3.2015)
mime-types-data (3.2019.0331)
- mini_mime (1.1.0)
- mini_portile2 (2.6.1)
+ mini_mime (1.1.2)
+ mini_portile2 (2.8.0)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh (5.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
- nio4r (2.5.7)
- nokogiri (1.12.5)
- mini_portile2 (~> 2.6.1)
+ nio4r (2.5.8)
+ nokogiri (1.13.3)
+ mini_portile2 (~> 2.8.0)
racc (~> 1.4)
npm-rails (0.2.1)
rails (>= 3.2)
rack (>= 1.2.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.6)
- actioncable (= 5.2.6)
- actionmailer (= 5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
- activemodel (= 5.2.6)
- activerecord (= 5.2.6)
- activestorage (= 5.2.6)
- activesupport (= 5.2.6)
+ rails (5.2.6.3)
+ actioncable (= 5.2.6.3)
+ actionmailer (= 5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
+ activestorage (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
bundler (>= 1.3.0)
- railties (= 5.2.6)
+ railties (= 5.2.6.3)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.3.0)
+ rails-html-sanitizer (1.4.2)
loofah (~> 2.3)
rails-perftest (0.0.7)
- railties (5.2.6)
- actionpack (= 5.2.6)
- activesupport (= 5.2.6)
+ railties (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
- rake (13.0.3)
+ rake (13.0.6)
raphael-rails (2.1.2)
rb-fsevent (0.10.3)
rb-inotify (0.10.0)
activesupport (>= 4.0)
sprockets (>= 3.0.0)
sshkey (2.0.0)
- thor (1.1.0)
+ thor (1.2.1)
thread_safe (0.3.6)
tilt (2.0.9)
tzinfo (1.2.9)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.4)
+ websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (2.1.0)
});
<% end %>
+<% if Rails.configuration.Login.LoginCluster.empty? %>
+
<div id="need-login" style="display: none">
<p>You are currently logged in as <b><%= Thread.current[:user].email %></b> (<%= Thread.current[:user].username%>, <%= Thread.current[:user].uuid %>) created at <b><%= Thread.current[:user].created_at%></b></p>
<i class="fa fa-fw fa-sign-in"></i> Use this login to access another account
</button>
-</p>
+ </p>
</div>
<div id="ready-to-link" style="display: none">
<input type="hidden" id="new-user-token-input" name="direction" value="<%=params[:direction]%>" />
<%= button_tag class: "btn btn-primary", id: "link-account-submit" do %>
<i class="fa fa-fw fa-link"></i> Link accounts
+ <% end %>
<% end %>
-<% end %>
</div>
+
+<% else %>
+<div>
+Self-serve account linking is not supported on this cluster. Please contact your Arvados administrator.
</div>
+<% end %>
target=${target#./}
echo $target
cd $target
- docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
+ docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
cd ..
done
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM centos:7 as build_x86_64
# Install go
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
FROM debian:buster as build_x86_64
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
FROM debian:bullseye as build_x86_64
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM ubuntu:bionic as build_x86_64
# Install go
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
+ARG BRANCH
FROM ubuntu:focal as build_x86_64
# Install go
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> /etc/gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
arv-put --version >/dev/null
-/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+PYTHON=`ls /usr/share/python3*/dist/python3-arvados-python-client/bin/python3 |head -n1`
+
+$PYTHON << EOF
import arvados
print("Successfully imported arvados")
EOF
echo $TARGET
cd $TARGET
-time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
popd
if test -z "$packages" ; then
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
+ # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
+ # docker run command below.
+ # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
+ cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+ printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
# Build packages.
if docker run \
--rm \
"${docker_volume_args[@]}" \
+ -v $tmpfile:/root/.bundle/config \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
--env "FORCE_BUILD=$FORCE_BUILD" \
--env "ARCH=$ARCH" \
+ --env "MAKE=make --jobs $cores" \
"$IMAGE" $COMMAND
then
echo
FINAL_EXITCODE=$?
echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
fi
+ # Clean up the bundle config file
+ rm -f $tmpfile
fi
if test -n "$package_fails" ; then
Build only a specific package (or ONLY_BUILD from environment)
--arch <arch>
Build a specific architecture (or ARCH from environment, defaults to native architecture)
+--force-build
+ Build even if the package exists upstream or if it has already been
+ built locally
+--force-test
+ Test even if there is no new untested package
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,rc,target:,only-build:,arch:,build-version: \
+ help,debug,upload,rc,target:,force-test,only-build:,force-build,arch:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --force-test)
+ FORCE_TEST=1
+ ;;
+ --force-build)
+ FORCE_BUILD=1
+ ;;
--only-build)
ONLY_BUILD="$2"; shift
;;
build_args+=(--only-build "$ONLY_BUILD")
fi
+if [[ -n "$FORCE_BUILD" ]]; then
+ build_args+=(--force-build)
+fi
+
+if [[ -n "$FORCE_TEST" ]]; then
+ build_args+=(--force-test)
+fi
+
if [[ -n "$ARCH" ]]; then
build_args+=(--arch "$ARCH")
fi
cd "$srcdir"
mkdir -p tmp
git rev-parse HEAD >git-commit.version
- bundle package --all
+ bundle config set cache_all true
+ bundle package
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
# Build the workbench server package
test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
if [[ "$?" == "0" ]] ; then
+ calculate_go_package_version arvados_server_version cmd/arvados-server
+ arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
(
set -e
- calculate_go_package_version arvados_server_version cmd/arvados-server
- arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
-
# The workbench package has a build-time dependency on the arvados-server
# package for config manipulation, so install it first.
cd $WORKSPACE/cmd/arvados-server
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
+ bundle config set --local system 'true' >"$STDOUT_IF_DEBUG"
+ bundle install >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
- user/getting_started/community.html.textile.liquid
- Walkthough:
- user/tutorials/wgs-tutorial.html.textile.liquid
- - Run a workflow using Workbench:
+ - Using Workbench:
- user/getting_started/workbench.html.textile.liquid
- user/tutorials/tutorial-workflow-workbench.html.textile.liquid
+ - user/topics/workbench-migration.html.textile.liquid
- Working at the Command Line:
- user/getting_started/setup-cli.html.textile.liquid
- user/reference/api-tokens.html.textile.liquid
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.5 or newer is required.
+Ruby 2.6 or newer is required.
* "Option 1: Install from packages":#packages
* "Option 2: Install with RVM":#rvm
h3. Centos 7
-The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install Ruby 2.5 or later.
+The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5 or newer, which is sufficient for Arvados.
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+
+Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
<notextile>
<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby ruby-dev</span></code></pre>
<notextile>
<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
</span></code></pre></notextile>
-This command installs the latest Ruby 2.5.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
<pre><code><span class="userinput">mkdir -p ~/src
cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.8.tar.gz | tar xz
-cd ruby-2.5.8
+curl -f https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz | tar xz
+cd ruby-2.7.5
./configure --disable-install-rdoc
make
sudo make install
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-If you plan to use custom certificates, please set the variable <i>USE_LETSENCRYPT=no</i> and copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ (usually "./certs") in the remote directory where you copied the @provision.sh@ script. From this dir, the provision script will install the certificates required for the role you're installing.
+Copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ in the remote directory where you copied the @provision.sh@ script. The provision script will find the certificates there.
The script expects cert/key files with these basenames (matching the role except for <i>keepweb</i>, which is split in both <i>download / collections</i>):
* "collections" # Part of keepweb
* "keepproxy"
-Ie., for 'keepproxy', the script will look for
+E.g. for 'keepproxy', the script will look for
<notextile>
<pre><code>${CUSTOM_CERTS_DIR}/keepproxy.crt
${CUSTOM_CERTS_DIR}/keepproxy.key
</code></pre>
</notextile>
+
+Make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
If a user needs to log in to Arvados with a upstream account or provider, they may end up with two Arvados user accounts. If the user still has the ability to log in with the old account, they can use the "self-serve account linking":{{site.baseurl}}/user/topics/link-accounts.html feature of workbench. However, if the user does not have the ability to log in with both upstream accounts, the admin can also link the accounts using the command line.
+bq. NOTE: self-serve account linking is currently not supported on LoginCluster federations and needs to be performed manually by the site admin.
+
h3. Step 1: Determine user uuids
User uuids can be determined by browsing workbench or using @arv user list@ at the command line.
h2. Configuration
-First, ensure automatic selection of preemptible instances is not disabled in your configuration file (this is enabled by default, but can be disabled with @AlwaysUsePreemptibleInstances: false@), and add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example:
+Add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example:
<pre>
Clusters:
- ClusterID:
- Containers:
- AlwaysUsePreemptibleInstances: true
+ ClusterID:
InstanceTypes:
m4.large:
Preemptible: false
Price: 0.1
</pre>
-When @AlwaysUsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible. Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance.
+Next, you can choose to enable automatic use of preemptible instances:
+
+<pre>
+ Containers:
+ AlwaysUsePreemptibleInstances: true
+</pre>
+
+If @AlwaysUsePreemptibleInstances@ is "true", child containers (workflow steps) will always select preemptible instances, regardless of user option.
+
+If @AlwaysUsePreemptibleInstances@ is "false" (the default) or unspecified, preemptible instance are "used when requested by the user.":{{site.baseurl}}/user/cwl/cwl-run-options.html#preemptible
+
+Note that regardless of the value of @AlwaysUsePreemptibleInstances@, the top level workflow runner container always runs in a reserved (non-preemptible) instance, to avoid situations where the workflow runner is killed requiring the entire to be restarted.
No additional configuration is required, "arvados-dispatch-cloud":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html will now start preemptible instances where appropriate.
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2022-02-10)
+h2(#main). development main (as of 2022-03-08)
"previous: Upgrading to 2.3.0":#v2_3_0
+h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
+
+Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
+
+This means that they are now installed under @/usr/share/python3.8@ (before, the path was @/usr/share/python3@). If you rely on the @python3@ executable from the packages (e.g. to load a virtualenv), you may need to update the path to that executable.
+
+h3. Minimum supported Ruby version is now 2.6
+
+The minimum supported Ruby version is now 2.6. If you are running Arvados on Debian 10 or Ubuntu 18.04, you may need to switch to using RVM or upgrade your OS. See "Install Ruby and Bundler":../install/ruby.html for more information.
+
h3. Anonymous token changes
The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary. If the anonymous token in @config.yml@ is specified as a full V2 token, that will now generate a warning - it should be updated to list just the secret (i.e. the part after the last forward slash).
The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config@. The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
+h2. Exported vocabulary definition
+
+When configured, the Controller also exports the "metadata vocabulary definition":{{site.baseurl}}/admin/metadata-vocabulary.html in JSON format. This functionality is useful for clients like Workbench2 and the Python SDK to provide "identifier to human-readable labels" translations facilities for reading and writing objects on the system. This is available at @/arvados/v1/vocabulary@, for example @https://{{ site.arvados_api_host }}/arvados/v1/vocabulary@.
+
h2. Workbench examples
Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
h3. Conditions of creating a Collection
-The @portable_data_hash@ and @manifest_text@ attributes must be provided when creating a Collection. The cryptographic digest of the supplied @manifest_text@ must match the supplied @portable_data_hash@.
+If a new @portable_data_hash@ is specified when creating or updating a Collection, it must match the cryptographic digest of the supplied @manifest_text@.
h3. Side effects of creating a Collection
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
|collection|object||query||
+|replace_files|object|Initialize files and directories using content from other collections|query||
+
+The new collection's content can be initialized by providing a @manifest_text@ key in the provided @collection@ object, or by using the @replace_files@ option (see "replace_files":#replace_files below).
h3. delete
|_. Argument |_. Type |_. Description |_. Location |_. Example |
{background:#ccffcc}.|uuid|string|The UUID of the Collection in question.|path||
|collection|object||query||
+|replace_files|object|Delete and replace files and directories using content from other collections|query||
+
+The collection's content can be updated by providing a @manifest_text@ key in the provided @collection@ object, or by using the @replace_files@ option (see "replace_files":#replace_files below).
h3. untrash
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
{background:#ccffcc}.|uuid|string|The UUID of the Collection to get usage.|path||
+
+h2(#replace_files). Using "replace_files" to create/update collections
+
+The @replace_files@ option can be used with the @create@ and @update@ APIs to efficiently copy individual files and directory trees from other collections, and copy/rename/delete items within an existing collection, without transferring any file data.
+
+@replace_files@ keys indicate target paths in the new collection, and values specify sources that should be copied to the target paths.
+* Each target path must be an absolute canonical path beginning with @/@. It must not contain @.@ or @..@ components, consecutive @/@ characters, or a trailing @/@ after the final component.
+* Each source must be either an empty string (signifying that the target path is to be deleted), or @PDH/path@ where @PDH@ is the portable data hash of a collection on the cluster and @/path@ is a file or directory in that collection.
+* In an @update@ request, sources may reference the current portable data hash of the collection being updated.
+
+Example: delete @foo.txt@ from a collection
+
+<notextile><pre>
+"replace_files": {
+ "/foo.txt": ""
+}
+</pre></notextile>
+
+Example: rename @foo.txt@ to @bar.txt@ in a collection with portable data hash @fa7aeb5140e2848d39b416daeef4ffc5+45@
+
+<notextile><pre>
+"replace_files": {
+ "/foo.txt": "",
+ "/bar.txt": "fa7aeb5140e2848d39b416daeef4ffc5+45/foo.txt"
+}
+</pre></notextile>
+
+Example: delete current contents, then add content from multiple collections
+
+<notextile><pre>
+"replace_files": {
+ "/": "",
+ "/copy of collection 1": "1f4b0bc7583c2a7f9102c395f4ffc5e3+45/",
+ "/copy of collection 2": "ea10d51bcf88862dbcc36eb292017dfd+45/"
+}
+</pre></notextile>
+
+Example: replace entire collection with a copy of a subdirectory from another collection
+
+<notextile><pre>
+"replace_files": {
+ "/": "1f4b0bc7583c2a7f9102c395f4ffc5e3+45/subdir"
+}
+</pre></notextile>
+
+A target path with a non-empty source cannot be the ancestor of another target path in the same request. For example, the following request is invalid:
+
+<notextile><pre>
+"replace_files": {
+ "/foo": "fa7aeb5140e2848d39b416daeef4ffc5+45/",
+ "/foo/this_will_return_an_error": ""
+}
+</pre></notextile>
h2(#supportedlinux). Supported GNU/Linux distributions
table(table table-bordered table-condensed).
-|_. Distribution|_. State|_. Last supported version|
+|_. Distribution|_. State|_. Last supported Arvados version|
|CentOS 7|Supported|Latest|
|Debian 11 ("bullseye")|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
## arvados controller
## arvados websocket
## arvados cloud dispatcher
+## arvados keepbalance
# WORKBENCH node:
## arvados workbench
## arvados workbench2
The <i>multi_host</i> example includes Let's Encrypt salt code to automatically request and install the certificates for the public-facing hosts (API/controller, Workbench, Keepproxy/Keepweb) using AWS' Route53.
-{% include 'install_custom_certificates' %}
+{% include 'multi_host_install_custom_certificates' %}
+
+If you want to use valid certificates provided by Let's Encrypt, set the variable <i>SSL_MODE=lets-encrypt</i> and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
h4. API
<notextile>
<pre><code>scp -r provision.sh local* user@host:
-ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher
+ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher,keepbalance
</code></pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-# "Single host install using the provision.sh script":#single_host
-# "Choose the desired configuration":#choose_configuration
-## "Single host / single hostname":#single_host_single_hostnames
-## "Single host / multiple hostnames (Alternative configuration)":#single_host_multiple_hostnames
-## "Further customization of the installation (modifying the salt pillars and states)":#further_customization
+# "Limitations of the single host install":#limitations
+# "Prerequisites":#prerequisites
+# "Download the installer":#single_host
+# "Choose the SSL configuration":#certificates
+## "Using a self-signed certificate":#self-signed
+## "Using a Let's Encrypt certificate":#lets-encrypt
+## "Bring your own certificate":#bring-your-own
+# "Further customization of the installation (modifying the salt pillars and states)":#further_customization
# "Run the provision.sh script":#run_provision_script
-# "Final configuration steps":#final_steps
-## "Install the CA root certificate (required in both alternatives)":#ca_root_certificate
-## "DNS configuration (single host / multiple hostnames)":#single_host_multiple_hostnames_dns_configuration
+# "Install the CA root certificate":#ca_root_certificate
# "Initial user and login":#initial_user
# "Test the installed cluster running a simple workflow":#test_install
# "After the installation":#post_install
-h2(#single_host). Single host install using the provision.sh script
+h2(#limitations). Limitations of the single host install
-<b>NOTE: The single host installation is not recommended for production use.</b>
+<b>NOTE: The single host installation is a good choice for evaluating Arvados, but it is not recommended for production use.</b>
-{% include 'branchname' %}
+Using the default configuration, this installation method has a number of limitations:
-This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
+* all services run on the same machine, and they will compete for resources. This includes any compute jobs.
+* it uses the local machine disk for Keep storage (under the @/tmp@ directory). There may not be a lot of space available.
+* it installs the @crunch-dispatch-local@ dispatcher, which can run just eight concurrent CWL jobs. These jobs will be executed on the same machine that runs all the Arvados services and may well starve them of resources.
-This procedure will install all the main Arvados components to get you up and running in a single host. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host resources and its network bandwidth. As a reference, on a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
+It is possible to start with the single host installation method and modify the Arvados configuration file later to address these limitations. E.g. switch to a "different storage volume setup":{{site.baseurl}}/install/configure-s3-object-storage.html for Keep, and switch to "the cloud dispatcher":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html to provision compute resources dynamically.
-The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+h2(#prerequisites). Prerequisites and planning
-After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+Prerequisites:
-h2(#choose_configuration). Choose the desired configuration
+* git
+* a dedicated (virtual) machine for your Arvados server with at least 2 cores and 8 GiB of RAM, running a "supported Arvados distribution":{{site.baseurl}}/install/install-manual-prerequisites.html#supportedlinux
+* a DNS hostname that resolves to the IP address of your Arvados server
+* ports 443, 8800-8805 need to be reachable from your client (configurable in @local.params@, see below)
+* port 80 needs to be reachable from everywhere on the internet (only when using "Let's Encrypt":#lets-encrypt)
+* an SSL certificate matching the hostname in use (only when using "bring your own certificate":#bring-your-own)
-For documentation's sake, we will use the cluster name <i>arva2</i> and the domain <i>arv.local</i>. If you don't change them as required in the next steps, installation won't proceed.
+h2(#single_host). Download the installer
-Arvados' single host installation can be done in two fashions:
+{% include 'branchname' %}
-* Using a single hostname, assigning <i>a different port (other than 443) for each user-facing service</i>: This choice is easier to setup, but the user will need to know the port/s for the different services she wants to connect to.
-* Using multiple hostnames on the same IP: this setup involves a few extra steps but each service will have a meaningful hostname so it will make easier to access them later.
+This procedure will install all the main Arvados components to get you up and running in a single host.
-Once you decide which of these choices you prefer, copy one the two example configuration files and directory, and edit them to suit your needs.
+This is a package-based installation method, however the installation script is currently distributed in source form via @git@:
+
+<notextile>
+<pre><code>git clone https://git.arvados.org/arvados.git
+git checkout {{ branchname }}
+cd arvados/tools/salt-install
+</code></pre>
+</notextile>
+
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located in the "arvados-formula git repository":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script. The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+First, copy the configuration files:
-h3(#single_host_single_hostnames). Single host / single hostname
<notextile>
<pre><code>cp local.params.example.single_host_single_hostname local.params
cp -r config_examples/single_host/single_hostname local_config_dir
</code></pre>
</notextile>
-Edit the variables in the <i>local.params</i> file. Pay attention to the <b>*_PORT, *_TOKEN</b> and <b>*KEY</b> variables.
+Edit the variables in the <i>local.params</i> file. Pay attention to the <b>*_PORT, *_TOKEN</b> and <b>*KEY</b> variables. The *SSL_MODE* variable is discussed in the next section.
+
+h2(#certificates). Choose the SSL configuration (SSL_MODE)
+
+Arvados requires an SSL certificate to work correctly. This installer supports these options:
-The <i>single_host</i> examples use self-signed SSL certificates, which are deployed using the same mechanism used to deploy custom certificates.
+* @self-signed@: let the installer create a self-signed certificate
+* @lets-encrypt@: automatically obtain and install an SSL certificate for your hostname
+* @bring-your-own@: supply your own certificate in the `certs` directory
-{% include 'install_custom_certificates' %}
+h3(#self-signed). Using a self-signed certificate
-If you want to use valid certificates provided by Let's Encrypt, please set the variable <i>USE_LETSENCRYPT=yes</i> and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
+In the default configuration, this installer uses self-signed certificate(s):
-h3(#single_host_multiple_hostnames). Single host / multiple hostnames (Alternative configuration)
<notextile>
-<pre><code>cp local.params.example.single_host_multiple_hostnames local.params
-cp -r config_examples/single_host/multiple_hostnames local_config_dir
+<pre><code>SSL_MODE="self-signed"
</code></pre>
</notextile>
-Edit the variables in the <i>local.params</i> file.
+When connecting to the Arvados web interface for the first time, you will need to accept the self-signed certificate as trusted to bypass the browser warnings.
-h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
+h3(#lets-encrypt). Using a Let's Encrypt certificate
-If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the <i>pillars/arvados.sls</i> one. Any extra <i>state</i> file you add under <i>local_config_dir/states</i> will be added to the salt run and applied to the host.
+To automatically get a valid certificate via Let's Encrypt, change the configuration like this:
-h2(#run_provision_script). Run the provision.sh script
+<notextile>
+<pre><code>SSL_MODE="lets-encrypt"
+</code></pre>
+</notextile>
-When you finished customizing the configuration, you are ready to copy the files to the host (if needed) and run the @provision.sh@ script:
+The hostname for your Arvados cluster must be defined in @HOSTNAME_EXT@ and resolve to the public IP address of your Arvados instance, so that Let's Encrypt can validate the domainname ownership and issue the certificate.
+
+When using AWS, EC2 instances can have a default hostname that ends with <i>amazonaws.com</i>. Let's Encrypt has a blacklist of domain names for which it will not issue certificates, and that blacklist includes the <i>amazonaws.com</i> domain, which means the default hostname can not be used to get a certificate from Let's Encrypt.
+
+h3(#bring-your-own). Bring your own certificate
+
+To supply your own certificate, change the configuration like this:
<notextile>
-<pre><code>scp -r provision.sh local* tests user@host:
-# if you use custom certificates (not Let's Encrypt), make sure to copy those too:
-# scp -r certs user@host:
-ssh user@host sudo ./provision.sh
+<pre><code>SSL_MODE="bring-your-own"
+CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
</code></pre>
</notextile>
-or, if you saved the @local.params@ in another directory or with some other name
+Copy your certificate files to the directory specified with the variable @CUSTOM_CERTS_DIR@. The provision script will find it there. The certificate and its key need to be copied to a file named after @HOSTNAME_EXT@. For example, if @HOSTNAME_EXT@ is defined as @my-arvados.example.net@, the script will look for
<notextile>
-<pre><code>scp -r provision.sh local* tests user@host:
-ssh user@host sudo ./provision.sh -c /path/to/your/local.params.file
+<pre><code>${CUSTOM_CERTS_DIR}/my-arvados.example.net.crt
+${CUSTOM_CERTS_DIR}/my-arvados.example.net.key
</code></pre>
</notextile>
-and wait for it to finish.
+All certificate files will be used by nginx. You may need to include intermediate certificates in your certificate file. See "the nginx documentation":http://nginx.org/en/docs/http/configuring_https_servers.html#chains for more details.
+
+h2(#further_customization). Further customization of the installation (modifying the salt pillars and states)
+
+If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the <i>pillars/arvados.sls</i> one. Any extra <i>state</i> file you add under <i>local_config_dir/states</i> will be added to the salt run and applied to the host.
+
+h2(#run_provision_script). Run the provision.sh script
-If everything goes OK, you'll get some final lines stating something like:
+When you finished customizing the configuration, you are ready to copy the files to the host (if needed) and run the @provision.sh@ script:
<notextile>
-<pre><code>arvados: Succeeded: 109 (changed=9)
-arvados: Failed: 0
+<pre><code>scp -r provision.sh local* tests user@host:
+# if you have set SSL_MODE to "bring-your-own", make sure to also copy the certificate files:
+# scp -r certs user@host:
+ssh user@host sudo ./provision.sh
</code></pre>
</notextile>
-h2(#final_steps). Final configuration steps
+and wait for it to finish. The script will need 5 to 10 minutes to install and configure everything.
+
+If everything goes OK, you'll get final output that looks similar to this:
-Once the deployment went OK, you'll need to perform a few extra steps in your local browser/host to access the cluster.
+<notextile>
+<pre><code>arvados: Succeeded: 151 (changed=36)
+arvados: Failed: 0
+</code></pre>
+</notextile>
-h3(#ca_root_certificate). Install the CA root certificate (required in both alternatives)
+h2(#ca_root_certificate). Install the CA root certificate (SSL_MODE=self-signed only)
-Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+Arvados uses SSL to encrypt communications. The web interface uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so you can add it to your workstation.
</code></pre>
</notextile>
-h3(#single_host_multiple_hostnames_dns_configuration). DNS configuration (single host / multiple hostnames)
-
-When using multiple hostnames, after the setup is done, you need to set up your DNS to be able to access the cluster.
-
-If you don't have access to the domain's DNS to add the required entries, the simplest way to do it is to edit your @/etc/hosts@ file (as root):
-
-<notextile>
-<pre><code>export CLUSTER="arva2"
-export DOMAIN="arv.local"
-export HOST_IP="127.0.0.2" # This is valid either if installing in your computer directly
- # or in a Vagrant VM. If you're installing it on a remote host
- # just change the IP to match that of the host.
-echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
-</code></pre>
-</notextile>
-
h2(#initial_user). Initial user and login
-At this point you should be able to log into the Arvados cluster. The initial URL will be:
-
-* https://workbench.arva2.arv.local
-
-or, in general, the url format will be:
+At this point you should be able to log on to your new Arvados cluster. The workbench URL will be
-* https://workbench.@<cluster>.<domain>@
+* https://@HOSTNAME_EXT@
-By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
-
-Assuming you didn't change these values in the @local.params@ file, the initial credentials are:
-
-* User: 'admin'
-* Password: 'password'
-* Email: 'admin@arva2.arv.local'
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster. The username, password and e-mail address for the initial user are configured in the @local.params@ file. Log in with the e-mail address and password.
h2(#test_install). Test the installed cluster running a simple workflow
"owner_uuid":"arva2-tpzed-000000000000000",
...
}
-Uploading arvados/jobs' docker image to the project
-2.1.1: Pulling from arvados/jobs
-8559a31e96f4: Pulling fs layer
-...
-Status: Downloaded newer image for arvados/jobs:2.1.1
-docker.io/arvados/jobs:2.1.1
-2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
-2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
-arva2-4zz18-1u5pvbld7cvxuy2
Creating initial user ('admin')
Setting up user ('admin')
{
This is a package-based installation method.
-
-
h2(#provisioning_arvados). Provisioning Arvados with Saltstack
The "tools/salt-install":https://git.arvados.org/arvados.git/tree/{{ branchname }}:/tools/salt-install directory in the Arvados git repository contains a script that you can run in the node/s where you want to install Arvados' components (the @provision.sh@ script) and a few configuration examples for different setups, that you can use to customize your installation.
for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
print("got collection " + c["uuid"])
{% endcodeblock %}
+
+h2. Querying the vocabulary definition
+
+The Python SDK provides facilities to interact with the "active metadata vocabulary":{{ site.baseurl }}/admin/metadata-vocabulary.html in the system. The developer can do key and value lookups in a case-insensitive manner:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+[k.identifier for k in set(voc.key_aliases.values())]
+# Example output: ['IDTAGCOLORS', 'IDTAGFRUITS', 'IDTAGCOMMENT', 'IDTAGIMPORTANCES', 'IDTAGCATEGORIES', 'IDTAGSIZES', 'IDTAGANIMALS']
+voc['IDTAGSIZES'].preferred_label
+# Example output: 'Size'
+[v.preferred_label for v in set(voc['size'].value_aliases.values())]
+# Example output: ['S', 'M', 'L', 'XL', 'XS']
+voc['size']['s'].aliases
+# Example output: ['S', 'small']
+voc['size']['Small'].identifier
+# Example output: 'IDVALSIZES2'
+{% endcodeblock %}
+
+h2. Translating between vocabulary identifiers and labels
+
+Client software might need to present properties to the user in a human-readable form or take input from the user without requiring them to remember identifiers. For these cases, there're a couple of conversion methods that take a dictionary as input like this:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+voc.convert_to_labels({'IDTAGIMPORTANCES': 'IDVALIMPORTANCES1'})
+# Example output: {'Importance': 'Critical'}
+voc.convert_to_identifiers({'creature': 'elephant'})
+# Example output: {'IDTAGANIMALS': 'IDVALANIMALS3'}
+{% endcodeblock %}
\ No newline at end of file
If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
+
<notextile>
<pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
(python-arvados-python-client) ~$ <code class="userinput">python</code>
cudaComputeCapabilityMin: "9.0"
deviceCountMin: 1
deviceCountMax: 1
+
+ arv:UsePreemptible:
+ usePreemptible: true
{% endcodeblock %}
h2(#RunInSingleContainer). arv:RunInSingleContainer
|deviceCountMin|integer|Minimum number of GPU devices to allocate on a single node. Required.|
|deviceCountMax|integer|Maximum number of GPU devices to allocate on a single node. Optional. If not specified, same as @minDeviceCount@.|
+h2(#UsePreemptible). arv:UsePreemptible
+
+Specify whether a workflow step should request preemptible (e.g. AWS Spot market) instances. Such instances are generally cheaper, but can be taken back by the cloud provider at any time (preempted) causing the step to fail. When this happens, Arvados will automatically re-try the step, up to the configuration value of @Containers.MaxRetryAttempts@ (default 3) times.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|usePreemptible|boolean|Required, true to opt-in to using preemptible instances, false to opt-out.|
+
h2. arv:dockerCollectionPDH
This is an optional extension field appearing on the standard @DockerRequirement@. It specifies the portable data hash of the Arvados collection containing the Docker image. If present, it takes precedence over @dockerPull@ or @dockerImageId@.
|==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower)|
|==--thread-count== THREAD_COUNT|Number of threads to use for container submit and output collection.|
|==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).|
+|==--enable-preemptible==|Use preemptible instances. Control individual steps with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint.|
+|==--disable-preemptible==|Don't use preemptible instances.|
+|==--skip-schemas==|Skip loading of extension schemas (the $schemas section).|
|==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.|
|==--no-trash-intermediate==|Do not trash intermediate outputs (default).|
h3(#federation). Run workflow on a remote federated cluster
By default, the workflow runner will run on the local (home) cluster. Using @--submit-runner-cluster@ you can specify that the runner should be submitted to a remote federated cluster. When doing this, @--project-uuid@ should specify a project on that cluster. Steps making up the workflow will be submitted to the remote federated cluster by default, but the behavior of @arv:ClusterTarget@ is unchanged. Note: when using this option, any resources that need to be uploaded in order to run the workflow (such as files or Docker images) will be uploaded to the local (home) cluster, and streamed to the federated cluster on demand.
+
+h3(#preemptible). Using preemptible (spot) instances
+
+Preemptible instances typically offer lower cost computation with a tradeoff of lower service guarantees. If a compute node is preempted, Arvados will restart the computation on a new instance.
+
+If the sitewide configuration @Containers.AlwaysUsePreemptibleInstances@ is true, workflow steps will always select preemptible instances, regardless of user option.
+
+If @Containers.AlwaysUsePreemptibleInstances@ is false, you can request preemptible instances for a specific run with the @arvados-cwl-runner --enable-preemptible@ option.
+
+Within the workflow, you can control whether individual steps should be preemptible with the "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint.
+
+If a workflow requests preemptible instances with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible , but you _do not_ want to use preemptible instances, you can override it for a specific run with the @arvados-cwl-runner --disable-preemptible@ option.
+
+h3(#gpu). Use CUDA GPU instances
+
+See "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement .
To get the best perfomance from your workflows, be aware of the following Arvados features, behaviors, and best practices.
-Does your application support NVIDIA GPU acceleration? Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs.
+h3. Does your application support NVIDIA GPU acceleration?
+
+Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs.
+
+h3. Trying to reduce costs?
+
+Try "using preemptible (spot) instances":cwl-run-options.html#preemptible .
+
+h3. You have a sequence of short-running steps
If you have a sequence of short-running steps (less than 1-2 minutes each), use the Arvados extension "arv:RunInSingleContainer":cwl-extensions.html#RunInSingleContainer to avoid scheduling and data transfer overhead by running all the steps together in the same container on the same node. To use this feature, @cwltool@ must be installed in the container image. Example:
run: subworkflow-with-short-steps.cwl
{% endcodeblock %}
+h3. Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@
+
Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@ unless you specifically need them. Don't include them "just in case" because they change the default behavior and may add extra overhead.
+h3. Prefer text substitution to Javascript
+
When combining a parameter value with a string, such as adding a filename extension, write @$(inputs.file.basename).ext@ instead of @$(inputs.file.basename + 'ext')@. The first form is evaluated as a simple text substitution, the second form (using the @+@ operator) is evaluated as an arbitrary Javascript expression and requires that you declare @InlineJavascriptRequirement@.
+h3. Use @ExpressionTool@ to efficiently rearrange input files
+
Use @ExpressionTool@ to efficiently rearrange input files between steps of a Workflow. For example, the following expression accepts a directory containing files paired by @_R1_@ and @_R2_@ and produces an array of Directories containing each pair.
{% codeblock as yaml %}
}
{% endcodeblock %}
-Available compute nodes types vary over time and across different cloud providers, so try to limit the RAM requirement to what the program actually needs. However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM
+h3. Limit RAM requests to what you really need
+
+Available compute nodes types vary over time and across different cloud providers, so it is important to limit the RAM requirement to what the program actually needs. However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM
-Instead of scattering separate steps, prefer to scatter over a subworkflow.
+h3. Avoid scattering by step by step
+
+Instead of a scatter step that feeds into another scatter step, prefer to scatter over a subworkflow.
With the following pattern, @step1@ has to wait for all samples to complete before @step2@ can start computing on any samples. This means a single long-running sample can prevent the rest of the workflow from moving on:
To write workflows that are easy to modify and portable across CWL runners (in the event you need to share your workflow with others), there are several best practices to follow:
+h3. Always provide @DockerRequirement@
+
Workflows should always provide @DockerRequirement@ in the @hints@ or @requirements@ section.
+h3. Build a reusable library of components
+
Build a reusable library of components. Share tool wrappers and subworkflows between projects. Make use of and contribute to "community maintained workflows and tools":https://github.com/common-workflow-library and tool registries such as "Dockstore":http://dockstore.org .
+h3. Supply scripts as input parameters
+
CommandLineTools wrapping custom scripts should represent the script as an input parameter with the script file as a default value. Use @secondaryFiles@ for scripts that consist of multiple files. For example:
{% codeblock as yaml %}
glob: "*.fastq"
{% endcodeblock %}
+h3. Getting the temporary and output directories
+
You can get the designated temporary directory using @$(runtime.tmpdir)@ in your CWL file, or from the @$TMPDIR@ environment variable in your script.
Similarly, you can get the designated output directory using $(runtime.outdir), or from the @HOME@ environment variable in your script.
-Avoid specifying resource requirements in CommandLineTool. Prefer to specify them in the workflow. You can provide a default resource requirement in the top level @hints@ section, and individual steps can override it with their own resource requirement.
+h3. Specifying @ResourceRequirement@
+
+Avoid specifying resources in the @requirements@ section of a @CommandLineTool@, put it in the @hints@ section instead. This enables you to override the tool resource hint with a workflow step level requirement:
{% codeblock as yaml %}
cwlVersion: v1.0
class: Workflow
inputs:
inp: File
-hints:
- ResourceRequirement:
- ramMin: 1000
- coresMin: 1
- tmpdirMin: 45000
steps:
step1:
in: {inp: inp}
in: {inp: step1/inp}
out: [out]
run: tool2.cwl
- hints:
+ requirements:
ResourceRequirement:
ramMin: 2000
coresMin: 2
{% endcomment %}
{% include 'notebox_begin' %}
-This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1". There is also a new Workbench web application under development called "Workbench 2". Sites which have both Workbench applications installed will have a dropdown menu option "Switch to Workbench 2" to switch between versions.
-
-This guide will be updated to cover "Workbench 2" in the future.
+This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1". There is also a new Workbench web application under development called "Workbench 2". This guide will be updated to cover "Workbench 2" in the future. See "Workbench 2 migration":{{site.baseurl}}/user/topics/workbench-migration.html for more information.
{% include 'notebox_end' %}
You can access the Arvados Workbench used in this guide using this link:
This page describes how to link additional login accounts to the same Arvados account. This can be used to migrate login accounts, for example, from one Google account to another. It can also be used to migrate login providers, for example from LDAP to Google. In order to do this, you must be able to log into both the "old" and "new" accounts.
+bq. NOTE: If you need to link your accounts on an Arvados cluster federation where user management is centralized, this feature may not be available. If that's the case, the federation admin can do the linking manually.
+
h2. Link accounts
Follow this process to link the "new" login to the "old" login.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Workbench 2 migration"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Arvados is in the process of migrating from the classic web application, referred to as "Workbench 1", to a completely new web application, referred to as "Workbench 2".
+
+!{width: 90%}{{ site.baseurl }}/images/wb2-example.png!
+
+Workbench 2 is the new Workbench web application that will, over time, replace Workbench 1. Workbench 2 is being built based on user feedback, and it is approaching feature parity with Workbench 1. Workbench 2 has a modern look and feel and offers several advanced features and performance enhancements. Arvados clusters typically have both Workbench applications installed and have a dropdown menu option in the user menu to switch between versions.
+
+!{{ site.baseurl }}/images/switch-to-wb2.png!
+
+Workbench 2 is stable and recommended for general use, but still lacks some features available in the classic Workbench 1 application. When necessary, you can easily switch back:
+
+!{{ site.baseurl }}/images/switch-to-wb1.png!
+
+Some major improvements of Workbench 2 include:
+
+h2. General
+
+* More responsive, only loads data needed for display
+* More familiar user interface, modeled on the file explorer of MacOS and Windows.
+* Advanced search capabilities
+
+h2. Project browsing
+
+* Expanded informational columns
+* Expanded filtering options
+* Right side informational panel providing details about selected item without navigating away from the project
+* Support for adding and querying user-supplied metadata properties on Projects
+
+h2. Collection browsing
+
+* Able to browse collections with millions of files
+* Support for adding and querying user-supplied metadata properties on Collections
+* Support for viewing past versions of a collection
+
+h2. User and Group management
+
+* Able to create user groups through the GUI
+* Able to add/view/remove members of user groups, and what permissions are shared with the group
+* Able to add/view/remove permissions shared with individual users
RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version
# use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python
-RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
+RUN PYTHON=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python|head -n1` && rm -f /usr/bin/python && ln -s $PYTHON /usr/bin/python
+RUN PYTHON3=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python3|head -n1` && rm -f /usr/bin/python3 && ln -s $PYTHON3 /usr/bin/python3
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
- github.com/containerd/containerd v1.5.8 // indirect
+ github.com/containerd/containerd v1.5.10 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/go-connections v0.3.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
- github.com/opencontainers/image-spec v1.0.1 // indirect
+ github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pelletier/go-buffruneio v0.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
+github.com/containerd/containerd v1.5.10 h1:3cQ2uRVCkJVcx5VombsE7105Gl9Wrl7ORAO3+4+ogf4=
+github.com/containerd/containerd v1.5.10/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
+github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
# If false, containers are scheduled on preemptible instances
# only when requested by the submitter.
#
- # Note that arvados-cwl-runner does not currently offer a
- # feature to request preemptible instances, so this value
- # effectively acts as a cluster-wide decision about whether to
- # use preemptible instances.
- #
# This flag is ignored if no preemptible instance types are
# configured, and has no effect on top-level containers.
AlwaysUsePreemptibleInstances: true
import (
"context"
+ "fmt"
+ "net/http"
+ "os"
+ "sort"
+ "strings"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
)
// CollectionGet defers to railsProxy for everything except blob
// them.
opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
}
+ if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, "", opts.Attrs, opts.ReplaceFiles); err != nil {
+ return arvados.Collection{}, err
+ }
resp, err := conn.railsProxy.CollectionCreate(ctx, opts)
if err != nil {
return resp, err
// them.
opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
}
+ if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, opts.UUID, opts.Attrs, opts.ReplaceFiles); err != nil {
+ return arvados.Collection{}, err
+ }
resp, err := conn.railsProxy.CollectionUpdate(ctx, opts)
if err != nil {
return resp, err
}
coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
}
+
+// If replaceFiles is non-empty, populate attrs["manifest_text"] by
+// starting with the content of fromUUID (or an empty collection if
+// fromUUID is empty) and applying the specified file/directory
+// replacements.
+//
+// Return value is the (possibly modified) attrs map.
+func (conn *Conn) applyReplaceFilesOption(ctx context.Context, fromUUID string, attrs map[string]interface{}, replaceFiles map[string]string) (map[string]interface{}, error) {
+ if len(replaceFiles) == 0 {
+ return attrs, nil
+ } else if mtxt, ok := attrs["manifest_text"].(string); ok && len(mtxt) > 0 {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "ambiguous request: both 'replace_files' and attrs['manifest_text'] values provided")
+ }
+
+ // Load the current collection (if any) and set up an
+ // in-memory filesystem.
+ var dst arvados.Collection
+ if _, replacingRoot := replaceFiles["/"]; !replacingRoot && fromUUID != "" {
+ src, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: fromUUID})
+ if err != nil {
+ return nil, err
+ }
+ dst = src
+ }
+ dstfs, err := dst.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+ if err != nil {
+ return nil, err
+ }
+
+ // Sort replacements by source collection to avoid redundant
+ // reloads when a source collection is used more than
+ // once. Note empty sources (which mean "delete target path")
+ // sort first.
+ dstTodo := make([]string, 0, len(replaceFiles))
+ {
+ srcid := make(map[string]string, len(replaceFiles))
+ for dst, src := range replaceFiles {
+ dstTodo = append(dstTodo, dst)
+ if i := strings.IndexRune(src, '/'); i > 0 {
+ srcid[dst] = src[:i]
+ }
+ }
+ sort.Slice(dstTodo, func(i, j int) bool {
+ return srcid[dstTodo[i]] < srcid[dstTodo[j]]
+ })
+ }
+
+ // Reject attempt to replace a node as well as its descendant
+ // (e.g., a/ and a/b/), which is unsupported, except where the
+ // source for a/ is empty (i.e., delete).
+ for _, dst := range dstTodo {
+ if dst != "/" && (strings.HasSuffix(dst, "/") ||
+ strings.HasSuffix(dst, "/.") ||
+ strings.HasSuffix(dst, "/..") ||
+ strings.Contains(dst, "//") ||
+ strings.Contains(dst, "/./") ||
+ strings.Contains(dst, "/../") ||
+ !strings.HasPrefix(dst, "/")) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "invalid replace_files target: %q", dst)
+ }
+ for i := 0; i < len(dst)-1; i++ {
+ if dst[i] != '/' {
+ continue
+ }
+ outerdst := dst[:i]
+ if outerdst == "" {
+ outerdst = "/"
+ }
+ if outersrc := replaceFiles[outerdst]; outersrc != "" {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "replace_files: cannot operate on target %q inside non-empty target %q", dst, outerdst)
+ }
+ }
+ }
+
+ var srcidloaded string
+ var srcfs arvados.FileSystem
+ // Apply the requested replacements.
+ for _, dst := range dstTodo {
+ src := replaceFiles[dst]
+ if src == "" {
+ if dst == "/" {
+ // In this case we started with a
+ // blank manifest, so there can't be
+ // anything to delete.
+ continue
+ }
+ err := dstfs.RemoveAll(dst)
+ if err != nil {
+ return nil, fmt.Errorf("RemoveAll(%s): %w", dst, err)
+ }
+ continue
+ }
+ srcspec := strings.SplitN(src, "/", 2)
+ srcid, srcpath := srcspec[0], "/"
+ if !arvadosclient.PDHMatch(srcid) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "invalid source %q for replace_files[%q]: must be \"\" or \"PDH\" or \"PDH/path\"", src, dst)
+ }
+ if len(srcspec) == 2 && srcspec[1] != "" {
+ srcpath = srcspec[1]
+ }
+ if srcidloaded != srcid {
+ srcfs = nil
+ srccoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: srcid})
+ if err != nil {
+ return nil, err
+ }
+ // We use StubClient here because we don't
+ // want srcfs to read/write any file data or
+ // sync collection state to/from the database.
+ srcfs, err = srccoll.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+ if err != nil {
+ return nil, err
+ }
+ srcidloaded = srcid
+ }
+ snap, err := arvados.Snapshot(srcfs, srcpath)
+ if err != nil {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "error getting snapshot of %q from %q: %w", srcpath, srcid, err)
+ }
+ // Create intermediate dirs, in case dst is
+ // "newdir1/newdir2/dst".
+ for i := 1; i < len(dst)-1; i++ {
+ if dst[i] == '/' {
+ err = dstfs.Mkdir(dst[:i], 0777)
+ if err != nil && !os.IsExist(err) {
+ return nil, httpserver.Errorf(http.StatusBadRequest, "error creating parent dirs for %q: %w", dst, err)
+ }
+ }
+ }
+ err = arvados.Splice(dstfs, dst, snap)
+ if err != nil {
+ return nil, fmt.Errorf("error splicing snapshot onto path %q: %w", dst, err)
+ }
+ }
+ mtxt, err := dstfs.MarshalManifest(".")
+ if err != nil {
+ return nil, err
+ }
+ if attrs == nil {
+ attrs = make(map[string]interface{}, 1)
+ }
+ attrs["manifest_text"] = mtxt
+ return attrs, nil
+}
import (
"context"
+ "io/fs"
+ "path/filepath"
"regexp"
+ "sort"
"strconv"
+ "strings"
"time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
s.localdb.vocabularyCache = voc
}
-func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
+func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
s.setUpVocabulary(c, "")
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
for _, tt := range tests {
c.Log(c.TestName()+" ", tt.name)
+ // Create with properties
coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
Select: []string{"uuid", "properties"},
Attrs: map[string]interface{}{
} else {
c.Assert(err, check.NotNil)
}
- }
-}
-
-func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
- s.setUpVocabulary(c, "")
- ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
- tests := []struct {
- name string
- props map[string]interface{}
- success bool
- }{
- {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
- {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
- {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
- {"Empty properties", map[string]interface{}{}, true},
- }
- for _, tt := range tests {
- c.Log(c.TestName()+" ", tt.name)
- coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
+ // Create, then update with properties
+ coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
c.Assert(err, check.IsNil)
coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
UUID: coll.UUID,
}
}
+func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
+ foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
+ }})
+ c.Assert(err, check.IsNil)
+ s.localdb.signCollection(ctx, &foo)
+ foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
+ }})
+ c.Assert(err, check.IsNil)
+ s.localdb.signCollection(ctx, &foobarbaz)
+ wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
+ }})
+ c.Assert(err, check.IsNil)
+ s.localdb.signCollection(ctx, &wazqux)
+
+ ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ // Create using content from existing collections
+ dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ ReplaceFiles: map[string]string{
+ "/f": foo.PortableDataHash + "/foo.txt",
+ "/b": foobarbaz.PortableDataHash + "/foo/bar",
+ "/q": wazqux.PortableDataHash + "/",
+ "/w": wazqux.PortableDataHash + "/waz",
+ },
+ Attrs: map[string]interface{}{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ }})
+ c.Assert(err, check.IsNil)
+ s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
+
+ // Delete a file and a directory
+ dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: dst.UUID,
+ ReplaceFiles: map[string]string{
+ "/f": "",
+ "/q/waz": "",
+ }})
+ c.Assert(err, check.IsNil)
+ s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
+
+ // Move and copy content within collection
+ dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: dst.UUID,
+ ReplaceFiles: map[string]string{
+ // Note splicing content to /b/corge.txt but
+ // removing everything else from /b
+ "/b": "",
+ "/b/corge.txt": dst.PortableDataHash + "/b/baz.txt",
+ "/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
+ }})
+ c.Assert(err, check.IsNil)
+ s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
+
+ // Remove everything except one file
+ dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: dst.UUID,
+ ReplaceFiles: map[string]string{
+ "/": "",
+ "/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
+ }})
+ c.Assert(err, check.IsNil)
+ s.expectFiles(c, dst, "b/corge.txt")
+
+ // Copy entire collection to root
+ dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ ReplaceFiles: map[string]string{
+ "/": dst.PortableDataHash,
+ }})
+ c.Check(err, check.IsNil)
+ c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
+ s.expectFiles(c, dstcopy, "b/corge.txt")
+
+ // Check invalid targets, sources, and combinations
+ for _, badrepl := range []map[string]string{
+ {
+ "/foo/nope": dst.PortableDataHash + "/b",
+ "/foo": dst.PortableDataHash + "/b",
+ },
+ {
+ "/foo": dst.PortableDataHash + "/b",
+ "/foo/nope": "",
+ },
+ {
+ "/": dst.PortableDataHash + "/",
+ "/nope": "",
+ },
+ {
+ "/": dst.PortableDataHash + "/",
+ "/nope": dst.PortableDataHash + "/b",
+ },
+ {"/bad/": ""},
+ {"/./bad": ""},
+ {"/b/./ad": ""},
+ {"/b/../ad": ""},
+ {"/b/.": ""},
+ {".": ""},
+ {"bad": ""},
+ {"": ""},
+ {"/bad": "/b"},
+ {"/bad": "bad/b"},
+ {"/bad": dst.UUID + "/b"},
+ } {
+ _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: dst.UUID,
+ ReplaceFiles: badrepl,
+ })
+ c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
+ c.Check(err, check.NotNil)
+ }
+
+ // Check conflicting replace_files and manifest_text
+ _, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: dst.UUID,
+ ReplaceFiles: map[string]string{"/": ""},
+ Attrs: map[string]interface{}{
+ "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
+ }})
+ c.Logf("replace_files+manifest_text\n... got err: %s", err)
+ c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
+}
+
+// expectFiles checks coll's directory structure against the given
+// list of expected files and empty directories. An expected path with
+// a trailing slash indicates an empty directory.
+func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
+ client := arvados.NewClientFromEnv()
+ ac, err := arvadosclient.New(client)
+ c.Assert(err, check.IsNil)
+ kc, err := keepclient.MakeKeepClient(ac)
+ c.Assert(err, check.IsNil)
+ cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
+ c.Assert(err, check.IsNil)
+ var found []string
+ nonemptydirs := map[string]bool{}
+ fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
+ dir, _ := filepath.Split(path)
+ nonemptydirs[dir] = true
+ if d.IsDir() {
+ if path != "/" {
+ path += "/"
+ }
+ if !nonemptydirs[path] {
+ nonemptydirs[path] = false
+ }
+ } else {
+ found = append(found, path)
+ }
+ return nil
+ })
+ for d, nonempty := range nonemptydirs {
+ if !nonempty {
+ found = append(found, d)
+ }
+ }
+ for i, path := range found {
+ if path != "/" {
+ found[i] = strings.TrimPrefix(path, "/")
+ }
+ }
+ sort.Strings(found)
+ sort.Strings(expected)
+ c.Check(found, check.DeepEquals, expected)
+}
+
func (s *CollectionSuite) TestSignatures(c *check.C) {
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
"github.com/coreos/go-oidc"
lru "github.com/hashicorp/golang-lru"
"github.com/jmoiron/sqlx"
+ "github.com/lib/pq"
"github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"google.golang.org/api/option"
tokenCacheNegativeTTL = time.Minute * 5
tokenCacheTTL = time.Minute * 10
tokenCacheRaceWindow = time.Minute
+ pqCodeUniqueViolation = pq.ErrorCode("23505")
)
type oidcLoginController struct {
// it's expiring.
exp := time.Now().UTC().Add(tokenCacheTTL + tokenCacheRaceWindow)
- var aca arvados.APIClientAuthorization
if updating {
_, err = tx.ExecContext(ctx, `update api_client_authorizations set expires_at=$1 where api_token=$2`, exp, hmac)
if err != nil {
}
ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: updated api_client_authorizations row")
} else {
- aca, err = ta.ctrl.Parent.CreateAPIClientAuthorization(ctx, ta.ctrl.Cluster.SystemRootToken, *authinfo)
+ aca, err := ta.ctrl.Parent.CreateAPIClientAuthorization(ctx, ta.ctrl.Cluster.SystemRootToken, *authinfo)
if err != nil {
return err
}
- _, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID)
+ _, err = tx.ExecContext(ctx, `savepoint upd`)
if err != nil {
+ return err
+ }
+ _, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID)
+ if e, ok := err.(*pq.Error); ok && e.Code == pqCodeUniqueViolation {
+ // unique_violation, given that the above
+ // query did not find a row with matching
+ // api_token, means another thread/process
+ // also received this same token and won the
+ // race to insert it -- in which case this
+ // thread doesn't need to update the database.
+ // Discard the redundant row.
+ _, err = tx.ExecContext(ctx, `rollback to savepoint upd`)
+ if err != nil {
+ return err
+ }
+ _, err = tx.ExecContext(ctx, `delete from api_client_authorizations where uuid=$1`, aca.UUID)
+ if err != nil {
+ return err
+ }
+ ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: api_client_authorizations row inserted by another thread")
+ } else if err != nil {
+ ctxlog.FromContext(ctx).Errorf("%#v", err)
return fmt.Errorf("error adding OIDC access token to database: %w", err)
+ } else {
+ ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row")
}
- aca.APIToken = hmac
- ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row")
}
err = tx.Commit()
if err != nil {
return err
}
- aca.ExpiresAt = exp
- ta.cache.Add(tok, aca)
+ ta.cache.Add(tok, arvados.APIClientAuthorization{ExpiresAt: exp})
return nil
}
"net/url"
"sort"
"strings"
+ "sync"
"testing"
"time"
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
var exp1 time.Time
- oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
- creds, ok := auth.FromContext(ctx)
- c.Assert(ok, check.Equals, true)
- c.Assert(creds.Tokens, check.HasLen, 1)
- c.Check(creds.Tokens[0], check.Equals, accessToken)
- err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp1)
- c.Check(err, check.IsNil)
- c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
- c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
- return nil, nil
- })(ctx, nil)
+ concurrent := 4
+ s.fakeProvider.HoldUserInfo = make(chan *http.Request)
+ s.fakeProvider.ReleaseUserInfo = make(chan struct{})
+ go func() {
+ for i := 0; ; i++ {
+ if i == concurrent {
+ close(s.fakeProvider.ReleaseUserInfo)
+ }
+ <-s.fakeProvider.HoldUserInfo
+ }
+ }()
+ var wg sync.WaitGroup
+ for i := 0; i < concurrent; i++ {
+ i := i
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ _, err := oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ c.Logf("concurrent req %d/%d", i, concurrent)
+ var exp time.Time
+
+ creds, ok := auth.FromContext(ctx)
+ c.Assert(ok, check.Equals, true)
+ c.Assert(creds.Tokens, check.HasLen, 1)
+ c.Check(creds.Tokens[0], check.Equals, accessToken)
+
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
+ c.Check(err, check.IsNil)
+ c.Check(exp.Sub(time.Now()) > -time.Second, check.Equals, true)
+ c.Check(exp.Sub(time.Now()) < time.Second, check.Equals, true)
+ if i == 0 {
+ exp1 = exp
+ }
+ return nil, nil
+ })(ctx, nil)
+ c.Check(err, check.IsNil)
+ }()
+ }
+ wg.Wait()
+ if c.Failed() {
+ c.Fatal("giving up")
+ }
// If the token is used again after the in-memory cache
// expires, oidcAuthorizer must re-check the token and update
var exp time.Time
err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
c.Check(err, check.IsNil)
- c.Check(exp.Sub(exp1) > 0, check.Equals, true)
- c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
+ c.Check(exp.Sub(exp1) > 0, check.Equals, true, check.Commentf("expect %v > 0", exp.Sub(exp1)))
+ c.Check(exp.Sub(exp1) < time.Second, check.Equals, true, check.Commentf("expect %v < 1s", exp.Sub(exp1)))
return nil, nil
})(ctx, nil)
//
// SPDX-License-Identifier: AGPL-3.0
+//go:build !static
+
package localdb
import (
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+//go:build static
+
+package localdb
+
+import (
+ "context"
+ "errors"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+type pamLoginController struct {
+ Cluster *arvados.Cluster
+ Parent *Conn
+}
+
+func (ctrl *pamLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+ return logout(ctx, ctrl.Cluster, opts)
+}
+
+func (ctrl *pamLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+ return arvados.LoginResponse{}, errors.New("interactive login is not available")
+}
+
+func (ctrl *pamLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+ return arvados.APIClientAuthorization{}, errors.New("support not available due to static compilation")
+}
"os"
"os/exec"
"os/signal"
+ "os/user"
"path"
"path/filepath"
"regexp"
// Run the full container lifecycle.
func (runner *ContainerRunner) Run() (err error) {
runner.CrunchLog.Printf("crunch-run %s started", cmd.Version.String())
+ runner.CrunchLog.Printf("%s", currentUserAndGroups())
runner.CrunchLog.Printf("Executing container '%s' using %s runtime", runner.Container.UUID, runner.executor.Runtime())
hostname, hosterr := os.Hostname()
os.Setenv("ARVADOS_KEEP_SERVICES", url)
return cmd, nil
}
+
+// return current uid, gid, groups in a format suitable for logging:
+// "crunch-run process has uid=1234(arvados) gid=1234(arvados)
+// groups=1234(arvados),114(fuse)"
+func currentUserAndGroups() string {
+ u, err := user.Current()
+ if err != nil {
+ return fmt.Sprintf("error getting current user ID: %s", err)
+ }
+ s := fmt.Sprintf("crunch-run process has uid=%s(%s) gid=%s", u.Uid, u.Username, u.Gid)
+ if g, err := user.LookupGroupId(u.Gid); err == nil {
+ s += fmt.Sprintf("(%s)", g.Name)
+ }
+ s += " groups="
+ if gids, err := u.GroupIds(); err == nil {
+ for i, gid := range gids {
+ if i > 0 {
+ s += ","
+ }
+ s += gid
+ if g, err := user.LookupGroupId(gid); err == nil {
+ s += fmt.Sprintf("(%s)", g.Name)
+ }
+ }
+ }
+ return s
+}
c.Assert(s.api.Logs["crunch-run"], NotNil)
c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
}
"time"
)
-// This magically allows us to look up userHz via _SC_CLK_TCK:
-
-/*
-#include <unistd.h>
-#include <sys/types.h>
-#include <pwd.h>
-#include <stdlib.h>
-*/
-import "C"
-
// A Reporter gathers statistics for a cgroup and writes them to a
// log.Logger.
type Reporter struct {
var userTicks, sysTicks int64
fmt.Sscanf(string(b), "user %d\nsystem %d", &userTicks, &sysTicks)
- userHz := float64(C.sysconf(C._SC_CLK_TCK))
+ userHz := float64(100)
nextSample := cpuSample{
hasData: true,
sampleTime: time.Now(),
def test_small_collection
uuid = Digest::MD5.hexdigest(foo_manifest) + '+' + foo_manifest.size.to_s
+ ok = nil
out, err = capture_subprocess_io do
- assert_arv('--format', 'uuid', 'collection', 'create', '--collection', {
- uuid: uuid,
- manifest_text: foo_manifest
- }.to_json)
+ ok = arv('--format', 'uuid', 'collection', 'create', '--collection', {
+ uuid: uuid,
+ manifest_text: foo_manifest
+ }.to_json)
end
- assert(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/.match(out))
- assert_equal '', err
+ assert_equal('', err)
+ assert_equal(true, ok)
+ assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, out)
+ end
+
+ def test_collection_replace_files
+ ok = nil
+ uuid, err = capture_subprocess_io do
+ ok = arv('--format', 'uuid', 'collection', 'create', '--collection', '{}')
+ end
+ assert_equal('', err)
+ assert_equal(true, ok)
+ assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, uuid)
+ uuid = uuid.strip
+
+ out, err = capture_subprocess_io do
+ ok = arv('--format', 'uuid',
+ 'collection', 'update',
+ '--uuid', uuid,
+ '--collection', '{}',
+ '--replace-files', {
+ "/gpl.pdf": "b519d9cb706a29fc7ea24dbea2f05851+93/GNU_General_Public_License,_version_3.pdf",
+ }.to_json)
+ end
+ assert_equal('', err)
+ assert_equal(true, ok)
+ assert_equal(uuid, out.strip)
+
+ ok = nil
+ out, err = capture_subprocess_io do
+ ok = arv('--format', 'json', 'collection', 'get', '--uuid', uuid)
+ end
+ assert_equal('', err)
+ assert_equal(true, ok)
+ assert_match(/\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025.* 0:249025:gpl.pdf\\n/, out)
end
def test_read_resource_object_from_file
begin
tempfile.write({manifest_text: foo_manifest}.to_json)
tempfile.close
+ ok = nil
out, err = capture_subprocess_io do
- assert_arv('--format', 'uuid',
- 'collection', 'create', '--collection', tempfile.path)
+ ok = arv('--format', 'uuid',
+ 'collection', 'create', '--collection', tempfile.path)
end
- assert(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/.match(out))
- assert_equal '', err
+ assert_equal('', err)
+ assert_equal(true, ok)
+ assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, out)
ensure
tempfile.unlink
end
end
protected
- def assert_arv(*args)
- expect = case args.first
- when true, false
- args.shift
- else
- true
- end
- assert_equal(expect,
- system(['./bin/arv', 'arv'], *args),
- "`arv #{args.join ' '}` " +
- "should exit #{if expect then 0 else 'non-zero' end}")
+ def arv(*args)
+ system(['./bin/arv', 'arv'], *args)
end
def foo_manifest
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
+ exgroup = parser.add_mutually_exclusive_group()
+ exgroup.add_argument("--enable-preemptible", dest="enable_preemptible", default=None, action="store_true", help="Use preemptible instances. Control individual steps with arv:UsePreemptible hint.")
+ exgroup.add_argument("--disable-preemptible", dest="enable_preemptible", default=None, action="store_false", help="Don't use preemptible instances.")
+
parser.add_argument(
"--skip-schemas",
action="store_true",
"http://arvados.org/cwl#ClusterTarget",
"http://arvados.org/cwl#OutputStorageClass",
"http://arvados.org/cwl#ProcessProperties",
- "http://commonwl.org/cwltool#CUDARequirement"
+ "http://commonwl.org/cwltool#CUDARequirement",
+ "http://arvados.org/cwl#UsePreemptible",
])
def exit_signal_handler(sigcode, frame):
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:UsePreemptible"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ usePreemptible: boolean
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:UsePreemptible"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ usePreemptible: boolean
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:UsePreemptible"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ usePreemptible: boolean
cuda_req, _ = self.get_requirement("http://commonwl.org/cwltool#CUDARequirement")
if cuda_req:
runtime_constraints["cuda"] = {
- "device_count": cuda_req.get("deviceCountMin", 1),
+ "device_count": resources.get("cudaDeviceCount", 1),
"driver_version": cuda_req["cudaVersionMin"],
- "hardware_capability": cuda_req["cudaComputeCapabilityMin"]
+ "hardware_capability": aslist(cuda_req["cudaComputeCapability"])[0]
}
+ if runtimeContext.enable_preemptible is False:
+ scheduling_parameters["preemptible"] = False
+ else:
+ preemptible_req, _ = self.get_requirement("http://arvados.org/cwl#UsePreemptible")
+ if preemptible_req:
+ scheduling_parameters["preemptible"] = preemptible_req["usePreemptible"]
+ elif runtimeContext.enable_preemptible is True:
+ scheduling_parameters["preemptible"] = True
+ elif runtimeContext.enable_preemptible is None:
+ pass
+
if self.timelimit is not None and self.timelimit > 0:
scheduling_parameters["max_run_time"] = self.timelimit
if self.enable_dev:
command.append("--enable-dev")
+ if runtimeContext.enable_preemptible is True:
+ command.append("--enable-preemptible")
+
+ if runtimeContext.enable_preemptible is False:
+ command.append("--disable-preemptible")
+
command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
container_req["command"] = command
self.always_submit_runner = False
self.collection_cache_size = 256
self.match_local_docker = False
+ self.enable_preemptible = None
super(ArvRuntimeContext, self).__init__(kwargs)
import arvados.collection
from .util import collectionUUID
-import ruamel.yaml as yaml
+from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap, CommentedSeq
import arvados_cwl.arvdocker
textIO = StringIO(text.decode('utf-8'))
else:
textIO = StringIO(text)
- return yaml.safe_load(textIO)
+ yamlloader = YAML(typ='safe', pure=True)
+ return yamlloader.load(textIO)
else:
return {}
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20220217222804',
+ 'cwltool==3.1.20220224085855',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
--- /dev/null
+{
+ "referenceGenomeSequence": {
+ "class": "File",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "referenceGenomeSequenceDrosophila": {
+ "class": "File",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "blacklistBed": {
+ "class": "File",
+ "location": "data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieHumanReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieDrosophilaReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "sampleName": "LED054_0p03nMR1.0",
+ "inputFastq1": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastq2": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastqUMI": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz",
+ "secondaryFiles": []
+ }
+}
+
--- /dev/null
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "doc": "Pipeline that is applied on single ChIP-seq samples.\n\nStarts with QC on the reads and trimming (for adapters and based on quality)\n\nAligns to human genome and adds UMI\n\nAligns to Drosophila genome and counts the number of reads.\n\nAfter the alignment to human genome the files are filtered for duplicates, multimappers and alignments in black listed regions",
+ "label": "ChIP-Seq (single sample)",
+ "inputs": [
+ {
+ "id": "#inputFastq1",
+ "type": "File",
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1726.25
+ },
+ {
+ "id": "#blacklistBed",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 746.4744873046875,
+ "https://www.sevenbridges.com/y": 1903.265625
+ },
+ {
+ "id": "#referenceGenomeSequence",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai",
+ "^.dict"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fasta, fa",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1405.203125
+ },
+ {
+ "id": "#sampleName",
+ "type": "string",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1191.171875
+ },
+ {
+ "id": "#inputFastq2",
+ "type": [
+ "null",
+ "File"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1619.234375
+ },
+ {
+ "id": "#inputFastqUMI",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1512.21875
+ },
+ {
+ "id": "#BowtieHumanReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1519.21875
+ },
+ {
+ "id": "#BowtieDrosophilaReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1626.234375
+ },
+ {
+ "id": "#referenceGenomeSequenceDrosophila",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai"
+ ],
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1298.1875
+ }
+ ],
+ "outputs": [
+ ],
+ "steps": [
+ {
+ "id": "#step1",
+ "in": {
+ "inp": "#inputFastq1"
+ },
+ "out": [],
+ "run": "../cat.cwl"
+ }
+ ],
+ "requirements": [
+ ]
+ },
+ ],
+ "cwlVersion": "v1.0"
+}
import unittest
import os
import functools
+import threading
import cwltool.process
import cwltool.secrets
import cwltool.load_tool
"basedir": "",
"make_fs_access": make_fs_access,
"construct_tool_object": runner.arv_make_tool,
- "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
+ "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
+ "loader": Loader({}),
+ "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"name": "test_run_"+str(enable_reuse),
"make_fs_access": make_fs_access,
"tmpdir": "/tmp",
+ "outdir": "/tmp",
"enable_reuse": enable_reuse,
"priority": 500,
- "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "workflow_eval_lock": threading.Condition(threading.RLock())
})
if isinstance(runner, mock.MagicMock):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = cmap({
- "inputs": [],
- "outputs": [],
- "baseCommand": "nvidia-smi",
- "arguments": [],
- "id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool",
- "requirements": [
- {
+ test_cwl_req = [{
"class": "http://commonwl.org/cwltool#CUDARequirement",
"cudaVersionMin": "11.0",
- "cudaComputeCapabilityMin": "9.0",
- }
- ]
- })
+ "cudaComputeCapability": "9.0",
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": "9.0",
+ "cudaDeviceCountMin": 2
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": ["4.0", "5.0"],
+ "cudaDeviceCountMin": 2
+ }]
+
+ test_arv_req = [{
+ 'device_count': 1,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "4.0"
+ }]
+
+ for test_case in range(0, len(test_cwl_req)):
- loadingContext, runtimeContext = self.helper(runner, True)
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "nvidia-smi",
+ "arguments": [],
+ "id": "",
+ "cwlVersion": "v1.2",
+ "class": "CommandLineTool",
+ "requirements": [test_cwl_req[test_case]]
+ })
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
- arvtool.formatgraph = None
+ loadingContext, runtimeContext = self.helper(runner, True)
- for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
- j.run(runtimeContext)
- runner.api.container_requests().create.assert_called_with(
- body=JsonDiffMatcher({
- 'environment': {
- 'HOME': '/var/spool/cwl',
- 'TMPDIR': '/tmp'
- },
- 'name': 'test_run_True',
- 'runtime_constraints': {
- 'vcpus': 1,
- 'ram': 268435456,
- 'cuda': {
- 'device_count': 1,
- 'driver_version': "11.0",
- 'hardware_capability': "9.0"
- }
- },
- 'use_existing': True,
- 'priority': 500,
- 'mounts': {
- '/tmp': {'kind': 'tmp',
- "capacity": 1073741824
- },
- '/var/spool/cwl': {'kind': 'tmp',
- "capacity": 1073741824 }
- },
- 'state': 'Committed',
- 'output_name': 'Output for step test_run_True',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'output_path': '/var/spool/cwl',
- 'output_ttl': 0,
- 'container_image': '99999999999999999999999999999993+99',
- 'command': ['nvidia-smi'],
- 'cwd': '/var/spool/cwl',
- 'scheduling_parameters': {},
- 'properties': {},
- 'secret_mounts': {},
- 'output_storage_classes': ["default"]
- }))
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456,
+ 'cuda': test_arv_req[test_case]
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['nvidia-smi'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
# The test passes no builder.resources
"baseCommand": "echo",
"arguments": [],
"id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool"
+ "cwlVersion": "v1.0",
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner, True)
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
container_request = {
'name': 'test_run_True',
'runtime_constraints': {
'vcpus': 1,
- 'ram': 268435456
+ 'ram': 1073741824,
},
'use_existing': True,
'priority': 500,
body=JsonDiffMatcher(container_request))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_run_preemptible_hint(self, keepdocker):
+ arvados_cwl.add_arv_hints()
+ for enable_preemptible in (None, True, False):
+ for preemptible_hint in (None, True, False):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ if preemptible_hint is not None:
+ hints = [{
+ "class": "http://arvados.org/cwl#UsePreemptible",
+ "usePreemptible": preemptible_hint
+ }]
+ else:
+ hints = []
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "",
+ "class": "CommandLineTool",
+ "cwlVersion": "v1.2",
+ "hints": hints
+ })
+
+ loadingContext, runtimeContext = self.helper(runner)
+
+ runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
+ runtimeContext.enable_preemptible = enable_preemptible
+
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ # Test the interactions between --enable/disable-preemptible
+ # and UsePreemptible hint
+
+ if enable_preemptible is None:
+ if preemptible_hint is None:
+ sched = {}
+ else:
+ sched = {'preemptible': preemptible_hint}
+ else:
+ if preemptible_hint is None:
+ sched = {'preemptible': enable_preemptible}
+ else:
+ sched = {'preemptible': enable_preemptible and preemptible_hint}
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': runtimeContext.name,
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step '+runtimeContext.name,
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['ls', '/var/spool/cwl'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': sched,
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
+
+
class TestWorkflow(unittest.TestCase):
def setUp(self):
self.assertEqual(exited, 0)
+ @stubs
+ def test_submit_enable_preemptible(self, stubs):
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate', '--disable-color',
+ '--eval-timeout=20', '--thread-count=0',
+ '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
+ '--enable-preemptible',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
+ @stubs
+ def test_submit_disable_preemptible(self, stubs):
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate', '--disable-color',
+ '--eval-timeout=20', '--thread-count=0',
+ '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
+ '--disable-preemptible',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
+
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
EnsureUniqueName bool `json:"ensure_unique_name"`
Select []string `json:"select"`
Attrs map[string]interface{} `json:"attrs"`
+ // ReplaceFiles only applies when creating a collection.
+ ReplaceFiles map[string]string `json:"replace_files"`
}
type UpdateOptions struct {
Attrs map[string]interface{} `json:"attrs"`
Select []string `json:"select"`
BypassFederation bool `json:"bypass_federation"`
+ // ReplaceFiles only applies when updating a collection.
+ ReplaceFiles map[string]string `json:"replace_files"`
}
type GroupContentsOptions struct {
import (
"context"
+ "errors"
"io"
)
type apiClient interface {
RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
}
+
+var errStubClient = errors.New("stub client")
+
+type StubClient struct{}
+
+func (*StubClient) ReadAt(string, []byte, int) (int, error) { return 0, errStubClient }
+func (*StubClient) LocalLocator(loc string) (string, error) { return loc, nil }
+func (*StubClient) BlockWrite(context.Context, BlockWriteOptions) (BlockWriteResponse, error) {
+ return BlockWriteResponse{}, errStubClient
+}
+func (*StubClient) RequestAndDecode(_ interface{}, _, _ string, _ io.Reader, _ interface{}) error {
+ return errStubClient
+}
"errors"
"fmt"
"io"
+ "io/fs"
"log"
"net/http"
"os"
MemorySize() int64
}
+type fsFS struct {
+ FileSystem
+}
+
+// FS returns an fs.FS interface to the given FileSystem, to enable
+// the use of fs.WalkDir, etc.
+func FS(fs FileSystem) fs.FS { return fsFS{fs} }
+func (fs fsFS) Open(path string) (fs.File, error) {
+ f, err := fs.FileSystem.Open(path)
+ return f, err
+}
+
type inode interface {
SetParent(parent inode, name string)
Parent() inode
default:
return nil, fmt.Errorf("invalid flags 0x%x", flag)
}
- if !writable && parent.IsDir() {
+ if parent.IsDir() {
// A directory can be opened via "foo/", "foo/.", or
// "foo/..".
switch name {
case ".", "":
- return &filehandle{inode: parent}, nil
+ return &filehandle{inode: parent, readable: readable, writable: writable}, nil
case "..":
- return &filehandle{inode: parent.Parent()}, nil
+ return &filehandle{inode: parent.Parent(), readable: readable, writable: writable}, nil
}
}
createMode := flag&os.O_CREATE != 0
f, err = fs.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0700)
}
if err != nil {
- return err
+ return fmt.Errorf("open %s: %w", target, err)
}
defer f.Close()
return f.Splice(newsubtree)
func (dn *dirnode) Splice(repl inode) error {
repl, err := repl.Snapshot()
if err != nil {
- return err
+ return fmt.Errorf("cannot copy snapshot: %w", err)
}
switch repl := repl.(type) {
default:
defer dn.Unlock()
_, err = dn.parent.Child(dn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
if err != nil {
- return err
+ return fmt.Errorf("error replacing filenode: dn.parent.Child(): %w", err)
}
repl.fs = dn.fs
}
}
}
+func (s *CollectionFSSuite) TestSnapshotSplice(c *check.C) {
+ filedata1 := "hello snapshot+splice world\n"
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ {
+ f, err := fs.OpenFile("file1", os.O_CREATE|os.O_RDWR, 0700)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte(filedata1))
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ }
+
+ snap, err := Snapshot(fs, "/")
+ c.Assert(err, check.IsNil)
+ err = Splice(fs, "dir1", snap)
+ c.Assert(err, check.IsNil)
+ f, err := fs.Open("dir1/file1")
+ c.Assert(err, check.IsNil)
+ buf, err := io.ReadAll(f)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Equals, filedata1)
+}
+
func (s *CollectionFSSuite) TestRefreshSignatures(c *check.C) {
filedata1 := "hello refresh signatures world\n"
fs, err := (&Collection{}).FileSystem(s.client, s.kc)
import (
"io"
+ "io/fs"
"os"
)
return
}
+// dirEntry implements fs.DirEntry, see (*filehandle)ReadDir().
+type dirEntry struct {
+ os.FileInfo
+}
+
+func (ent dirEntry) Type() fs.FileMode {
+ return ent.Mode().Type()
+}
+func (ent dirEntry) Info() (fs.FileInfo, error) {
+ return ent, nil
+}
+
+// ReadDir implements fs.ReadDirFile.
+func (f *filehandle) ReadDir(count int) ([]fs.DirEntry, error) {
+ fis, err := f.Readdir(count)
+ if len(fis) == 0 {
+ return nil, err
+ }
+ ents := make([]fs.DirEntry, len(fis))
+ for i, fi := range fis {
+ ents[i] = dirEntry{fi}
+ }
+ return ents, err
+}
+
func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
if !f.inode.IsDir() {
return nil, ErrInvalidOperation
"io/ioutil"
"net/http"
"os"
+ "strings"
"syscall"
"time"
c.Check(string(buf), check.Equals, string(thisfile))
}
- // Cannot splice a file onto a collection root, or anywhere
- // outside a collection
+ // Cannot splice a file onto a collection root; cannot splice
+ // anything to a target outside a collection.
for _, badpath := range []string{
+ dstPath + "/",
dstPath,
+ "/home/A Project/newnodename/",
"/home/A Project/newnodename",
+ "/home/A Project/",
"/home/A Project",
+ "/home/newnodename/",
"/home/newnodename",
+ "/home/",
"/home",
+ "/newnodename/",
"/newnodename",
+ "/",
} {
err = Splice(s.fs, badpath, snapFile)
c.Check(err, check.NotNil)
- c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
- if badpath == dstPath {
- c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %s", badpath))
+ if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
+ c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
+ } else {
+ c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
+ }
+ if strings.TrimSuffix(badpath, "/") == dstPath {
+ c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
continue
}
- err = Splice(s.fs, badpath, snap1)
- c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
- }
- // Destination cannot have trailing slash
- for _, badpath := range []string{
- dstPath + "/ctxlog/",
- dstPath + "/",
- "/home/A Project/",
- "/home/",
- "/",
- "",
- } {
err = Splice(s.fs, badpath, snap1)
- c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
- err = Splice(s.fs, badpath, snapFile)
- c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+ if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
+ c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
+ } else {
+ c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
+ }
}
// Destination's parent must already exist
}
snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
- c.Check(err, check.IsNil)
- err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
- c.Check(err, check.IsNil)
+ if c.Check(err, check.IsNil) {
+ err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
+ c.Check(err, check.IsNil)
+ }
// Snapshot entire collection, splice into same collection at
// a new path, remove file from original location, verify
_, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
c.Check(err, check.Equals, os.ErrNotExist)
f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
- c.Check(err, check.IsNil)
- defer f.Close()
- buf, err := ioutil.ReadAll(f)
- c.Check(err, check.IsNil)
- c.Check(string(buf), check.Equals, string(thisfile))
+ if c.Check(err, check.IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, string(thisfile))
+ }
}
PeopleAPIResponse map[string]interface{}
+ // send incoming /userinfo requests to HoldUserInfo (if not
+ // nil), then receive from ReleaseUserInfo (if not nil),
+ // before responding (these are used to set up races)
+ HoldUserInfo chan *http.Request
+ ReleaseUserInfo chan struct{}
+
key *rsa.PrivateKey
Issuer *httptest.Server
PeopleAPI *httptest.Server
case "/auth":
w.WriteHeader(http.StatusInternalServerError)
case "/userinfo":
+ if p.HoldUserInfo != nil {
+ p.HoldUserInfo <- req
+ }
+ if p.ReleaseUserInfo != nil {
+ <-p.ReleaseUserInfo
+ }
authhdr := req.Header.Get("Authorization")
if _, err := jwt.ParseSigned(strings.TrimPrefix(authhdr, "Bearer ")); err != nil {
p.c.Logf("OIDCProvider: bad auth %q", authhdr)
import (
"encoding/json"
+ "fmt"
"net/http"
)
+func Errorf(status int, tmpl string, args ...interface{}) error {
+ return errorWithStatus{fmt.Errorf(tmpl, args...), status}
+}
+
func ErrorWithStatus(err error, status int) error {
return errorWithStatus{err, status}
}
svc.insecure = insecure
svc.request_id = request_id
svc.config = lambda: util.get_config_once(svc)
+ svc.vocabulary = lambda: util.get_vocabulary_once(svc)
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
return (None, None)
fn = os.path.join(root, ".arvados#collection")
if os.path.exists(fn):
- with file(fn, 'r') as f:
+ with open(fn, 'r') as f:
c = json.load(f)
return (c["portable_data_hash"], branch)
else:
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if self.insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ curl.setopt(pycurl.SSL_VERIFYHOST, 0)
else:
curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
if method == "HEAD":
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if self.insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ curl.setopt(pycurl.SSL_VERIFYHOST, 0)
else:
curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
self._setcurltimeouts(curl, timeout)
if not hasattr(svc, '_cached_config'):
svc._cached_config = svc.configs().get().execute()
return svc._cached_config
+
+def get_vocabulary_once(svc):
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+from . import api
+
+_logger = logging.getLogger('arvados.vocabulary')
+
+def load_vocabulary(api_client=None):
+ """Load the Arvados vocabulary from the API.
+ """
+ if api_client is None:
+ api_client = api('v1')
+ return Vocabulary(api_client.vocabulary())
+
+class VocabularyError(Exception):
+ """Base class for all vocabulary errors.
+ """
+ pass
+
+class VocabularyKeyError(VocabularyError):
+ pass
+
+class VocabularyValueError(VocabularyError):
+ pass
+
+class Vocabulary(object):
+ def __init__(self, voc_definition={}):
+ self.strict_keys = voc_definition.get('strict_tags', False)
+ self.key_aliases = {}
+
+ for key_id, val in (voc_definition.get('tags') or {}).items():
+ strict = val.get('strict', False)
+ key_labels = [l['label'] for l in val.get('labels', [])]
+ values = {}
+ for v_id, v_val in (val.get('values') or {}).items():
+ labels = [l['label'] for l in v_val.get('labels', [])]
+ values[v_id] = VocabularyValue(v_id, labels)
+ vk = VocabularyKey(key_id, key_labels, values, strict)
+ self.key_aliases[key_id.lower()] = vk
+ for alias in vk.aliases:
+ self.key_aliases[alias.lower()] = vk
+
+ def __getitem__(self, key):
+ return self.key_aliases[key.lower()]
+
+ def convert_to_identifiers(self, obj={}):
+ """Translate key/value pairs to machine readable identifiers.
+ """
+ return self._convert_to_what(obj, 'identifier')
+
+ def convert_to_labels(self, obj={}):
+ """Translate key/value pairs to human readable labels.
+ """
+ return self._convert_to_what(obj, 'preferred_label')
+
+ def _convert_to_what(self, obj={}, what=None):
+ if not isinstance(obj, dict):
+ raise ValueError("obj must be a dict")
+ if what not in ['preferred_label', 'identifier']:
+ raise ValueError("what attr must be 'preferred_label' or 'identifier'")
+ r = {}
+ for k, v in obj.items():
+ # Key validation & lookup
+ key_found = False
+ if not isinstance(k, str):
+ raise VocabularyKeyError("key '{}' must be a string".format(k))
+ k_what, v_what = k, v
+ try:
+ k_what = getattr(self[k], what)
+ key_found = True
+ except KeyError:
+ if self.strict_keys:
+ raise VocabularyKeyError("key '{}' not found in vocabulary".format(k))
+
+ # Value validation & lookup
+ if isinstance(v, list):
+ v_what = []
+ for x in v:
+ if not isinstance(x, str):
+ raise VocabularyValueError("value '{}' for key '{}' must be a string".format(x, k))
+ try:
+ v_what.append(getattr(self[k][x], what))
+ except KeyError:
+ if self[k].strict:
+ raise VocabularyValueError("value '{}' not found for key '{}'".format(x, k))
+ v_what.append(x)
+ else:
+ if not isinstance(v, str):
+ raise VocabularyValueError("{} value '{}' for key '{}' must be a string".format(type(v).__name__, v, k))
+ try:
+ v_what = getattr(self[k][v], what)
+ except KeyError:
+ if key_found and self[k].strict:
+ raise VocabularyValueError("value '{}' not found for key '{}'".format(v, k))
+
+ r[k_what] = v_what
+ return r
+
+class VocabularyData(object):
+ def __init__(self, identifier, aliases=[]):
+ self.identifier = identifier
+ self.aliases = aliases
+
+ def __getattribute__(self, name):
+ if name == 'preferred_label':
+ return self.aliases[0]
+ return super(VocabularyData, self).__getattribute__(name)
+
+class VocabularyValue(VocabularyData):
+ def __init__(self, identifier, aliases=[]):
+ super(VocabularyValue, self).__init__(identifier, aliases)
+
+class VocabularyKey(VocabularyData):
+ def __init__(self, identifier, aliases=[], values={}, strict=False):
+ super(VocabularyKey, self).__init__(identifier, aliases)
+ self.strict = strict
+ self.value_aliases = {}
+ for v_id, v_val in values.items():
+ self.value_aliases[v_id.lower()] = v_val
+ for v_alias in v_val.aliases:
+ self.value_aliases[v_alias.lower()] = v_val
+
+ def __getitem__(self, key):
+ return self.value_aliases[key.lower()]
\ No newline at end of file
'google-api-python-client >=1.6.2, <2',
'google-auth<2',
'httplib2 >=0.9.2, <0.20.2',
- 'pycurl >=7.19.5.1',
+ 'pycurl >=7.19.5.1, <7.45.0',
'ruamel.yaml >=0.15.54, <0.17.11',
'setuptools',
'ws4py >=0.4.2',
self.assertEqual(
mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
0)
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.SSL_VERIFYHOST),
+ 0)
api_client.insecure = False
with tutil.mock_keep_responses(b'foo', 200) as mock:
self.assertEqual(
mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
None)
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.SSL_VERIFYHOST),
+ None)
def test_refresh_signature(self):
blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11'
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import unittest
+import mock
+
+from arvados import api, vocabulary
+
+class VocabularyTest(unittest.TestCase):
+ EXAMPLE_VOC = {
+ 'tags': {
+ 'IDTAGANIMALS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Animal'},
+ {'label': 'Creature'},
+ ],
+ 'values': {
+ 'IDVALANIMAL1': {
+ 'labels': [
+ {'label': 'Human'},
+ {'label': 'Homo sapiens'},
+ ],
+ },
+ 'IDVALANIMAL2': {
+ 'labels': [
+ {'label': 'Elephant'},
+ {'label': 'Loxodonta'},
+ ],
+ },
+ },
+ },
+ 'IDTAGIMPORTANCES': {
+ 'strict': True,
+ 'labels': [
+ {'label': 'Importance'},
+ {'label': 'Priority'},
+ ],
+ 'values': {
+ 'IDVALIMPORTANCE1': {
+ 'labels': [
+ {'label': 'High'},
+ {'label': 'High priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE2': {
+ 'labels': [
+ {'label': 'Medium'},
+ {'label': 'Medium priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE3': {
+ 'labels': [
+ {'label': 'Low'},
+ {'label': 'Low priority'},
+ ],
+ },
+ },
+ },
+ 'IDTAGCOMMENTS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Comment'},
+ {'label': 'Notes'},
+ ],
+ 'values': None,
+ },
+ },
+ }
+
+ def setUp(self):
+ self.api = arvados.api('v1')
+ self.voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ self.api.vocabulary = mock.MagicMock(return_value=self.EXAMPLE_VOC)
+
+ def test_vocabulary_keys(self):
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(
+ self.voc.key_aliases.keys(),
+ set(['idtaganimals', 'creature', 'animal',
+ 'idtagimportances', 'importance', 'priority',
+ 'idtagcomments', 'comment', 'notes'])
+ )
+
+ vk = self.voc.key_aliases['creature']
+ self.assertEqual(vk.strict, False)
+ self.assertEqual(vk.identifier, 'IDTAGANIMALS')
+ self.assertEqual(vk.aliases, ['Animal', 'Creature'])
+ self.assertEqual(vk.preferred_label, 'Animal')
+ self.assertEqual(
+ vk.value_aliases.keys(),
+ set(['idvalanimal1', 'human', 'homo sapiens',
+ 'idvalanimal2', 'elephant', 'loxodonta'])
+ )
+
+ def test_vocabulary_values(self):
+ vk = self.voc.key_aliases['creature']
+ vv = vk.value_aliases['human']
+ self.assertEqual(vv.identifier, 'IDVALANIMAL1')
+ self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
+ self.assertEqual(vv.preferred_label, 'Human')
+
+ def test_vocabulary_indexing(self):
+ self.assertEqual(self.voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+ with self.assertRaises(KeyError):
+ inexistant = self.voc['foo']
+
+ def test_empty_vocabulary(self):
+ voc = vocabulary.Vocabulary({})
+ self.assertEqual(voc.strict_keys, False)
+ self.assertEqual(voc.key_aliases, {})
+
+ def test_load_vocabulary_with_api(self):
+ voc = vocabulary.load_vocabulary(self.api)
+ self.assertEqual(voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+
+ def test_convert_to_identifiers(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_value_lists(self):
+ cases = [
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'IDTAGIMPORTANCES': ['High', 'Medium']},
+ {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'priority': ['high', 'medium']},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ strict_voc.convert_to_identifiers({'foo': 'bar'})
+
+ def test_convert_to_identifiers_invalid_key(self):
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({42: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({None: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({('f', 'o', 'o'): 'bar'})
+
+ def test_convert_to_identifiers_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'Animal': 'foo'}), {'IDTAGANIMALS': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Priority': 'foo'})
+
+ def test_convert_to_identifiers_invalid_value(self):
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': 42})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': None})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': {'hello': 'world'}})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [42]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [None]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [{'hello': 'world'}]})
+
+ def test_convert_to_identifiers_unknown_value_list(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(
+ self.voc.convert_to_identifiers({'Animal': ['foo', 'loxodonta']}),
+ {'IDTAGANIMALS': ['foo', 'IDVALANIMAL2']}
+ )
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Priority': ['foo', 'bar']})
+
+ def test_convert_to_labels(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High', 'Animal': 'Human', 'Comment': 'Very important person'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_value_lists(self):
+ cases = [
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'IDTAGIMPORTANCES': ['High', 'Medium']},
+ {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'priority': ['high', 'medium']},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': ['High', 'Medium']},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_labels({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ strict_voc.convert_to_labels({'foo': 'bar'})
+
+ def test_convert_to_labels_invalid_key(self):
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({42: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({None: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({('f', 'o', 'o'): 'bar'})
+
+ def test_convert_to_labels_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_labels({'IDTAGANIMALS': 'foo'}), {'Animal': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': 'foo'})
+
+ def test_convert_to_labels_invalid_value(self):
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': {'high': True}})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': None})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': 42})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': False})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [42]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [None]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [{'high': True}]})
+
+ def test_convert_to_labels_unknown_value_list(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(
+ self.voc.convert_to_labels({'IDTAGANIMALS': ['foo', 'IDVALANIMAL1']}),
+ {'Animal': ['foo', 'Human']}
+ )
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': ['foo', 'bar']})
+
+ def test_convert_roundtrip(self):
+ initial = {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'}
+ converted = self.voc.convert_to_labels(initial)
+ self.assertNotEqual(converted, initial)
+ self.assertEqual(self.voc.convert_to_identifiers(converted), initial)
gem 'jquery-rails'
-gem 'rvm-capistrano', :group => :test
-
gem 'acts_as_api'
gem 'passenger'
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.2.6)
- actionpack (= 5.2.6)
+ actioncable (5.2.6.3)
+ actionpack (= 5.2.6.3)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
+ actionmailer (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.6)
- actionview (= 5.2.6)
- activesupport (= 5.2.6)
+ actionpack (5.2.6.3)
+ actionview (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.6)
- activesupport (= 5.2.6)
+ actionview (5.2.6.3)
+ activesupport (= 5.2.6.3)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.6)
- activesupport (= 5.2.6)
+ activejob (5.2.6.3)
+ activesupport (= 5.2.6.3)
globalid (>= 0.3.6)
- activemodel (5.2.6)
- activesupport (= 5.2.6)
- activerecord (5.2.6)
- activemodel (= 5.2.6)
- activesupport (= 5.2.6)
+ activemodel (5.2.6.3)
+ activesupport (= 5.2.6.3)
+ activerecord (5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
arel (>= 9.0)
- activestorage (5.2.6)
- actionpack (= 5.2.6)
- activerecord (= 5.2.6)
+ activestorage (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
marcel (~> 1.0.0)
- activesupport (5.2.6)
+ activesupport (5.2.6.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
multi_json (>= 1.0.0)
builder (3.2.4)
byebug (11.0.1)
- capistrano (2.15.9)
- highline
- net-scp (>= 1.0.0)
- net-sftp (>= 2.0.0)
- net-ssh (>= 2.0.14)
- net-ssh-gateway (>= 1.1.0)
concurrent-ruby (1.1.9)
crass (1.0.6)
erubi (1.10.0)
faraday (0.15.4)
multipart-post (>= 1.2, < 3)
ffi (1.9.25)
- globalid (0.4.2)
- activesupport (>= 4.2.0)
+ globalid (1.0.0)
+ activesupport (>= 5.0)
googleauth (0.9.0)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
signet (~> 0.7)
- highline (2.0.1)
httpclient (2.8.3)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.10.0)
+ loofah (2.14.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (1.0.1)
+ marcel (1.0.2)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
- mini_mime (1.1.0)
- mini_portile2 (2.6.1)
+ mini_mime (1.1.2)
+ mini_portile2 (2.8.0)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
multi_json (1.15.0)
multipart-post (2.1.1)
- net-scp (2.0.0)
- net-ssh (>= 2.6.5, < 6.0.0)
- net-sftp (2.1.2)
- net-ssh (>= 2.6.5)
- net-ssh (5.2.0)
- net-ssh-gateway (2.0.0)
- net-ssh (>= 4.0.0)
- nio4r (2.5.7)
- nokogiri (1.12.5)
- mini_portile2 (~> 2.6.1)
+ nio4r (2.5.8)
+ nokogiri (1.13.3)
+ mini_portile2 (~> 2.8.0)
racc (~> 1.4)
oj (3.9.2)
optimist (3.0.0)
rack (2.2.3)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.6)
- actioncable (= 5.2.6)
- actionmailer (= 5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
- activemodel (= 5.2.6)
- activerecord (= 5.2.6)
- activestorage (= 5.2.6)
- activesupport (= 5.2.6)
+ rails (5.2.6.3)
+ actioncable (= 5.2.6.3)
+ actionmailer (= 5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
+ activestorage (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
bundler (>= 1.3.0)
- railties (= 5.2.6)
+ railties (= 5.2.6.3)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.3.0)
+ rails-html-sanitizer (1.4.2)
loofah (~> 2.3)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.2.6)
- actionpack (= 5.2.6)
- activesupport (= 5.2.6)
+ railties (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
- rake (13.0.3)
+ rake (13.0.6)
rb-fsevent (0.10.3)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
railties (>= 4.2.0, < 6.0)
retriable (1.4.1)
ruby-prof (0.15.9)
- rvm-capistrano (1.5.6)
- capistrano (~> 2.15.4)
safe_yaml (1.0.5)
signet (0.11.0)
addressable (~> 2.3)
sshkey (2.0.0)
test-unit (3.3.1)
power_assert
- thor (1.1.0)
+ thor (1.2.1)
thread_safe (0.3.6)
tzinfo (1.2.9)
thread_safe (~> 0.1)
- websocket-driver (0.7.4)
+ websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
rails-perftest
responders (~> 2.0)
ruby-prof (~> 0.15.0)
- rvm-capistrano
safe_yaml
signet (< 0.12)
simplecov (~> 0.7.1)
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20210628",
+ revision: "20220222",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
end
end
+ # The 'replace_files' option is implemented in lib/controller,
+ # not Rails -- we just need to add it here so discovery-aware
+ # clients know how to validate it.
+ [:create, :update].each do |action|
+ discovery[:resources]['collections'][:methods][action][:parameters]['replace_files'] = {
+ type: 'object',
+ description: 'Files and directories to initialize/replace with content from other collections.',
+ required: false,
+ location: 'query',
+ properties: {},
+ additionalProperties: {type: 'string'},
+ }
+ end
+
discovery[:resources]['configs'] = {
methods: {
get: {
}
}
+ discovery[:resources]['vocabularies'] = {
+ methods: {
+ get: {
+ id: "arvados.vocabularies.get",
+ path: "vocabulary",
+ httpMethod: "GET",
+ description: "Get vocabulary definition",
+ parameters: {
+ },
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
+ }
+ }
+
discovery[:resources]['sys'] = {
methods: {
get: {
UNLOGGED_CHANGES = ['last_used_at', 'last_used_by_ip_address', 'updated_at']
def assign_random_api_token
- self.api_token ||= rand(2**256).to_s(36)
+ begin
+ self.api_token ||= rand(2**256).to_s(36)
+ rescue ActiveModel::MissingAttributeError
+ # Ignore the case where self.api_token doesn't exist, which happens when
+ # the select=[...] is used.
+ end
end
def owner_uuid
uniqueness: true,
allow_nil: true)
validate :must_unsetup_to_deactivate
+ validate :identity_url_nil_if_empty
before_update :prevent_privilege_escalation
before_update :prevent_inactive_admin
before_update :verify_repositories_empty, :if => Proc.new {
repo.save!
end
end
+
+ def identity_url_nil_if_empty
+ if identity_url == ""
+ self.identity_url = nil
+ end
+ end
end
desc "Remove old container log entries from the logs table"
task delete_old_container_logs: :environment do
- delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND clock_timestamp() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
+ delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND now() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
ActiveRecord::Base.connection.execute(delete_sql)
end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This task finds jobs that have been finished for at least as long as
-# the duration specified in the `clean_job_log_rows_after`
-# configuration setting, and deletes their stderr logs from the logs table.
-
-namespace :db do
- desc "Remove old job stderr entries from the logs table"
- task delete_old_job_logs: :environment do
- delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
-
- ActiveRecord::Base.connection.execute(delete_sql)
- end
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-# This is needed instead of just including CurrentApiClient so that its
-# methods don't get imported as Object's class methods; this is a problem because
-# the methods would be imported only on test environment. See #15716 for more info.
-class CurrentApiClientHelper
- extend CurrentApiClient
-end
-
-def has_symbols? x
- if x.is_a? Hash
- x.each do |k,v|
- return true if has_symbols?(k) or has_symbols?(v)
- end
- elsif x.is_a? Array
- x.each do |k|
- return true if has_symbols?(k)
- end
- elsif x.is_a? Symbol
- return true
- elsif x.is_a? String
- return true if x.start_with?(':') && !x.start_with?('::')
- end
- false
-end
-
-def check_for_serialized_symbols rec
- jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
- (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
- if has_symbols? rec.attributes[colname]
- st = recursive_stringify rec.attributes[colname]
- puts "Found value potentially containing Ruby symbols in #{colname} attribute of #{rec.uuid}, current value is\n#{rec.attributes[colname].to_s[0..1024]}\nrake symbols:stringify will update it to:\n#{st.to_s[0..1024]}\n\n"
- end
- end
-end
-
-def recursive_stringify x
- if x.is_a? Hash
- Hash[x.collect do |k,v|
- [recursive_stringify(k), recursive_stringify(v)]
- end]
- elsif x.is_a? Array
- x.collect do |k|
- recursive_stringify k
- end
- elsif x.is_a? Symbol
- x.to_s
- elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
- x[1..-1]
- else
- x
- end
-end
-
-def stringify_serialized_symbols rec
- # ensure_serialized_attribute_type should prevent symbols from
- # getting into the database in the first place. If someone managed
- # to get them into the database (perhaps using an older version)
- # we'll convert symbols to strings when loading from the
- # database. (Otherwise, loading and saving an object with existing
- # symbols in a serialized field will crash.)
- jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
- (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
- if has_symbols? rec.attributes[colname]
- begin
- st = recursive_stringify rec.attributes[colname]
- puts "Updating #{colname} attribute of #{rec.uuid} from\n#{rec.attributes[colname].to_s[0..1024]}\nto\n#{st.to_s[0..1024]}\n\n"
- rec.write_attribute(colname, st)
- rec.save!
- rescue => e
- puts "Failed to update #{rec.uuid}: #{e}"
- end
- end
- end
-end
-
-namespace :symbols do
- desc 'Warn about serialized values starting with ":" that may be symbols'
- task check: :environment do
- [ApiClientAuthorization, ApiClient,
- AuthorizedKey, Collection,
- Container, ContainerRequest, Group,
- Human, Job, JobTask, KeepDisk, KeepService, Link,
- Node, PipelineInstance, PipelineTemplate,
- Repository, Specimen, Trait, User, VirtualMachine,
- Workflow].each do |klass|
- CurrentApiClientHelper.act_as_system_user do
- klass.all.each do |c|
- check_for_serialized_symbols c
- end
- end
- end
- end
-
- task stringify: :environment do
- [ApiClientAuthorization, ApiClient,
- AuthorizedKey, Collection,
- Container, ContainerRequest, Group,
- Human, Job, JobTask, KeepDisk, KeepService, Link,
- Node, PipelineInstance, PipelineTemplate,
- Repository, Specimen, Trait, User, VirtualMachine,
- Workflow].each do |klass|
- CurrentApiClientHelper.act_as_system_user do
- klass.all.each do |c|
- stringify_serialized_symbols c
- end
- end
- end
- end
-end
get :current
assert_response 401
end
+
+ # Tests regression #18801
+ test "select param is respected in 'show' response" do
+ authorize_with :active
+ get :show, params: {
+ id: api_client_authorizations(:active).uuid,
+ select: ["uuid"],
+ }
+ assert_response :success
+ assert_raises ActiveModel::MissingAttributeError do
+ assigns(:object).api_token
+ end
+ assert_nil json_response["expires_at"]
+ assert_nil json_response["api_token"]
+ assert_equal api_client_authorizations(:active).uuid, json_response["uuid"]
+ end
end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'test_helper'
-require 'rake'
-
-Rake.application.rake_require "tasks/delete_old_job_logs"
-Rake::Task.define_task(:environment)
-
-class DeleteOldJobLogsTaskTest < ActiveSupport::TestCase
- TASK_NAME = "db:delete_old_job_logs"
-
- def log_uuids(*fixture_names)
- fixture_names.map { |name| logs(name).uuid }
- end
-
- def run_with_expiry(clean_after)
- Rails.configuration.Containers.Logging.MaxAge = clean_after
- Rake::Task[TASK_NAME].reenable
- Rake.application.invoke_task TASK_NAME
- end
-
- def job_stderr_logs
- Log.where("object_uuid LIKE :pattern AND event_type = :etype",
- pattern: "_____-8i9sb-_______________",
- etype: "stderr")
- end
-
- def check_existence(test_method, fixture_uuids)
- uuids_now = job_stderr_logs.map(&:uuid)
- fixture_uuids.each do |expect_uuid|
- send(test_method, uuids_now, expect_uuid)
- end
- end
-
- test "delete all logs" do
- uuids_to_keep = log_uuids(:crunchstat_for_running_job)
- uuids_to_clean = log_uuids(:crunchstat_for_previous_job,
- :crunchstat_for_ancient_job)
- run_with_expiry(1)
- check_existence(:assert_includes, uuids_to_keep)
- check_existence(:refute_includes, uuids_to_clean)
- end
-
- test "delete only old logs" do
- uuids_to_keep = log_uuids(:crunchstat_for_running_job,
- :crunchstat_for_previous_job)
- uuids_to_clean = log_uuids(:crunchstat_for_ancient_job)
- run_with_expiry(360.days)
- check_existence(:assert_includes, uuids_to_keep)
- check_existence(:refute_includes, uuids_to_clean)
- end
-end
assert user.save
end
+ test "empty identity_url saves as null" do
+ set_user_from_auth :admin
+ user = users(:active)
+ assert user.update_attributes(identity_url: '')
+ user.reload
+ assert_nil user.identity_url
+ end
+
end
arvswitch <name>
Set ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment based on
$HOME/.config/arvados/<name>.conf
- With no arguments, list available Arvados configurations.
+ With no arguments, print current API host and available Arvados configurations.
arvsave <name>
- Save values of ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment to
+ Save current values of ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment to
$HOME/.config/arvados/<name>.conf
arvrm <name>
arvboxswitch <name>
Set ARVBOX_CONTAINER to <name>
- With no arguments, list available arvboxes.
+ With no arguments, print current arvbox and available arvboxes.
-arvopen:
+arvopen <uuid>
Open an Arvados uuid in web browser (http://arvadosapi.com)
-arvissue
+arvissue <issue number>
Open an Arvados ticket in web browser (http://dev.arvados.org)
EOF
fi
else
echo "Switch Arvados environment conf"
- echo "Usage: arvswitch name"
+ echo "Current host: ${ARVADOS_API_HOST}"
+ echo "Usage: arvswitch <name>"
echo "Available confs:" $((cd $HOME/.config/arvados && ls --indicator-style=none *.conf) | rev | cut -c6- | rev)
fi
}
env | grep ARVADOS_ > $HOME/.config/arvados/$1.conf
else
echo "Save current Arvados environment variables to conf file"
- echo "Usage: arvsave name"
+ echo "Usage: arvsave <name>"
fi
}
fi
else
echo "Delete Arvados environment conf"
- echo "Usage: arvrm name"
+ echo "Usage: arvrm <name>"
fi
}
arvboxswitch() {
if [[ -n "$1" ]] ; then
+ export ARVBOX_CONTAINER=$1
if [[ -d $HOME/.arvbox/$1 ]] ; then
- export ARVBOX_CONTAINER=$1
echo "Arvbox switched to $1"
else
- echo "$1 unknown"
+ echo "Warning: $1 doesn't exist, will be created."
fi
else
if test -z "$ARVBOX_CONTAINER" ; then
ARVBOX_CONTAINER=arvbox
fi
echo "Switch Arvbox environment conf"
- echo "Usage: arvboxswitch name"
echo "Your current container is: $ARVBOX_CONTAINER"
+ echo "Usage: arvboxswitch <name>"
echo "Available confs:" $(cd $HOME/.arvbox && ls --indicator-style=none)
fi
}
xdg-open https://arvadosapi.com/$1
else
echo "Open Arvados uuid in browser"
- echo "Usage: arvopen uuid"
+ echo "Usage: arvopen <uuid>"
fi
}
xdg-open https://dev.arvados.org/issues/$1
else
echo "Open Arvados issue in browser"
- echo "Usage: arvissue uuid"
+ echo "Usage: arvissue <issue number>"
fi
}
chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH \
/var/lib/passenger /var/lib/postgresql \
/var/lib/nginx /var/log/nginx /etc/ssl/private \
- /var/lib/gopath /var/lib/pip /var/lib/npm \
- /var/lib/arvados
+ /var/lib/gopath /var/lib/pip /var/lib/npm
fi
mkdir -p /tmp/crunch0 /tmp/crunch1
chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
- # singularity needs to be owned by root and suid
- chown root /var/lib/arvados/bin/singularity \
- /var/lib/arvados/etc/singularity/singularity.conf \
- /var/lib/arvados/etc/singularity/capability.json \
- /var/lib/arvados/etc/singularity/ecl.toml
- chmod u+s /var/lib/arvados/bin/singularity
-
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
if [[ ! -f /usr/local/bin/arvados-server ]]; then
$RUNSU flock /var/lib/gopath/gopath.lock go mod download
+ $RUNSU flock /var/lib/gopath/gopath.lock go mod vendor
$RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
$RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin
fi
"region": "{{user `aws_default_region`}}",
"ena_support": "true",
"source_ami": "{{user `aws_source_ami`}}",
- "instance_type": "m4.large",
+ "instance_type": "m5.large",
"vpc_id": "{{user `vpc_id`}}",
"subnet_id": "{{user `subnet_id`}}",
"associate_public_ip_address": "{{user `associate_public_ip_address`}}",
"launch_block_device_mappings": [{
"device_name": "/dev/xvda",
"volume_size": 20,
- "volume_type": "gp2",
+ "volume_type": "gp3",
"delete_on_termination": true
}],
"ami_block_device_mappings": [
done
}
+# $DIST should not have a dot if there is one in /etc/os-release (e.g. 18.04)
+DIST=$(. /etc/os-release; echo $ID$VERSION_ID | tr -d '.')
+
# Run apt-get update
$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
TMP_LSB=`/usr/bin/lsb_release -c -s`
LSB_RELEASE_CODENAME=${TMP_LSB//[$'\t\r\n ']}
+SET_RESOLVER=
+if [ -n "$RESOLVER" ]; then
+ SET_RESOLVER="--dns ${RESOLVER}"
+fi
+
# Add the arvados apt repository
echo "# apt.arvados.org" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
# Install the Arvados packages we need
wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
python3-arvados-fuse \
- arvados-docker-cleaner \
- docker.io
+ arvados-docker-cleaner
+
+# We want Docker 20.10 or later so that we support glibc 2.33 and up in the container, cf.
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005906
+dockerversion=5:20.10.13~3-0
+if [[ "$DIST" =~ ^debian ]]; then
+ family="debian"
+ if [ "$DIST" == "debian10" ]; then
+ distro="buster"
+ elif [ "$DIST" == "debian11" ]; then
+ distro="bullseye"
+ fi
+elif [[ "$DIST" =~ ^ubuntu ]]; then
+ family="ubuntu"
+ if [ "$DIST" == "ubuntu1804" ]; then
+ distro="bionic"
+ elif [ "$DIST" == "ubuntu2004" ]; then
+ distro="focal"
+ fi
+else
+ echo "Unsupported distribution $DIST"
+ exit 1
+fi
+curl -fsSL https://download.docker.com/linux/$family/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/$family/ $distro stable | \
+ $SUDO tee /etc/apt/sources.list.d/docker.list
+$SUDO apt-get update
+$SUDO apt-get -yq --no-install-recommends install docker-ce=${dockerversion}~${family}-${distro}
+
+# Set a higher ulimit and the resolver (if set) for docker
+$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
+ /lib/systemd/system/docker.service \
+ > /etc/systemd/system/docker.service
+
+$SUDO systemctl daemon-reload
+
+# docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
+# and the BootProbeCommand might be "docker ps -q"
+$SUDO systemctl disable docker
# Get Go and build singularity
goversion=1.17.1
$SUDO sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"/g' /etc/default/grub
$SUDO update-grub
-# Set a higher ulimit and the resolver (if set) for docker
-if [ "x$RESOLVER" != "x" ]; then
- SET_RESOLVER="--dns ${RESOLVER}"
-fi
-
-$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
- /lib/systemd/system/docker.service \
- > /etc/systemd/system/docker.service
-
-$SUDO systemctl daemon-reload
-
-# docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
-# and the BootProbeCommand might be "docker ps -q"
-$SUDO systemctl disable docker
-
# Make sure user_allow_other is set in fuse.conf
$SUDO sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf
$SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
fi
-if [ "$AWS_EBS_AUTOSCALE" != "1" ]; then
+# AWS_EBS_AUTOSCALE is not always set, work around unset variable check
+EBS_AUTOSCALE=${AWS_EBS_AUTOSCALE:-}
+
+if [ "$EBS_AUTOSCALE" != "1" ]; then
# Set up the cloud-init script that will ensure encrypted disks
$SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
else
$SUDO chown root:root /etc/cloud/cloud.cfg.d/07_compute_arvados_dispatch_cloud.cfg
if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
- # $DIST should not have a dot if there is one in /etc/os-release (e.g. 18.04)
- DIST=$(. /etc/os-release; echo $ID$VERSION_ID | tr -d '.')
# We need a kernel and matching headers
if [[ "$DIST" =~ ^debian ]]; then
$SUDO apt-get -y install linux-image-cloud-amd64 linux-headers-cloud-amd64
$SUDO apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/7fa2af80.pub
$SUDO apt-get -y install software-properties-common
$SUDO add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/ /"
- $SUDO add-apt-repository contrib
+ # Ubuntu 18.04's add-apt-repository does not understand 'contrib'
+ $SUDO add-apt-repository contrib || true
$SUDO apt-get update
$SUDO apt-get -y install cuda
$SUDO tee /etc/apt/sources.list.d/libnvidia-container.list
fi
- if [ "$DIST" == "debian10" ]; then
- # Debian 10 comes with Docker 18.xx, we need 19.03 or later
- curl -fsSL https://download.docker.com/linux/debian/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
- echo deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian/ buster stable | \
- $SUDO tee /etc/apt/sources.list.d/docker.list
- $SUDO apt-get update
- $SUDO apt-get -yq --no-install-recommends install docker-ce=5:19.03.15~3-0~debian-buster
-
- $SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
- /lib/systemd/system/docker.service \
- > /etc/systemd/system/docker.service
-
- $SUDO systemctl daemon-reload
-
- # docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
- # and the BootProbeCommand might be "docker ps -q"
- $SUDO systemctl disable docker
- fi
$SUDO apt-get update
$SUDO apt-get -y install libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit
# This service fails to start when the image is booted without Nvidia GPUs present, which makes
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+ pysdk_dep = "=={}".format(version)
+else:
+ # On dev releases, arvados-python-client may have a different timestamp
+ pysdk_dep = "<={}".format(version)
short_tests_only = False
if '--short-tests-only' in sys.argv:
('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client',
+ 'arvados-python-client{}'.format(pysdk_dep),
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0'],
s#domain_fixme_or_this_wont_work#local#g;
s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g;
s#RELEASE=\"production\"#RELEASE=\"development\"#g;
- s/# VERSION=.*$/VERSION=\"latest\"/g;
- s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g' \
+ s/# VERSION=.*$/VERSION=\"latest\"/g;' \
/vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
+ # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;' \
+ arv.vm.provision "shell",
+ inline: "cp -vr /tmp/local.params.single_host_multiple_hostnames /tmp/local.params.single_host_multiple_hostnames.falla;
+ cp -vr /vagrant/centos7-local.params.single_host_single_hostname-f258b604f831bb3bd7fab506c670b975ae8e4118 /tmp/local.params.single_host_multiple_hostnames"
arv.vm.provision "shell",
path: "provision.sh",
args: [
# A single_host single_hostname example
config.vm.define "arvados-sh-sn" do |arv|
- arv.vm.box = "bento/debian-10"
+ #arv.vm.box = "bento/centos-7"
+ arv.vm.box = "bento/ubuntu-20.04"
arv.vm.hostname = "zeppo"
# CPU/RAM
config.vm.provider :virtualbox do |v|
arv.vm.provision "shell",
inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
cp -vr /vagrant/tests /home/vagrant/tests;
- sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
- s#cluster_fixme_or_this_wont_work#zeppo#g;
- s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
- s#domain_fixme_or_this_wont_work#local#g;' \
+ sed 's#cluster_fixme_or_this_wont_work#cnts7#g;
+ s#domain_fixme_or_this_wont_work#local#g;
+ s#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"cnts7.local\"#g;
+ s#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g;
+ s#RELEASE=\"production\"#RELEASE=\"development\"#g;
+ s/# BRANCH=\"main\"/BRANCH=\"main\"/g;
+ s/# VERSION=.*$/VERSION=\"latest\"/g' \
/vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
+
arv.vm.provision "shell",
path: "provision.sh",
args: [
- # "--debug",
+ "--debug",
"--config /tmp/local.params.single_host_single_hostname",
"--test",
"--vagrant"
DispatchCloud:
InternalURLs:
'http://__CONTROLLER_INT_IP__:9006': {}
+ Keepbalance:
+ InternalURLs:
+ 'http://localhost:9005': {}
Keepproxy:
ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__'
InternalURLs:
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
nginx:
- install_from_phusionpassenger: true
+ __NGINX_INSTALL_SOURCE__: true
lookup:
passenger_package: {{ passenger_pkg }}
### PASSENGER
### SERVER
server:
config:
+ # As we now differentiate where passenger is required or not, we need to
+ # load this module conditionally, so we add this conditional just to use
+ # the same pillar file
+ {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
# This is required to get the passenger module loaded
# In Debian it can be done with this
# include: 'modules-enabled/*.conf'
load_module: {{ passenger_mod }}
-
+ {% endif %}
worker_processes: 4
### SNIPPETS
### POSTGRESQL
postgres:
use_upstream_repo: true
- version: '11'
+ version: '12'
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
acls:
'http://__CLUSTER__.__DOMAIN__:9006': {}
Keepbalance:
InternalURLs:
- 'http://__CLUSTER__.__DOMAIN__:9005': {}
+ 'http://localhost:9005': {}
Keepproxy:
ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__'
InternalURLs:
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
nginx:
- install_from_phusionpassenger: true
+ __NGINX_INSTALL_SOURCE__: true
lookup:
passenger_package: {{ passenger_pkg }}
### PASSENGER
### SERVER
server:
config:
+ # As we now differentiate where passenger is required or not, we need to
+ # load this module conditionally, so we add this conditional just to use
+ # the same pillar file
+ {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
# This is required to get the passenger module loaded
# In Debian it can be done with this
# include: 'modules-enabled/*.conf'
load_module: {{ passenger_mod }}
-
+ {% endif %}
worker_processes: 4
### SNIPPETS
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+dns:
+ pkg.installed:
+ - pkgs:
+ - dnsmasq
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
host: 127.0.0.1
password: "__DATABASE_PASSWORD__"
user: __CLUSTER___arvados
- encoding: en_US.utf8
+ extra_conn_params:
+ client_encoding: UTF8
+ # Centos7 does not enable SSL by default, so we disable
+ # it here just for testing of the formula purposes only.
+ # You should not do this in production, and should
+ # configure Postgres certificates correctly
+ {%- if grains.os_family in ('RedHat',) %}
+ sslmode: disable
+ {%- endif %}
tls:
# certificate: ''
# When using arvados-snakeoil certs set insecure: true
insecure: true
+ resources:
+ virtual_machines:
+ shell:
+ name: webshell
+ backend: 127.0.1.1
+ port: 4200
+
### TOKENS
tokens:
system_root: __SYSTEM_ROOT_TOKEN__
management: __MANAGEMENT_TOKEN__
anonymous_user: __ANONYMOUS_USER_TOKEN__
- rails_secret: YDLxHf4GqqmLXYAMgndrAmFEdqgC0sBqX7TEjMN2rw9D6EVwgx
### KEYS
secrets:
# <cluster>-nyw5e-<volume>
__CLUSTER__-nyw5e-000000000000000:
AccessViaHosts:
- 'http://__HOSTNAME_INT__:25107':
+ 'http://__IP_INT__:25107':
ReadOnly: false
Replication: 2
Driver: Directory
Controller:
ExternalURL: 'https://__HOSTNAME_EXT__:__CONTROLLER_EXT_SSL_PORT__'
InternalURLs:
- 'http://__HOSTNAME_INT__:8003': {}
+ 'http://__IP_INT__:8003': {}
+ Keepbalance:
+ InternalURLs:
+ 'http://__IP_INT__:9005': {}
Keepproxy:
ExternalURL: 'https://__HOSTNAME_EXT__:__KEEP_EXT_SSL_PORT__'
InternalURLs:
- 'http://__HOSTNAME_INT__:25100': {}
+ 'http://__IP_INT__:25100': {}
Keepstore:
InternalURLs:
- 'http://__HOSTNAME_INT__:25107': {}
+ 'http://__IP_INT__:25107': {}
RailsAPI:
InternalURLs:
- 'http://__HOSTNAME_INT__:8004': {}
+ 'http://__IP_INT__:8004': {}
WebDAV:
ExternalURL: 'https://__HOSTNAME_EXT__:__KEEPWEB_EXT_SSL_PORT__'
InternalURLs:
- 'http://__HOSTNAME_INT__:9003': {}
+ 'http://__IP_INT__:9003': {}
WebDAVDownload:
ExternalURL: 'https://__HOSTNAME_EXT__:__KEEPWEB_EXT_SSL_PORT__'
WebShell:
Websocket:
ExternalURL: 'wss://__HOSTNAME_EXT__:__WEBSOCKET_EXT_SSL_PORT__/websocket'
InternalURLs:
- 'http://__HOSTNAME_INT__:8005': {}
+ 'http://__IP_INT__:8005': {}
Workbench1:
ExternalURL: 'https://__HOSTNAME_EXT__:__WORKBENCH1_EXT_SSL_PORT__'
Workbench2:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+aws_credentials:
+ region: __LE_AWS_REGION__
+ access_key_id: __LE_AWS_ACCESS_KEY_ID__
+ secret_access_key: __LE_AWS_SECRET_ACCESS_KEY__
pkg:
docker:
use_upstream: package
+ daemon_config: {"dns": ["__IP_INT__"]}
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### LETSENCRYPT
+letsencrypt:
+ use_package: true
+ pkgs:
+ - certbot: latest
+ - python3-certbot-nginx
+ config:
+ server: https://acme-v02.api.letsencrypt.org/directory
+ email: __INITIAL_USER_EMAIL__
+ authenticator: nginx
+ agree-tos: true
+ keep-until-expiring: true
+ expand: true
+ max-log-backups: 0
+ deploy-hook: systemctl reload nginx
+
+ domainsets:
+ __HOSTNAME_EXT__:
+ - __HOSTNAME_EXT__
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
- arvados_api:
+ arvados_api.conf:
enabled: true
overwrite: true
config:
- server:
- - listen: '__HOSTNAME_INT__:8004'
+ - listen: '__IP_INT__:8004'
- server_name: api
- root: /var/www/arvados-api/current/public
- index: index.html index.htm
default: 1
'127.0.0.0/8': 0
upstream controller_upstream:
- - server: '__HOSTNAME_INT__:8003 fail_timeout=10s'
+ - server: '__IP_INT__:8003 fail_timeout=10s'
### SITES
servers:
managed:
### DEFAULT
- arvados_controller_default:
+ arvados_controller_default.conf:
enabled: true
overwrite: true
config:
- server:
- server_name: _
- listen:
- - 80 default_server
+ - 80
- location /.well-known:
- root: /var/www
- location /:
- return: '301 https://$host$request_uri'
- arvados_controller_ssl:
+ arvados_controller_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- proxy_set_header: 'X-External-Client $external_client'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
- client_max_body_size: 128m
### STREAMS
http:
upstream keepproxy_upstream:
- - server: '__HOSTNAME_INT__:25100 fail_timeout=10s'
+ - server: '__IP_INT__:25100 fail_timeout=10s'
servers:
managed:
- arvados_keepproxy_ssl:
+ ### DEFAULT
+ arvados_keepproxy_default.conf:
enabled: true
overwrite: true
+ config:
+ - server:
+ - server_name: keep.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_keepproxy_ssl.conf:
+ enabled: true
+ overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
### STREAMS
http:
upstream collections_downloads_upstream:
- - server: '__HOSTNAME_INT__:9003 fail_timeout=10s'
+ - server: '__IP_INT__:9003 fail_timeout=10s'
servers:
managed:
### COLLECTIONS / DOWNLOAD
- arvados_collections_download_ssl:
+ arvados_collections_download_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- client_max_body_size: 0
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.error.log
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
nginx:
- install_from_phusionpassenger: true
+ __NGINX_INSTALL_SOURCE__: true
lookup:
passenger_package: {{ passenger_pkg }}
### PASSENGER
### SERVER
server:
config:
+ # As we now differentiate where passenger is required or not, we need to
+ # load this module conditionally, so we add this conditional just to use
+ # the same pillar file
+ {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
# This is required to get the passenger module loaded
# In Debian it can be done with this
# include: 'modules-enabled/*.conf'
load_module: {{ passenger_mod }}
-
+ {% endif %}
worker_processes: 4
### SNIPPETS
### SITES
servers:
managed:
- # Remove default webserver
+ # Update default config to redirect to https
default:
- enabled: false
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: _
+ - listen:
+ - 80 default_server
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
### STREAMS
http:
upstream webshell_upstream:
- - server: '__HOSTNAME_INT__:4200 fail_timeout=10s'
+ - server: '__IP_INT__:4200 fail_timeout=10s'
### SITES
servers:
managed:
- arvados_webshell_ssl:
+ arvados_webshell_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
- add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
### STREAMS
http:
upstream websocket_upstream:
- - server: '__HOSTNAME_INT__:8005 fail_timeout=10s'
+ - server: '__IP_INT__:8005 fail_timeout=10s'
servers:
managed:
- arvados_websocket_ssl:
+ arvados_websocket_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
- arvados_workbench2_ssl:
+ ### DEFAULT
+ arvados_workbench2_default.conf:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench2_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- return: 503
- location /config.json:
- return: {{ "200 '" ~ '{"API_HOST":"__HOSTNAME_EXT__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### STREAMS
http:
upstream workbench_upstream:
- - server: '__HOSTNAME_INT__:9000 fail_timeout=10s'
+ - server: '__IP_INT__:9000 fail_timeout=10s'
### SITES
servers:
managed:
- arvados_workbench_ssl:
+ ### DEFAULT
+ arvados_workbench_default.conf:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ __CERT_REQUIRES__
config:
- server:
- server_name: __HOSTNAME_EXT__
- proxy_set_header: 'Host $http_host'
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
- access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
overwrite: true
config:
- server:
- - listen: '__HOSTNAME_INT__:9000'
+ - listen: '__IP_INT__:9000'
- server_name: workbench
- root: /var/www/arvados-workbench/current/public
- index: index.html index.htm
### POSTGRESQL
postgres:
+ # Centos-7's postgres package is too old, so we need to force using upstream's
+ # This is not required in Debian's family as they already ship with PG +11
+ {%- if salt['grains.get']('os_family') == 'RedHat' %}
+ use_upstream_repo: true
+ version: '12'
+
+ pkgs_deps:
+ - libicu
+ - libxslt
+ - systemd-sysv
+
+ pkgs_extra:
+ - postgresql12-contrib
+
+ {%- else %}
use_upstream_repo: false
pkgs_extra:
- postgresql-contrib
+ {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
+ # If you want to enable communications' encryption to the DB server,
+ # uncomment these entries
+ # ssl = on
+ # ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+ # ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
acls:
- ['local', 'all', 'postgres', 'peer']
- ['local', 'all', 'all', 'peer']
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs') %}
+{%- set dest_cert_dir = '/etc/nginx/ssl' %}
+{%- set certs = salt['pillar.get']('extra_custom_certs', []) %}
+
+{% if certs %}
+extra_custom_certs_file_directory_certs_dir:
+ file.directory:
+ - name: /etc/nginx/ssl
+ - require:
+ - pkg: nginx_install
+
+ {%- for cert in certs %}
+ {%- set cert_file = 'arvados-' ~ cert ~ '.pem' %}
+ {#- set csr_file = 'arvados-' ~ cert ~ '.csr' #}
+ {%- set key_file = 'arvados-' ~ cert ~ '.key' %}
+ {% for c in [cert_file, key_file] %}
+extra_custom_certs_file_copy_{{ c }}:
+ file.copy:
+ - name: {{ dest_cert_dir }}/{{ c }}
+ - source: {{ orig_cert_dir }}/{{ c }}
+ - force: true
+ - user: root
+ - group: root
+ - unless: cmp {{ dest_cert_dir }}/{{ c }} {{ orig_cert_dir }}/{{ c }}
+ - require:
+ - file: extra_custom_certs_file_directory_certs_dir
+ {%- endfor %}
+ {%- endfor %}
+{%- endif %}
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+dns:
+ pkg.installed:
+ - pkgs:
+ - dnsmasq
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
+# We need the external hostname to resolve to the internal IP for docker. We
+# tell docker to resolve via the local dnsmasq, which reads from /etc/hosts by
+# default.
+arvados_local_access_to_hostname_ext:
+ host.present:
+ - ip: __IP_INT__
+ - names:
+ - __HOSTNAME_EXT__
+
arvados_test_salt_states_examples_single_host_etc_hosts_host_present:
host.present:
- ip: 127.0.1.1
- names:
- {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- # FIXME! This just works for our testings.
+ # FIXME! This just works for our testing.
# Won't work if the cluster name != host name
{%- for entry in [
'api',
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs') %}
+
include:
- nginx.passenger
- nginx.config
# we'll keep it simple here.
{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
{%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
-{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
-{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
{%- if grains.get('os_family') == 'Debian' %}
{%- set arvados_ca_cert_dest = '/usr/local/share/ca-certificates/arvados-snakeoil-ca.crt' %}
{%- set update_ca_cert = '/usr/sbin/update-ca-certificates' %}
{%- set openssl_conf = '/etc/ssl/openssl.cnf' %}
+
+extra_snakeoil_certs_ssl_cert_pkg_installed:
+ pkg.installed:
+ - name: ssl-cert
+ - require_in:
+ - sls: postgres
+
{%- else %}
{%- set arvados_ca_cert_dest = '/etc/pki/ca-trust/source/anchors/arvados-snakeoil-ca.pem' %}
{%- set update_ca_cert = '/usr/bin/update-ca-trust' %}
{%- set openssl_conf = '/etc/pki/tls/openssl.cnf' %}
+
{%- endif %}
-arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed:
+extra_snakeoil_certs_dependencies_pkg_installed:
pkg.installed:
- pkgs:
- openssl
- ca-certificates
-arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
+# Remove the RANDFILE parameter in openssl.cnf as it makes openssl fail in Ubuntu 18.04
+# Saving and restoring the rng state is not necessary anymore in the openssl 1.1.1
+# random generator, cf
+# https://github.com/openssl/openssl/issues/7754
+#
+extra_snakeoil_certs_file_comment_etc_openssl_conf:
+ file.comment:
+ - name: /etc/ssl/openssl.cnf
+ - regex: ^RANDFILE.*
+ - onlyif: grep -q ^RANDFILE /etc/ssl/openssl.cnf
+ - require_in:
+ - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
+
+extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run:
# Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
- # These dirs are not to CentOS-ish, but this is a helper script
+ # These dirs are not too CentOS-ish, but this is a helper script
# and they should be enough
mkdir -p /etc/ssl/certs/ /etc/ssl/private/ && \
openssl req \
- test -f {{ arvados_ca_cert_file }}
- openssl verify -CAfile {{ arvados_ca_cert_file }} {{ arvados_ca_cert_file }}
- require:
- - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
+ - pkg: extra_snakeoil_certs_dependencies_pkg_installed
-arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run:
+{%- set arvados_cert_file = orig_cert_dir ~ '/arvados-__HOSTNAME_EXT__.pem' %}
+{%- set arvados_csr_file = orig_cert_dir ~ '/arvadoos-__HOSTNAME_EXT__.csr' %}
+{%- set arvados_key_file = orig_cert_dir ~ '/arvados-__HOSTNAME_EXT__.key' %}
+
+extra_snakeoil_certs_arvados_snakeoil_cert___HOSTNAME_EXT___cmd_run:
cmd.run:
- name: |
- cat > /tmp/openssl.cnf <<-CNF
+ cat > /tmp/__HOSTNAME_EXT__.openssl.cnf <<-CNF
[req]
default_bits = 2048
prompt = no
default_md = sha256
- req_extensions = rext
distinguished_name = dn
+ req_extensions = rext
+ [rext]
+ subjectAltName = @alt_names
[dn]
C = CC
ST = Some State
L = Some Location
- O = Arvados Formula
- OU = arvados-formula
+ O = Arvados Provision Example Single Host / Single Hostname
+ OU = arvados-provision-example-single_host_single_hostname
CN = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
emailAddress = admin@{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- [rext]
- subjectAltName = @alt_names
[alt_names]
{%- for entry in grains.get('ipv4') %}
IP.{{ loop.index }} = {{ entry }}
{%- endfor %}
- {%- for entry in [
- 'keep',
- 'collections',
- 'download',
- 'keepweb',
- 'ws',
- 'workbench',
- 'workbench2',
- ]
- %}
- DNS.{{ loop.index }} = {{ entry }}
- {%- endfor %}
- DNS.8 = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
- DNS.9 = '__HOSTNAME_EXT__'
- DNS.10 = '__HOSTNAME_INT__'
+ DNS.1 = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
+ DNS.2 = '__HOSTNAME_EXT__'
CNF
# The req
openssl req \
- -config /tmp/openssl.cnf \
+ -config /tmp/__HOSTNAME_EXT__.openssl.cnf \
-new \
-nodes \
-sha256 \
-out {{ arvados_csr_file }} \
- -keyout {{ arvados_key_file }} > /tmp/snake_oil_certs.output 2>&1 && \
+ -keyout {{ arvados_key_file }} > /tmp/snake_oil_certs.__HOSTNAME_EXT__.output 2>&1 && \
# The cert
openssl x509 \
-req \
-days 365 \
-in {{ arvados_csr_file }} \
-out {{ arvados_cert_file }} \
- -extfile /tmp/openssl.cnf \
+ -extfile /tmp/__HOSTNAME_EXT__.openssl.cnf \
-extensions rext \
-CA {{ arvados_ca_cert_file }} \
-CAkey {{ arvados_ca_key_file }} \
- test -f {{ arvados_key_file }}
- openssl verify -CAfile {{ arvados_ca_cert_file }} {{ arvados_cert_file }}
- require:
- - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
- - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
- # We need this before we can add the nginx's snippet
+ - pkg: extra_snakeoil_certs_dependencies_pkg_installed
+ - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
- require_in:
- - file: nginx_snippet_arvados-snakeoil.conf
+ - file: extra_custom_certs_file_copy_arvados-__HOSTNAME_EXT__.pem
+ - file: extra_custom_certs_file_copy_arvados-__HOSTNAME_EXT__.key
-{%- if grains.get('os_family') == 'Debian' %}
-arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
- pkg.installed:
- - name: ssl-cert
- - require_in:
- - sls: postgres
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
+ {%- if grains.get('os_family') == 'Debian' %}
+extra_snakeoil_certs_certs_permissions___HOSTNAME_EXT___cmd_run:
file.managed:
- name: {{ arvados_key_file }}
- owner: root
- group: ssl-cert
- require:
- - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
- - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
- - require_in:
- - file: nginx_snippet_arvados-snakeoil.conf
-{%- endif %}
+ - cmd: extra_snakeoil_certs_arvados_snakeoil_cert___HOSTNAME_EXT___cmd_run
+ - pkg: extra_snakeoil_certs_ssl_cert_pkg_installed
+ {%- endif %}
DATABASE_PASSWORD=please_set_this_to_some_secure_value
# SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will fail
-# to communicate and can silently drop traffic. You can try to use the Letsencrypt
-# salt formula (https://github.com/saltstack-formulas/letsencrypt-formula) to try to
-# automatically obtain and install SSL certificates for your instances or set this
-# variable to "no", provide and upload your own certificates to the instances and
-# modify the 'nginx_*' salt pillars accordingly (see CUSTOM_CERTS_DIR below)
-USE_LETSENCRYPT="yes"
-USE_LETSENCRYPT_IAM_USER="yes"
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
+#
+# See https://doc.arvados.org/intall/salt-multi-host.html for more information.
+SSL_MODE="lets-encrypt"
+USE_LETSENCRYPT_ROUTE53="yes"
# For collections, we need to obtain a wildcard certificate for
# '*.collections.<cluster>.<domain>'. This is only possible through a DNS-01 challenge.
# For that reason, you'll need to provide AWS credentials with permissions to manage
LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
+# help you deploy them. In order to do that, you need to set `SSL_MODE=bring-your-own` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
# Formulas versions
# ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
# The domainname you want tou give to your cluster's hosts
DOMAIN="domain_fixme_or_this_wont_work"
-# Host SSL port where you want to point your browser to access Arvados
-# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
-# You can point it to another port if desired
-# In Vagrant, make sure it matches what you set in the Vagrantfile (8443)
+# External ports used by the Arvados services
CONTROLLER_EXT_SSL_PORT=443
KEEP_EXT_SSL_PORT=25101
-# Both for collections and downloads
KEEPWEB_EXT_SSL_PORT=9002
WEBSHELL_EXT_SSL_PORT=4202
WEBSOCKET_EXT_SSL_PORT=8002
WORKBENCH2_EXT_SSL_PORT=3001
INITIAL_USER="admin"
-
# If not specified, the initial user email will be composed as
# INITIAL_USER@CLUSTER.DOMAIN
INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"
DATABASE_PASSWORD=please_set_this_to_some_secure_value
# SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
#
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
-# If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
-# and copy the required certificates under the directory specified in the next line.
-# The certs will be copied from this directory by the provision script.
-# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
-# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
-# directory where the "provision.sh" script was copied in the destination host.
+# CUSTOM_CERTS_DIR is only used when SSL_MODE is set to "bring-your-own".
+# See https://doc.arvados.org/intall/salt-single-host.html#bring-your-own for more information.
# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
-# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both download/collections):
-# "controller"
-# "websocket"
-# "workbench"
-# "workbench2"
-# "webshell"
-# "download" # Part of keepweb
-# "collections" # Part of keepweb
-# "keepproxy"
-# Ie., 'keepproxy', the script will lookup for
-# ${CUSTOM_CERTS_DIR}/keepproxy.crt
-# ${CUSTOM_CERTS_DIR}/keepproxy.key
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
# CONFIG_DIR="local_config_dir"
+
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
# Formulas versions
# ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
# These are the basic parameters to configure the installation
-# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster
+# The Arvados cluster ID, needs to be five alphanumeric characters.
CLUSTER="cluster_fixme_or_this_wont_work"
-# The domainname you want tou give to your cluster's hosts
+# The domainname for your cluster's hosts
DOMAIN="domain_fixme_or_this_wont_work"
-# Set this value when installing a cluster in a single host with a single hostname
-# to access all the instances. Not used in the other examples.
-# When using virtualization (ie AWS), this should be
-# the EXTERNAL/PUBLIC hostname for the instance.
-# If empty, ${CLUSTER}.${DOMAIN} will be used
-HOSTNAME_EXT=""
-# The internal hostname for the host. In the example files, only used in the
-# single_host/single_hostname example
-HOSTNAME_INT="127.0.1.1"
-# Host SSL port where you want to point your browser to access Arvados
-# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
-# You can point it to another port if desired
-# In Vagrant, make sure it matches what you set in the Vagrantfile (8443)
-CONTROLLER_EXT_SSL_PORT=9443
-KEEP_EXT_SSL_PORT=35101
-# Both for collections and downloads
-KEEPWEB_EXT_SSL_PORT=11002
-WEBSHELL_EXT_SSL_PORT=14202
-WEBSOCKET_EXT_SSL_PORT=18002
-WORKBENCH1_EXT_SSL_PORT=9444
-WORKBENCH2_EXT_SSL_PORT=9445
+# Set this value when installing a cluster in a single host with a single
+# hostname to access all the instances. HOSTNAME_EXT should be set to the
+# external hostname for the instance.
+HOSTNAME_EXT="hostname_ext_fixme_or_this_wont_work"
-INITIAL_USER="admin"
+# The internal IP address for the host.
+IP_INT="ip_int_fixme_or_this_wont_work"
+
+# External ports used by the Arvados services
+CONTROLLER_EXT_SSL_PORT=8800
+KEEP_EXT_SSL_PORT=8801
+KEEPWEB_EXT_SSL_PORT=8802
+WEBSHELL_EXT_SSL_PORT=8803
+WEBSOCKET_EXT_SSL_PORT=8804
+WORKBENCH1_EXT_SSL_PORT=8805
+WORKBENCH2_EXT_SSL_PORT=443
+INITIAL_USER="admin"
# If not specified, the initial user email will be composed as
# INITIAL_USER@CLUSTER.DOMAIN
INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"
INITIAL_USER_PASSWORD="password"
-# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS
+# Populate these values with random strings
BLOB_SIGNING_KEY=blobsigningkeymushaveatleast32characters
MANAGEMENT_TOKEN=managementtokenmushaveatleast32characters
SYSTEM_ROOT_TOKEN=systemroottokenmushaveatleast32characters
DATABASE_PASSWORD=please_set_this_to_some_secure_value
# SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
#
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
+
+# CUSTOM_CERTS_DIR is only used when SSL_MODE is set to "bring-your-own".
+# See https://doc.arvados.org/intall/salt-single-host.html#bring-your-own for more information.
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
# CONFIG_DIR="local_config_dir"
+
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
# Formulas versions
# ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
echo >&2 " controller"
echo >&2 " dispatcher"
echo >&2 " keepproxy"
+ echo >&2 " keepbalance"
echo >&2 " keepstore"
echo >&2 " keepweb"
echo >&2 " shell"
for i in ${2//,/ }
do
# Verify the role exists
- if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+ if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
echo "The role '${i}' is not a valid role"
usage
exit 1
CONTROLLER_EXT_SSL_PORT=443
TESTS_DIR="tests"
+NGINX_INSTALL_SOURCE="install_from_repo"
+
CLUSTER=""
DOMAIN=""
# Hostnames/IPs used for single-host deploys
-HOSTNAME_EXT=""
-HOSTNAME_INT="127.0.1.1"
+IP_INT="127.0.1.1"
# Initial user setup
INITIAL_USER=""
WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
-USE_LETSENCRYPT="no"
+SSL_MODE="self-signed"
+USE_LETSENCRYPT_ROUTE53="no"
CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
## These are ARVADOS-related parameters
# BRANCH="main"
# Other formula versions we depend on
-POSTGRES_TAG="v0.43.0"
-NGINX_TAG="temp-fix-missing-statements-in-pillar"
-DOCKER_TAG="v2.0.7"
+POSTGRES_TAG="v0.44.0"
+NGINX_TAG="v2.8.1"
+DOCKER_TAG="v2.4.2"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
fi
# Only used in single_host/single_name deploys
-if [ "x${HOSTNAME_EXT}" = "x" ] ; then
+if [ ! -z "${HOSTNAME_EXT}" ] ; then
+ # We need to add some extra control vars to manage a single certificate vs. multiple
+ USE_SINGLE_HOSTNAME="yes"
+ # Make sure that the value configured as IP_INT is a real IP on the system.
+ # If we don't error out early here when there is a mismatch, the formula will
+ # fail with hard to interpret nginx errors later on.
+ ip addr list |grep -q "${IP_INT}/"
+ if [[ $? -ne 0 ]]; then
+ echo "Unable to find the IP_INT address '${IP_INT}' on the system, please correct the value in local.params. Exiting..."
+ exit 1
+ fi
+else
+ USE_SINGLE_HOSTNAME="no"
+ # We set this variable, anyway, so sed lines do not fail and we don't need to add more
+ # conditionals
HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
fi
case ${OS_ID} in
"centos")
echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
- sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+ sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
setenforce permissive
yum install -y curl git jq
;;
git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+echo "...locale"
git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
-git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+echo "...nginx"
+git clone --quiet https://github.com/saltstack-formulas/nginx-formula.git ${F_DIR}/nginx
( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+echo "...postgres"
git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+echo "...letsencrypt"
git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
+echo "...arvados"
git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
# If we want to try a specific branch of the formula
s#__CLUSTER__#${CLUSTER}#g;
s#__DOMAIN__#${DOMAIN}#g;
s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
- s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+ s#__IP_INT__#${IP_INT}#g;
s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
s#__INITIAL_USER__#${INITIAL_USER}#g;
mkdir -p ${T_DIR}
# Replace cluster and domain name in the test files
for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
- sed "s#__CLUSTER__#${CLUSTER}#g;
+ FILTERS="s#__CLUSTER__#${CLUSTER}#g;
s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
s#__DOMAIN__#${DOMAIN}#g;
- s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+ s#__IP_INT__#${IP_INT}#g;
s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g
s#__INITIAL_USER__#${INITIAL_USER}#g;
s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
- s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
- "${f}" > ${T_DIR}/$(basename "${f}")
+ s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g"
+ if [ "$USE_SINGLE_HOSTNAME" = "yes" ]; then
+ FILTERS="s#__CLUSTER__.__DOMAIN__#${HOSTNAME_EXT}#g;
+ $FILTERS"
+ fi
+ sed "$FILTERS" \
+ "${f}" > ${T_DIR}/$(basename "${f}")
done
chmod 755 ${T_DIR}/run-test.sh
s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
s#__DOMAIN__#${DOMAIN}#g;
s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
- s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+ s#__IP_INT__#${IP_INT}#g;
s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
s#__INITIAL_USER__#${INITIAL_USER}#g;
s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
+ s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
# States, extra states
if [ -d "${F_DIR}"/extra/extra ]; then
- if [ "$DEV_MODE" = "yes" ]; then
+ SKIP_SNAKE_OIL="snakeoil_certs"
+
+ if [[ "$DEV_MODE" = "yes" || "${SSL_MODE}" == "self-signed" ]] ; then
# In dev mode, we create some snake oil certs that we'll
- # use as CUSTOM_CERTS, so we don't skip the states file
- SKIP_SNAKE_OIL="dont_snakeoil_certs"
- else
- SKIP_SNAKE_OIL="snakeoil_certs"
+ # use as CUSTOM_CERTS, so we don't skip the states file.
+ # Same when using self-signed certificates.
+ SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
fi
for f in $(ls "${F_DIR}"/extra/extra/*.sls | grep -v ${SKIP_SNAKE_OIL}); do
echo " - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
done
- # Use custom certs
- if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+ # Use byo or self-signed certificates
+ if [ "${SSL_MODE}" != "lets-encrypt" ]; then
mkdir -p "${F_DIR}"/extra/extra/files
fi
fi
if [ -z "${ROLES}" ]; then
# States
echo " - nginx.passenger" >> ${S_DIR}/top.sls
- # Currently, only available on config_examples/multi_host/aws
- if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - extra.aws_credentials" >> ${S_DIR}/top.sls
fi
grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
- # Use custom certs
+ # Use custom certs, as both bring-your-own and self-signed are copied using this state
# Copy certs to formula extra/files
# In dev mode, the files will be created and put in the destination directory by the
# snakeoil_certs.sls state file
echo " - nginx_workbench_configuration" >> ${P_DIR}/top.sls
echo " - postgresql" >> ${P_DIR}/top.sls
- # Currently, only available on config_examples/multi_host/aws
- if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+ # We need to tweak the Nginx's pillar depending whether we want plan nginx or nginx+passenger
+ NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+ sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
+
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
fi
- grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
# As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
- for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
- grep -q ${c} ${P_DIR}/extra_custom_certs.sls || echo " - ${c}" >> ${P_DIR}/extra_custom_certs.sls
-
- # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
- sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${c}.pem/g;
- s#__CERT_PEM__#/etc/nginx/ssl/arvados-${c}.pem#g;
- s#__CERT_KEY__#/etc/nginx/ssl/arvados-${c}.key#g" \
+ for c in controller websocket workbench workbench2 webshell keepweb keepproxy; do
+ # Are we in a single-host-single-hostname env?
+ if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
+ # Are we in a single-host-single-hostname env?
+ CERT_NAME=${HOSTNAME_EXT}
+ else
+ # We are in a multiple-hostnames env
+ CERT_NAME=${c}
+ fi
+
+ if [[ "$SSL_MODE" == "bring-your-own" ]]; then
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${CERT_NAME}
+ fi
+
+ grep -q ${CERT_NAME} ${P_DIR}/extra_custom_certs.sls || echo " - ${CERT_NAME}" >> ${P_DIR}/extra_custom_certs.sls
+
+ # As the pillar differs whether we use LE or custom certs, we need to do a final edition on them
+ sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${CERT_NAME}.pem/g;
+ s#__CERT_PEM__#/etc/nginx/ssl/arvados-${CERT_NAME}.pem#g;
+ s#__CERT_KEY__#/etc/nginx/ssl/arvados-${CERT_NAME}.key#g" \
${P_DIR}/nginx_${c}_configuration.sls
done
fi
else
# If we add individual roles, make sure we add the repo first
echo " - arvados.repo" >> ${S_DIR}/top.sls
- # We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ # We add the extra_custom_certs state
+ grep -q "extra.custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
# And we add the basic part for the certs pillar
- if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+ if [ "${SSL_MODE}" != "lets-encrypt" ]; then
# And add the certs in the custom_certs pillar
echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
# States
# FIXME: https://dev.arvados.org/issues/17352
grep -q "postgres.client" ${S_DIR}/top.sls || echo " - postgres.client" >> ${S_DIR}/top.sls
- grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ if grep -q " - nginx.*$" ${S_DIR}/top.sls; then
+ sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${S_DIR}/top.sls
+ else
+ echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ fi
### If we don't install and run LE before arvados-api-server, it fails and breaks everything
### after it. So we add this here as we are, after all, sharing the host for api and controller
- # Currently, only available on config_examples/multi_host/aws
- if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
fi
grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
# Use custom certs
- copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+ if [ "${SSL_MODE}" = "bring-your-own" ]; then
+ copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+ fi
grep -q controller ${P_DIR}/extra_custom_certs.sls || echo " - controller" >> ${P_DIR}/extra_custom_certs.sls
fi
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls
grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls
grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+
+ # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+ NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+ sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
;;
"controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
# States
- grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls
- # Currently, only available on config_examples/multi_host/aws
- if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+ if [ "${R}" = "workbench" ]; then
+ NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+ if grep -q " - nginx$" ${S_DIR}/top.sls; then
+ sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${S_DIR}/top.sls
+ else
+ echo " - nginx.passenger" >> ${S_DIR}/top.sls
+ fi
+ else
+ grep -q "nginx" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls
+ fi
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
fi
grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
# Use custom certs, special case for keepweb
if [ ${R} = "keepweb" ]; then
- copy_custom_cert ${CUSTOM_CERTS_DIR} download
- copy_custom_cert ${CUSTOM_CERTS_DIR} collections
+ if [ "${SSL_MODE}" = "bring-your-own" ]; then
+ copy_custom_cert ${CUSTOM_CERTS_DIR} download
+ copy_custom_cert ${CUSTOM_CERTS_DIR} collections
+ fi
else
- copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
+ if [ "${SSL_MODE}" = "bring-your-own" ]; then
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
+ fi
fi
fi
# webshell role is just a nginx vhost, so it has no state
grep -q "nginx_collections_configuration" ${P_DIR}/top.sls || echo " - nginx_collections_configuration" >> ${P_DIR}/top.sls
fi
- # Currently, only available on config_examples/multi_host/aws
- if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
- if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
fi
grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
fi
fi
+ # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+ sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
;;
"shell")
# States
# Pillars
grep -q "docker" ${P_DIR}/top.sls || echo " - docker" >> ${P_DIR}/top.sls
;;
- "dispatcher")
- # States
- grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
- # Pillars
- # ATM, no specific pillar needed
- ;;
- "keepstore")
+ "dispatcher" | "keepbalance" | "keepstore")
# States
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
exit 0
fi
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ -e /root/.psqlrc ]; then
- if ! ( grep 'pset pager off' /root/.psqlrc ); then
- RESTORE_PSQL="yes"
- cp /root/.psqlrc /root/.psqlrc.provision.backup
- fi
-else
- DELETE_PSQL="yes"
-fi
-
-echo '\pset pager off' >> /root/.psqlrc
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
-
# Now run the install
salt-call --local state.apply -l ${LOG_LEVEL}
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ "x${DELETE_PSQL}" = "xyes" ]; then
- echo "Removing .psql file"
- rm /root/.psqlrc
-fi
-
-if [ "x${RESTORE_PSQL}" = "xyes" ]; then
- echo "Restoring .psql file"
- mv -v /root/.psqlrc.provision.backup /root/.psqlrc
+# Finally, make sure that /etc/hosts is not overwritten on reboot
+if [ -d /etc/cloud/cloud.cfg.d ]; then
+ # TODO: will this work on CentOS?
+ sed -i 's/^manage_etc_hosts: true/#manage_etc_hosts: true/g' /etc/cloud/cloud.cfg.d/*
fi
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
# Leave a copy of the Arvados CA so the user can copy it where it's required
if [ "$DEV_MODE" = "yes" ]; then
echo "Arvados project uuid is '${project_uuid}'"
-echo "Uploading arvados/jobs' docker image to the project"
-VERSION="2.1.1"
-arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
-
# Create the initial user
echo "Creating initial user '__INITIAL_USER__'"
user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')
projects / arvados.git / commitdiff
