# SPDX-License-Identifier: AGPL-3.0
SHELL := '/bin/bash'
-all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
+all: centos7/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -f -rlt centos7/generated common-generated/*
+all: debian10/generated
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
cp -f -rlt debian10/generated common-generated/*
+all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
cp -f -rlt debian11/generated common-generated/*
+all: debian12/generated
+debian12/generated: common-generated-all
+ test -d debian12/generated || mkdir debian12/generated
+ cp -f -rlt debian12/generated common-generated/*
+
+all: rocky8/generated
rocky8/generated: common-generated-all
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
+all: ubuntu1804/generated
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -f -rlt ubuntu1804/generated common-generated/*
+all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
+all: ubuntu2204/generated
+ubuntu2204/generated: common-generated-all
+ test -d ubuntu2204/generated || mkdir ubuntu2204/generated
+ cp -f -rlt ubuntu2204/generated common-generated/*
+
GOTARBALL_=DOES_NOT_EXIST
NODETARBALL_=DOES_NOT_EXIST
GOVERSION=$(shell grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $$2}')
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN cd /usr/local/src \
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ARG HOSTTYPE
+ARG BRANCH
+ARG GOVERSION
+
+## dont use debian:12 here since the word 'bookworm' is used for rvm precompiled binaries
+FROM debian:bookworm as build_x86_64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64
+
+FROM debian:bookworm as build_aarch64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+RUN echo HOSTTYPE ${HOSTTYPE}
+
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+SHELL ["/bin/bash", "-c"]
+# Install dependencies.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-virtualenv python3-dev libpam-dev equivs
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
+
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs 8"
+
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian12"]
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ARG HOSTTYPE
+ARG BRANCH
+ARG GOVERSION
+
+FROM ubuntu:jammy as build_x86_64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64, and that requires some sources.list mangling
+ONBUILD RUN /bin/sed -i 's/deb http/deb [ arch=amd64 ] http/' /etc/apt/sources.list
+ONBUILD ADD ports.list /etc/apt/sources.list.d/
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64
+
+FROM ubuntu:jammy as build_aarch64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
+LABEL org.opencontainers.image.authors="Arvados Package Maintainers <packaging@arvados.org>"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+SHELL ["/bin/bash", "-c"]
+# Install dependencies.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
+
+# Install virtualenv
+RUN /usr/bin/pip3 install 'virtualenv<20'
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
+
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs 8"
+
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
+
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2204"]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
-
+all: centos7/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -f -rlt centos7/generated common-generated/*
+all: debian10/generated
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
cp -f -rlt debian10/generated common-generated/*
+all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
cp -f -rlt debian11/generated common-generated/*
+all: debian12/generated
+debian12/generated: common-generated-all
+ test -d debian12/generated || mkdir debian12/generated
+ cp -f -rlt debian12/generated common-generated/*
+
+all: rocky8/generated
rocky8/generated: common-generated-all
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
+all: ubuntu1804/generated
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -f -rlt ubuntu1804/generated common-generated/*
+all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
+all: ubuntu2204/generated
+ubuntu2204/generated: common-generated-all
+ test -d ubuntu2204/generated || mkdir ubuntu2204/generated
+ cp -f -rlt ubuntu2204/generated common-generated/*
+
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM debian:bookworm
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies
+RUN apt-get update && \
+ apt-get -y install --no-install-recommends curl ca-certificates gpg procps gpg-agent
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> /etc/gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian12/ /" >>/etc/apt/sources.list
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM ubuntu:jammy
+LABEL org.opencontainers.image.authors="Arvados Package Maintainers <packaging@arvados.org>"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies
+RUN apt-get update && \
+ apt-get -y install --no-install-recommends curl ca-certificates gnupg2
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu2204/ /" >>/etc/apt/sources.list
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
declare -A dd
dd[debian10]=buster
dd[debian11]=bullseye
+ dd[debian12]=bookworm
dd[ubuntu1804]=bionic
dd[ubuntu2004]=focal
+ dd[ubuntu2204]=jammy
D=${dd[$TARGET]}
if [ ${pkgname:0:3} = "lib" ]; then
repo_subdir=${pkgname:0:4}
rm -rf dist/*
- # Get the latest setuptools
- if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ # Get the latest setuptools.
+ #
+ # Note "pip3 install setuptools" fails on debian12 ("error:
+ # externally-managed-environment") even if that requirement is
+ # already satisfied, so we parse "pip3 list" output instead to check
+ # whether we need to do anything.
+ if [[ "$($pip list | grep -P -o '^setuptools\s+\K[0-9]+')" -ge 66 ]]; then
+ : # OK, already installed
+ elif ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'; then
echo "Error, unable to upgrade setuptools with"
- echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'"
exit 1
fi
# filter a useless warning (when building the cwltest package) from the stderr output
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.6 or newer is required.
+Ruby 2.7 or newer is required.
* "Option 1: Install from packages":#packages
* "Option 2: Install with RVM":#rvm
-* "Option 3: Install from source":#fromsource
h2(#packages). Option 1: Install from packages
-{% include 'notebox_begin' %}
-Future versions of Arvados may require a newer version of Ruby than is packaged with your OS. Using OS packages simplifies initial install, but may complicate upgrades that rely on a newer Ruby. If this is a concern, we recommend using "RVM":#rvm.
-{% include 'notebox_end' %}
-
h3. Alma/CentOS/Red Hat/Rocky
-The Ruby version shipped with version 7 of these distributions is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+Version 7 of these distributions does not provide a new enough Ruby version. Use "RVM":#rvm to install Ruby 2.7 or newer.
-Ruby 2.7 is available in a module with version 8 of these distributions. You can install it by running:
+Version 8 of these distributions provides Ruby 2.7. You can install it by running:
<notextile>
<pre><code># <span class="userinput">dnf module enable ruby:2.7</span>
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install Ruby 2.7 or newer.
Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
h2(#rvm). Option 2: Install with RVM
+{% include 'notebox_begin_warning' %}
+We do not recommend using RVM unless the Ruby version provided by your OS distribution is older than 2.7.
+{% include 'notebox_end' %}
+
h3. Install gpg and curl
-h4. Alma/CentOS/Red Hat/Rocky
+h4. CentOS/Red Hat 7
+
+<pre>
+yum install gpg curl which findutils procps
+</pre>
+
+{% comment %}
+To build ruby 3.2.2 on CentOS 7, add: "yum --enablerepo=powertools install libyaml-devel"
+{% endcomment %}
+
+h4. Alma/CentOS/Red Hat/Rocky 8+
<pre>
-dnf install gpg curl which
+dnf install gpg curl which findutils procps
</pre>
h4. Debian and Ubuntu
<pre>
-apt-get --no-install-recommends install gpg curl
+apt-get --no-install-recommends install gpg curl ca-certificates dirmngr procps
</pre>
h3. Install RVM, Ruby and Bundler
<notextile>
-<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
+<pre><code><span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7.7
</span></code></pre></notextile>
-This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the Ruby 2.7.7 release, as well as the @gem@ and @bundle@ commands.
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
<pre><code><span class="userinput">rvm-exec default ruby -v
</span></code></pre></notextile>
-
-h2(#fromsource). Option 3: Install from source
-
-Install prerequisites for Debian 10, Ubuntu 18.04 and Ubuntu 20.04:
-
-<notextile>
-<pre><code><span class="userinput">sudo apt-get install \
- bison build-essential gettext libcurl4 \
- libcurl4-openssl-dev libpcre3-dev libreadline-dev \
- libssl-dev libxslt1.1 zlib1g-dev
-</span></code></pre></notextile>
-
-Install prerequisites for Alma/CentOS/Red Hat/Rocky:
-
-<notextile>
-<pre><code><span class="userinput">sudo dnf install \
- libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel \
- patch readline-devel zlib-devel libffi-devel openssl-devel \
- make automake libtool bison sqlite-devel tar
-</span></code></pre></notextile>
-
-Build and install Ruby:
-
-<notextile>
-<pre><code><span class="userinput">mkdir -p ~/src
-cd ~/src
-curl -f https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz | tar xz
-cd ruby-3.2.2
-./configure --disable-install-static-library --enable-shared --disable-install-doc
-make -j8
-sudo make install
-
-# Make sure the post install script can find the gem and ruby executables
-sudo ln -s /usr/local/bin/gem /usr/bin/gem
-sudo ln -s /usr/local/bin/ruby /usr/bin/ruby
-# Install bundler
-sudo -i gem install bundler --no-document</span>
-</code></pre></notextile>
"previous: Upgrading to 2.7.0":#v2_7_0
+h3. Remove Workbench1 packages after upgrading the salt installer
+
+If you installed a previous version of Arvados with the Salt installer, and you upgrade your installer to upgrade the cluster, you should uninstall the @arvados-workbench@ package from the workbench instance afterwards.
+
h3. Remove Workbench1 packages and configuration
The Workbench1 application has been removed from the Arvados distribution. We recommend the following follow-up steps.
# KEEPSTORE nodes (at least 1 if using S3 as a Keep backend, else 2)
## arvados keepstore (recommendend hostnames @keep0.${DOMAIN}@ and @keep1.${DOMAIN}@)
# WORKBENCH node
-## arvados workbench (recommendend hostname @workbench.${DOMAIN}@)
-## arvados workbench2 (recommendend hostname @workbench2.${DOMAIN}@)
-## arvados webshell (recommendend hostname @webshell.${DOMAIN}@)
-## arvados websocket (recommendend hostname @ws.${DOMAIN}@)
+## arvados legacy workbench URLs (recommendend hostname @workbench.${DOMAIN}@)
+## arvados workbench2 (recommendend hostname @workbench2.${DOMAIN}@)
+## arvados webshell (recommendend hostname @webshell.${DOMAIN}@)
+## arvados websocket (recommendend hostname @ws.${DOMAIN}@)
## arvados cloud dispatcher
## arvados keepbalance
## arvados keepproxy (recommendend hostname @keep.${DOMAIN}@)
MANAGEMENT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
SYSTEM_ROOT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
ANONYMOUS_USER_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-WORKBENCH_SECRET_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
DATABASE_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
</code></pre>
# Set @DATABASE_PASSWORD@ to a random string (unless you "already have a database":#ext-database then you should set it to that database's password)
Most service logs go to @/var/log/syslog@.
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance(s).
Workbench 2 is a client-side Javascript application. If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
If you are using multiple hostnames, determine the base domain for the cluster. This will be referred to as @${DOMAIN}@.
-For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@" means @controller.xarv1.example.com@.
+For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@ means @controller.xarv1.example.com@.
h3. Machine specification
Most service logs go to @/var/log/syslog@.
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance.
Workbench 2 is a client-side Javascript application. If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
</code></pre>
</notextile>
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev python3-llfuse
-</pre>
-
h3. Usage
Please refer to the "Accessing Keep from GNU/Linux":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html tutorial for more information.
github.com/coreos/go-oidc/v3 v3.5.0
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
github.com/creack/pty v1.1.18
- github.com/docker/docker v24.0.5+incompatible
+ github.com/docker/docker v24.0.7+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/fsnotify/fsnotify v1.4.9
github.com/ghodss/yaml v1.0.0
github.com/prometheus/client_model v0.3.0
github.com/prometheus/common v0.39.0
github.com/sirupsen/logrus v1.8.1
- golang.org/x/crypto v0.5.0
- golang.org/x/net v0.9.0
- golang.org/x/oauth2 v0.7.0
- golang.org/x/sys v0.7.0
- google.golang.org/api v0.114.0
+ golang.org/x/crypto v0.16.0
+ golang.org/x/net v0.19.0
+ golang.org/x/oauth2 v0.11.0
+ golang.org/x/sys v0.15.0
+ google.golang.org/api v0.126.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
gopkg.in/square/go-jose.v2 v2.5.1
gopkg.in/src-d/go-billy.v4 v4.0.1
)
require (
- cloud.google.com/go/compute v1.19.1 // indirect
+ cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/gliderlabs/ssh v0.2.2 // indirect
github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
- github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+ github.com/go-jose/go-jose/v3 v3.0.1 // indirect
github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
- github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
- github.com/google/uuid v1.3.0 // indirect
+ github.com/google/s2a-go v0.1.4 // indirect
+ github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
- github.com/googleapis/gax-go/v2 v2.7.1 // indirect
+ github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
- golang.org/x/text v0.9.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
golang.org/x/tools v0.6.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
- google.golang.org/grpc v1.56.1 // indirect
- google.golang.org/protobuf v1.30.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/grpc v1.59.0 // indirect
+ google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
-cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
-cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
+cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
github.com/Azure/azure-sdk-for-go v45.1.0+incompatible h1:kxtaPD8n2z5Za+9e3sKsYG2IX6PG2R6VXtgS7gAbh3A=
github.com/Azure/azure-sdk-for-go v45.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQkfoEA2HqE=
github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 h1:0Di2onNnlN5PAyWPbqlPyN45eOQ+QW/J9eqLynt4IV4=
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092/go.mod h1:8IzBjZCRSnsvM6MJMG8HNNtnzMl48H22rbJL2kRUJ0Y=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw=
github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
-github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
+github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
+github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
-github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
+github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
+github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/satori/go.uuid v1.2.1-0.20180103174451-36e9d2ebbde5 h1:Jw7W4WMfQDxsXvfeFSaS2cHlY7bAF4MGrgnbd0+Uo78=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk=
-golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
-golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
+golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE=
-google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
+google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
+google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
var nopCancelFunc context.CancelFunc = func() {}
+var reqErrorRe = regexp.MustCompile(`net/http: invalid header `)
+
// Do augments (*http.Client)Do(): adds Authorization and X-Request-Id
// headers, delays in order to comply with rate-limiting restrictions,
// and retries failed requests when appropriate.
var lastResp *http.Response
var lastRespBody io.ReadCloser
var lastErr error
+ var checkRetryCalled int
rclient := retryablehttp.NewClient()
rclient.HTTPClient = c.httpClient()
rclient.RetryMax = 0
}
rclient.CheckRetry = func(ctx context.Context, resp *http.Response, respErr error) (bool, error) {
+ checkRetryCalled++
if c.requestLimiter.Report(resp, respErr) {
c.last503.Store(time.Now())
}
if c.Timeout == 0 {
- return false, err
+ return false, nil
+ }
+ if respErr != nil && reqErrorRe.MatchString(respErr.Error()) {
+ return false, nil
}
retrying, err := retryablehttp.DefaultRetryPolicy(ctx, resp, respErr)
if retrying {
}
resp, err := rclient.Do(rreq)
if (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) && (lastResp != nil || lastErr != nil) {
- resp, err = lastResp, lastErr
+ resp = lastResp
+ err = lastErr
+ if checkRetryCalled > 0 && err != nil {
+ // Mimic retryablehttp's "giving up after X
+ // attempts" message, even if we gave up
+ // because of time rather than maxretries.
+ err = fmt.Errorf("%s %s giving up after %d attempt(s): %w", req.Method, req.URL.String(), checkRetryCalled, err)
+ }
if resp != nil {
resp.Body = lastRespBody
}
if scheme == "" {
scheme = "https"
}
- return scheme + "://" + c.APIHost + "/" + path
+ // Double-slash in URLs tend to cause subtle hidden problems
+ // (e.g., they can behave differently when a load balancer is
+ // in the picture). Here we ensure exactly one "/" regardless
+ // of whether the given APIHost or path has a superfluous one.
+ return scheme + "://" + strings.TrimSuffix(c.APIHost, "/") + "/" + strings.TrimPrefix(path, "/")
}
// DiscoveryDocument is the Arvados server's description of itself.
c.Check(s.reqs, check.HasLen, 1)
}
+// as of 0.7.2., retryablehttp does not recognize this as a
+// non-retryable error.
+func (s *clientRetrySuite) TestNonRetryableStdlibError(c *check.C) {
+ s.respStatus <- http.StatusOK
+ req, err := http.NewRequest(http.MethodGet, "https://"+s.client.APIHost+"/test", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Good-Header", "T\033rrible header value")
+ err = s.client.DoAndDecode(&struct{}{}, req)
+ c.Check(err, check.ErrorMatches, `.*after 1 attempt.*net/http: invalid header .*`)
+ if !c.Check(s.reqs, check.HasLen, 0) {
+ c.Logf("%v", s.reqs[0])
+ }
+}
+
func (s *clientRetrySuite) TestNonRetryableAfter503s(c *check.C) {
time.AfterFunc(time.Second, func() { s.respStatus <- http.StatusNotFound })
err := s.client.RequestAndDecode(&struct{}{}, http.MethodGet, "test", nil, nil)
// Load plain token from "Authorization: OAuth2 ..." header
// (typically used by smart API clients)
if toks := strings.SplitN(r.Header.Get("Authorization"), " ", 2); len(toks) == 2 && (toks[0] == "OAuth2" || toks[0] == "Bearer") {
- a.Tokens = append(a.Tokens, toks[1])
+ a.Tokens = append(a.Tokens, strings.TrimSpace(toks[1]))
}
// Load base64-encoded token from "Authorization: Basic ..."
// header (typically used by git via credential helper)
if _, password, ok := r.BasicAuth(); ok {
- a.Tokens = append(a.Tokens, password)
+ a.Tokens = append(a.Tokens, strings.TrimSpace(password))
}
// Load tokens from query string. It's generally not a good
// find/report decoding errors in a suitable way.
qvalues, _ := url.ParseQuery(r.URL.RawQuery)
if val, ok := qvalues["api_token"]; ok {
- a.Tokens = append(a.Tokens, val...)
+ for _, token := range val {
+ a.Tokens = append(a.Tokens, strings.TrimSpace(token))
+ }
}
a.loadTokenFromCookie(r)
if err != nil {
return
}
- a.Tokens = append(a.Tokens, string(token))
+ a.Tokens = append(a.Tokens, strings.TrimSpace(string(token)))
}
// LoadTokensFromHTTPRequestBody loads credentials from the request
return err
}
if t := r.PostFormValue("api_token"); t != "" {
- a.Tokens = append(a.Tokens, t)
+ a.Tokens = append(a.Tokens, strings.TrimSpace(t))
}
return nil
}
import (
"net/http"
"net/http/httptest"
+ "net/url"
+ "strings"
"testing"
check "gopkg.in/check.v1"
func (s *HandlersSuite) TestLoadToken(c *check.C) {
handler := LoadToken(s)
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/foo/bar?api_token=xyzzy", nil))
- c.Assert(s.gotCredentials, check.NotNil)
- c.Assert(s.gotCredentials.Tokens, check.HasLen, 1)
- c.Check(s.gotCredentials.Tokens[0], check.Equals, "xyzzy")
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+
+// Ignore leading and trailing spaces, newlines, etc. in case a user
+// has added them inadvertently during copy/paste.
+func (s *HandlersSuite) TestTrimSpaceInQuery(c *check.C) {
+ handler := LoadToken(s)
+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/foo/bar?api_token=%20xyzzy%0a", nil))
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInPostForm(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("POST", "/foo/bar", strings.NewReader(url.Values{"api_token": []string{"\nxyzzy\n"}}.Encode()))
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInCookie(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("GET", "/foo/bar", nil)
+ req.AddCookie(&http.Cookie{Name: "arvados_api_token", Value: EncodeTokenCookie([]byte("\vxyzzy\n"))})
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInBasicAuth(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("GET", "/foo/bar", nil)
+ req.SetBasicAuth("username", "\txyzzy\n")
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
}
func (s *HandlersSuite) TestRequireLiteralTokenEmpty(c *check.C) {
func (s *HandlersSuite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.served++
s.gotCredentials = CredentialsFromRequest(r)
+ s.gotCredentials.LoadTokensFromHTTPRequestBody(r)
}
public class ExternalConfigProvider implements ConfigProvider {
+ private static final int DEFAULT_CONNECTION_TIMEOUT = 60000;
+ private static final int DEFAULT_READ_TIMEOUT = 60000;
+ private static final int DEFAULT_WRITE_TIMEOUT = 60000;
+
private boolean apiHostInsecure;
private String keepWebHost;
private int keepWebPort;
this.fileSplitDirectory = fileSplitDirectory;
this.numberOfCopies = numberOfCopies;
this.numberOfRetries = numberOfRetries;
- this.connectTimeout = 60000;
- this.readTimeout = 60000;
- this.writeTimeout = 60000;
+ this.connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+ this.readTimeout = DEFAULT_READ_TIMEOUT;
+ this.writeTimeout = DEFAULT_WRITE_TIMEOUT;
}
ExternalConfigProvider(boolean apiHostInsecure, String keepWebHost, int keepWebPort, String apiHost, int apiPort,
private File fileSplitDirectory;
private int numberOfCopies;
private int numberOfRetries;
+ private int connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+ private int readTimeout = DEFAULT_READ_TIMEOUT;
+ private int writeTimeout = DEFAULT_WRITE_TIMEOUT;
ExternalConfigProviderBuilder() {
}
return this;
}
+ public ExternalConfigProvider.ExternalConfigProviderBuilder connectTimeout(int connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ return this;
+ }
+
+ public ExternalConfigProvider.ExternalConfigProviderBuilder readTimeout(int readTimeout) {
+ this.readTimeout = readTimeout;
+ return this;
+ }
+
+ public ExternalConfigProvider.ExternalConfigProviderBuilder writeTimeout(int writeTimeout) {
+ this.writeTimeout = writeTimeout;
+ return this;
+ }
+
public ExternalConfigProvider build() {
- return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries);
+ return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries, connectTimeout, readTimeout, writeTimeout);
}
}
.fileSplitDirectory(CONFIG.getFileSplitDirectory())
.numberOfCopies(CONFIG.getNumberOfCopies())
.numberOfRetries(CONFIG.getNumberOfRetries())
+ .connectTimeout(CONFIG.getConnectTimeout())
+ .readTimeout(CONFIG.getReadTimeout())
+ .writeTimeout(CONFIG.getWriteTimeout())
.build();
}
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados Python SDK
+
+This module provides the entire Python SDK for Arvados. The most useful modules
+include:
+
+* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
+ `arvados.api` to construct a client object.
+
+* arvados.collection - The `arvados.collection.Collection` class provides a
+ high-level interface to read and write collections. It coordinates sending
+ data to and from Keep, and synchronizing updates with the collection object.
+
+* arvados.util - Utility functions to use mostly in conjunction with the API
+ client object and the results it returns.
+
+Other submodules provide lower-level functionality.
+"""
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-import bz2
-import fcntl
-import hashlib
-import http.client
-import httplib2
-import json
import logging as stdliblog
import os
-import pprint
-import re
-import string
import sys
-import time
import types
-import zlib
-if sys.version_info >= (3, 0):
- from collections import UserDict
-else:
- from UserDict import UserDict
+from collections import UserDict
from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
client constructors are `api` and `api_from_config`.
"""
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
import collections
-import http.client
import httplib2
import json
import logging
import time
import types
+from typing import (
+ Any,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+)
+
import apiclient
import apiclient.http
from apiclient import discovery as apiclient_discovery
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type):
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
"""Set up an HTTP file cache
This function constructs and returns an `arvados.cache.SafeHTTPCache`
return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
- version,
- discoveryServiceUrl,
- token,
+ version: str,
+ discoveryServiceUrl: str,
+ token: str,
*,
- cache=True,
- http=None,
- insecure=False,
- num_retries=10,
- request_id=None,
- timeout=5*60,
- **kwargs,
-):
+ cache: bool=True,
+ http: Optional[httplib2.Http]=None,
+ insecure: bool=False,
+ num_retries: int=10,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ **kwargs: Any,
+) -> apiclient_discovery.Resource:
"""Build an Arvados API client
This function returns a `googleapiclient.discovery.Resource` object
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
-
"""
if http is None:
http = httplib2.Http(
return svc
def normalize_api_kwargs(
- version=None,
- discoveryServiceUrl=None,
- host=None,
- token=None,
- **kwargs,
-):
+ version: Optional[str]=None,
+ discoveryServiceUrl: Optional[str]=None,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ **kwargs: Any,
+) -> Dict[str, Any]:
"""Validate kwargs from `api` and build kwargs for `api_client`
This method takes high-level keyword arguments passed to the `api`
**kwargs,
}
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> Dict[str, Any]:
"""Build `api_client` keyword arguments from configuration
This function accepts a mapping with Arvados configuration settings like
**kwargs,
)
-def api(version=None, cache=True, host=None, token=None, insecure=False,
- request_id=None, timeout=5*60, *,
- discoveryServiceUrl=None, **kwargs):
+def api(
+ version: Optional[str]=None,
+ cache: bool=True,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ insecure: bool=False,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ *,
+ discoveryServiceUrl: Optional[str]=None,
+ **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
from .safeapi import ThreadSafeApiCache
return ThreadSafeApiCache({}, {}, kwargs, version)
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
#
# SPDX-License-Identifier: Apache-2.0
-from builtins import range
-from builtins import object
import functools
import inspect
import pycurl
import time
from collections import deque
+from typing import (
+ Callable,
+ Generic,
+ Optional,
+ TypeVar,
+)
import arvados.errors
_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504])
-class RetryLoop(object):
+CT = TypeVar('CT', bound=Callable)
+T = TypeVar('T')
+
+class RetryLoop(Generic[T]):
"""Coordinate limited retries of code.
`RetryLoop` coordinates a loop that runs until it records a
it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- * success_check: Callable --- This is a function that will be called
- each time the loop saves a result. The function should return `True`
- if the result indicates the code succeeded, `False` if it represents a
- permanent failure, and `None` if it represents a temporary failure.
- If no function is provided, the loop will end after any result is
- saved.
+ * success_check: Callable[[T], bool | None] --- This is a function that
+ will be called each time the loop saves a result. The function should
+ return `True` if the result indicates the code succeeded, `False` if
+ it represents a permanent failure, and `None` if it represents a
+ temporary failure. If no function is provided, the loop will end
+ after any result is saved.
* backoff_start: float --- The number of seconds that must pass before
the loop's second iteration. Default 0, which disables all waiting.
* max_wait: float --- Maximum number of seconds to wait between
retries. Default 60.
"""
- def __init__(self, num_retries, success_check=lambda r: True,
- backoff_start=0, backoff_growth=2, save_results=1,
- max_wait=60):
+ def __init__(
+ self,
+ num_retries: int,
+ success_check: Callable[[T], Optional[bool]]=lambda r: True,
+ backoff_start: float=0,
+ backoff_growth: float=2,
+ save_results: int=1,
+ max_wait: float=60
+ ) -> None:
self.tries_left = num_retries + 1
self.check_result = success_check
self.backoff_wait = backoff_start
self._running = None
self._success = None
- def __iter__(self):
+ def __iter__(self) -> 'RetryLoop':
"""Return an iterator of retries."""
return self
- def running(self):
+ def running(self) -> Optional[bool]:
"""Return whether this loop is running.
Returns `None` if the loop has never run, `True` if it is still running,
"""
return self._running and (self._success is None)
- def __next__(self):
+ def __next__(self) -> int:
"""Record a loop attempt.
If the loop is still running, decrements the number of tries left and
self.tries_left -= 1
return self.tries_left
- def save_result(self, result):
+ def save_result(self, result: T) -> None:
"""Record a loop result.
Save the given result, and end the loop if it indicates
Arguments:
- * result: Any --- The result from this loop attempt to check and
- save.
+ * result: T --- The result from this loop attempt to check and save.
"""
if not self.running():
raise arvados.errors.AssertionError(
self._success = self.check_result(result)
self._attempts += 1
- def success(self):
+ def success(self) -> Optional[bool]:
"""Return the loop's end state.
Returns `True` if the loop recorded a successful result, `False` if it
"""
return self._success
- def last_result(self):
+ def last_result(self) -> T:
"""Return the most recent result the loop saved.
Raises `arvados.errors.AssertionError` if called before any result has
raise arvados.errors.AssertionError(
"queried loop results before any were recorded")
- def attempts(self):
+ def attempts(self) -> int:
"""Return the number of results that have been saved.
This count includes all kinds of results: success, permanent failure,
"""
return self._attempts
- def attempts_str(self):
+ def attempts_str(self) -> str:
"""Return a human-friendly string counting saved results.
This method returns '1 attempt' or 'N attempts', where the number
return '{} attempts'.format(self._attempts)
-def check_http_response_success(status_code):
+def check_http_response_success(status_code: int) -> Optional[bool]:
"""Convert a numeric HTTP status code to a loop control flag.
This method takes a numeric HTTP status code and returns `True` if
else:
return None # Get well soon, server.
-def retry_method(orig_func):
+def retry_method(orig_func: CT) -> CT:
"""Provide a default value for a method's num_retries argument.
This is a decorator for instance and class methods that accept a
Arvados API client.
"""
-from __future__ import absolute_import
-
-from builtins import object
import sys
import threading
+from typing import (
+ Any,
+ Mapping,
+ Optional,
+)
+
from . import config
from . import keep
from . import util
Arguments:
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`,
- and optionally `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, uses
`arvados.config.settings` to get these parameters from user
configuration. You can pass an empty mapping to build the client
solely from `api_params`.
- keep_params: Mapping[str, Any]
- : Keyword arguments used to construct an associated
- `arvados.keep.KeepClient`.
+ * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+ an associated `arvados.keep.KeepClient`.
- api_params: Mapping[str, Any]
- : Keyword arguments used to construct each thread's API client. These
- have the same meaning as in the `arvados.api.api` function.
+ * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+ each thread's API client. These have the same meaning as in the
+ `arvados.api.api` function.
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ `'v1'`.
"""
-
- def __init__(self, apiconfig=None, keep_params={}, api_params={}, version=None):
+ def __init__(
+ self,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ keep_params: Optional[Mapping[str, Any]]={},
+ api_params: Optional[Mapping[str, Any]]={},
+ version: Optional[str]=None,
+ ) -> None:
if apiconfig or apiconfig is None:
self._api_kwargs = api.api_kwargs_from_config(version, apiconfig, **api_params)
else:
self.local = threading.local()
self.keep = keep.KeepClient(api_client=self, **keep_params)
- def localapi(self):
+ def localapi(self) -> 'googleapiclient.discovery.Resource':
try:
client = self.local.api
except AttributeError:
self.local.api = client
return client
- def __getattr__(self, name):
+ def __getattr__(self, name: str) -> Any:
# Proxy nonexistent attributes to the thread-local API client.
return getattr(self.localapi(), name)
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados utilities
-from __future__ import division
-from builtins import range
+This module provides functions and constants that are useful across a variety
+of Arvados resource types, or extend the Arvados API client (see `arvados.api`).
+"""
+import errno
import fcntl
import functools
import hashlib
import random
import re
import subprocess
-import errno
import sys
import warnings
import arvados.errors
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ TypeVar,
+ Union,
+)
+
+T = TypeVar('T')
+
HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+"""Regular expression to match a hexadecimal string (case-insensitive)"""
CR_UNCOMMITTED = 'Uncommitted'
+"""Constant `state` value for uncommited container requests"""
CR_COMMITTED = 'Committed'
+"""Constant `state` value for committed container requests"""
CR_FINAL = 'Final'
+"""Constant `state` value for finalized container requests"""
keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+"""Regular expression to match any Keep block locator"""
signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+"""Regular expression to match any Keep block locator with an access token hint"""
portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
+"""Regular expression to match any collection portable data hash"""
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+"""Regular expression to match an Arvados collection manifest text"""
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a file path from a collection identified by portable data hash"""
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a `keep:` URI with a collection identified by portable data hash"""
+
uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+"""Regular expression to match any Arvados object UUID"""
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+"""Regular expression to match any Arvados collection UUID"""
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
+"""Regular expression to match any Arvados container UUID"""
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
-user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados group UUID"""
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+"""Regular expression to match any Arvados link UUID"""
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados user UUID"""
job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
-keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
-keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match any Arvados job UUID
+
+.. WARNING:: Deprecated
+ Arvados job resources are deprecated and will be removed in a future
+ release. Prefer the containers API instead.
+"""
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
If the following arguments are given, they'll be included in the
notices:
- preferred: str | None
- : The name of an alternative that users should use instead.
+ * preferred: str | None --- The name of an alternative that users should
+ use instead.
- version: str | None
- : The version of Arvados when the callable is scheduled to be
- removed.
+ * version: str | None --- The version of Arvados when the callable is
+ scheduled to be removed.
"""
if version is None:
version = ''
return deprecated_wrapper
return deprecated_decorator
+def is_hex(s: str, *length_args: int) -> bool:
+ """Indicate whether a string is a hexadecimal number
+
+ This method returns true if all characters in the string are hexadecimal
+ digits. It is case-insensitive.
+
+ You can also pass optional length arguments to check that the string has
+ the expected number of digits. If you pass one integer, the string must
+ have that length exactly, otherwise the method returns False. If you
+ pass two integers, the string's length must fall within that minimum and
+ maximum (inclusive), otherwise the method returns False.
+
+ Arguments:
+
+ * s: str --- The string to check
+
+ * length_args: int --- Optional length limit(s) for the string to check
+ """
+ num_length_args = len(length_args)
+ if num_length_args > 2:
+ raise arvados.errors.ArgumentError(
+ "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
+ elif num_length_args == 2:
+ good_len = (length_args[0] <= len(s) <= length_args[1])
+ elif num_length_args == 1:
+ good_len = (len(s) == length_args[0])
+ else:
+ good_len = True
+ return bool(good_len and HEX_RE.match(s))
+
+def keyset_list_all(
+ fn: Callable[..., 'arvados.api_resources.ArvadosAPIRequest'],
+ order_key: str="created_at",
+ num_retries: int=0,
+ ascending: bool=True,
+ **kwargs: Any,
+) -> Iterator[Dict[str, Any]]:
+ """Iterate all Arvados resources from an API list call
+
+ This method takes a method that represents an Arvados API list call, and
+ iterates the objects returned by the API server. It can make multiple API
+ calls to retrieve and iterate all objects available from the API server.
+
+ Arguments:
+
+ * fn: Callable[..., arvados.api_resources.ArvadosAPIRequest] --- A
+ function that wraps an Arvados API method that returns a list of
+ objects. If you have an Arvados API client named `arv`, examples
+ include `arv.collections().list` and `arv.groups().contents`. Note
+ that you should pass the function *without* calling it.
+
+ * order_key: str --- The name of the primary object field that objects
+ should be sorted by. This name is used to build an `order` argument
+ for `fn`. Default `'created_at'`.
+
+ * num_retries: int --- This argument is passed through to
+ `arvados.api_resources.ArvadosAPIRequest.execute` for each API call. See
+ that method's docstring for details. Default 0 (meaning API calls will
+ use the `num_retries` value set when the Arvados API client was
+ constructed).
+
+ * ascending: bool --- Used to build an `order` argument for `fn`. If True,
+ all fields will be sorted in `'asc'` (ascending) order. Otherwise, all
+ fields will be sorted in `'desc'` (descending) order.
+
+ Additional keyword arguments will be passed directly to `fn` for each API
+ call. Note that this function sets `count`, `limit`, and `order` as part of
+ its work.
+ """
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ asc = "asc" if ascending else "desc"
+ kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
+ other_filters = kwargs.get("filters", [])
+
+ try:
+ select = set(kwargs['select'])
+ except KeyError:
+ pass
+ else:
+ select.add(order_key)
+ select.add('uuid')
+ kwargs['select'] = list(select)
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+def ca_certs_path(fallback: T=httplib2.CA_CERTS) -> Union[str, T]:
+ """Return the path of the best available source of CA certificates
+
+ This function checks various known paths that provide trusted CA
+ certificates, and returns the first one that exists. It checks:
+
+ * the path in the `SSL_CERT_FILE` environment variable (used by OpenSSL)
+ * `/etc/arvados/ca-certificates.crt`, respected by all Arvados software
+ * `/etc/ssl/certs/ca-certificates.crt`, the default store on Debian-based
+ distributions
+ * `/etc/pki/tls/certs/ca-bundle.crt`, the default store on Red Hat-based
+ distributions
+
+ If none of these paths exist, this function returns the value of `fallback`.
+
+ Arguments:
+
+ * fallback: T --- The value to return if none of the known paths exist.
+ The default value is the certificate store of Mozilla's trusted CAs
+ included with the Python [certifi][] package.
+
+ [certifi]: https://pypi.org/project/certifi/
+ """
+ for ca_certs_path in [
+ # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+ # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+ os.environ.get('SSL_CERT_FILE'),
+ # Arvados specific:
+ '/etc/arvados/ca-certificates.crt',
+ # Debian:
+ '/etc/ssl/certs/ca-certificates.crt',
+ # Red Hat:
+ '/etc/pki/tls/certs/ca-bundle.crt',
+ ]:
+ if ca_certs_path and os.path.exists(ca_certs_path):
+ return ca_certs_path
+ return fallback
+
+def new_request_id() -> str:
+ """Return a random request ID
+
+ This function generates and returns a random string suitable for use as a
+ `X-Request-Id` header value in the Arvados API.
+ """
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
+
+def get_config_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's configuration, with caching
+
+ This function gets and returns the Arvados configuration from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster configuration.
+ """
+ if not svc._rootDesc.get('resources').get('configs', False):
+ # Old API server version, no config export endpoint
+ return {}
+ if not hasattr(svc, '_cached_config'):
+ svc._cached_config = svc.configs().get().execute()
+ return svc._cached_config
+
+def get_vocabulary_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's vocabulary, with caching
+
+ This function gets and returns the Arvados vocabulary from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ .. HINT:: Low-level method
+ This is a relatively low-level wrapper around the Arvados API. Most
+ users will prefer to use `arvados.vocabulary.load_vocabulary`.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster vocabulary.
+ """
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
+
+def trim_name(collectionname: str) -> str:
+ """Limit the length of a name to fit within Arvados API limits
+
+ This function ensures that a string is short enough to use as an object
+ name in the Arvados API, leaving room for text that may be added by the
+ `ensure_unique_name` argument. If the source name is short enough, it is
+ returned unchanged. Otherwise, this function returns a string with excess
+ characters removed from the middle of the source string and replaced with
+ an ellipsis.
+
+ Arguments:
+
+ * collectionname: str --- The desired source name
+ """
+ max_name_len = 254 - 28
+
+ if len(collectionname) > max_name_len:
+ over = len(collectionname) - max_name_len
+ split = int(max_name_len/2)
+ collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
+
+ return collectionname
+
+@_deprecated('3.0', 'arvados.util.keyset_list_all')
+def list_all(fn, num_retries=0, **kwargs):
+ # Default limit to (effectively) api server's MAX_LIMIT
+ kwargs.setdefault('limit', sys.maxsize)
+ items = []
+ offset = 0
+ items_available = sys.maxsize
+ while len(items) < items_available:
+ c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
+ items += c['items']
+ items_available = c['items_available']
+ offset = c['offset'] + len(c['items'])
+ return items
+
@_deprecated('3.0')
def clear_tmpdir(path=None):
"""
else:
allfiles += [ent_base]
return allfiles
-
-def is_hex(s, *length_args):
- """is_hex(s[, length[, max_length]]) -> boolean
-
- Return True if s is a string of hexadecimal digits.
- If one length argument is given, the string must contain exactly
- that number of digits.
- If two length arguments are given, the string must contain a number of
- digits between those two lengths, inclusive.
- Return False otherwise.
- """
- num_length_args = len(length_args)
- if num_length_args > 2:
- raise arvados.errors.ArgumentError(
- "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
- elif num_length_args == 2:
- good_len = (length_args[0] <= len(s) <= length_args[1])
- elif num_length_args == 1:
- good_len = (len(s) == length_args[0])
- else:
- good_len = True
- return bool(good_len and HEX_RE.match(s))
-
-@_deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
- # Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxsize)
- items = []
- offset = 0
- items_available = sys.maxsize
- while len(items) < items_available:
- c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
- items += c['items']
- items_available = c['items_available']
- offset = c['offset'] + len(c['items'])
- return items
-
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
- pagesize = 1000
- kwargs["limit"] = pagesize
- kwargs["count"] = 'none'
- asc = "asc" if ascending else "desc"
- kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
- other_filters = kwargs.get("filters", [])
-
- try:
- select = set(kwargs['select'])
- except KeyError:
- pass
- else:
- select.add(order_key)
- select.add('uuid')
- kwargs['select'] = list(select)
-
- nextpage = []
- tot = 0
- expect_full_page = True
- seen_prevpage = set()
- seen_thispage = set()
- lastitem = None
- prev_page_all_same_order_key = False
-
- while True:
- kwargs["filters"] = nextpage+other_filters
- items = fn(**kwargs).execute(num_retries=num_retries)
-
- if len(items["items"]) == 0:
- if prev_page_all_same_order_key:
- nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
- prev_page_all_same_order_key = False
- continue
- else:
- return
-
- seen_prevpage = seen_thispage
- seen_thispage = set()
-
- for i in items["items"]:
- # In cases where there's more than one record with the
- # same order key, the result could include records we
- # already saw in the last page. Skip them.
- if i["uuid"] in seen_prevpage:
- continue
- seen_thispage.add(i["uuid"])
- yield i
-
- firstitem = items["items"][0]
- lastitem = items["items"][-1]
-
- if firstitem[order_key] == lastitem[order_key]:
- # Got a page where every item has the same order key.
- # Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
- prev_page_all_same_order_key = True
- else:
- # Start from the last order key seen, but skip the last
- # known uuid to avoid retrieving the same row twice. If
- # there are multiple rows with the same order key it is
- # still likely we'll end up retrieving duplicate rows.
- # That's handled by tracking the "seen" rows for each page
- # so they can be skipped if they show up on the next page.
- nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
- prev_page_all_same_order_key = False
-
-def ca_certs_path(fallback=httplib2.CA_CERTS):
- """Return the path of the best available CA certs source.
-
- This function searches for various distribution sources of CA
- certificates, and returns the first it finds. If it doesn't find any,
- it returns the value of `fallback` (httplib2's CA certs by default).
- """
- for ca_certs_path in [
- # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
- # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
- os.environ.get('SSL_CERT_FILE'),
- # Arvados specific:
- '/etc/arvados/ca-certificates.crt',
- # Debian:
- '/etc/ssl/certs/ca-certificates.crt',
- # Red Hat:
- '/etc/pki/tls/certs/ca-bundle.crt',
- ]:
- if ca_certs_path and os.path.exists(ca_certs_path):
- return ca_certs_path
- return fallback
-
-def new_request_id():
- rid = "req-"
- # 2**104 > 36**20 > 2**103
- n = random.getrandbits(104)
- for _ in range(20):
- c = n % 36
- if c < 10:
- rid += chr(c+ord('0'))
- else:
- rid += chr(c+ord('a')-10)
- n = n // 36
- return rid
-
-def get_config_once(svc):
- if not svc._rootDesc.get('resources').get('configs', False):
- # Old API server version, no config export endpoint
- return {}
- if not hasattr(svc, '_cached_config'):
- svc._cached_config = svc.configs().get().execute()
- return svc._cached_config
-
-def get_vocabulary_once(svc):
- if not svc._rootDesc.get('resources').get('vocabularies', False):
- # Old API server version, no vocabulary export endpoint
- return {}
- if not hasattr(svc, '_cached_vocabulary'):
- svc._cached_vocabulary = svc.vocabularies().get().execute()
- return svc._cached_vocabulary
-
-def trim_name(collectionname):
- """
- trim_name takes a record name (collection name, project name, etc)
- and trims it to fit the 255 character name limit, with additional
- space for the timestamp added by ensure_unique_name, by removing
- excess characters from the middle and inserting an ellipse
- """
-
- max_name_len = 254 - 28
-
- if len(collectionname) > max_name_len:
- over = len(collectionname) - max_name_len
- split = int(max_name_len/2)
- collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
-
- return collectionname
#
# SPDX-License-Identifier: Apache-2.0
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
import argparse
import atexit
import errno
import shutil
import signal
import socket
-import string
import subprocess
import sys
import tempfile
import unittest
import yaml
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
+from urllib.parse import urlparse
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
if __name__ == '__main__' and os.path.exists(
before_destroy :ensure_owner_uuid_is_permitted
before_destroy :ensure_permission_to_destroy
before_create :update_modified_by_fields
+ before_create :add_uuid_to_name, :if => Proc.new { @_add_uuid_to_name }
before_update :maybe_update_modified_by_fields
after_create :log_create
after_update :log_update
end
def save_with_unique_name!
- uuid_was = uuid
- name_was = name
max_retries = 2
transaction do
conn = ActiveRecord::Base.connection
conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- if new_name == name
- # If the database is fast enough to do two attempts in the
- # same millisecond, we need to wait to ensure we try a
- # different timestamp on each attempt.
- sleep 0.002
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- end
-
- self[:name] = new_name
- if uuid_was.nil? && !uuid.nil?
+ if uuid_was.nil?
+ # new record, the uuid caused a name collision (very
+ # unlikely but possible), so generate new uuid
self[:uuid] = nil
if self.is_a? Collection
- # Reset so that is assigned to the new UUID
+ # Also needs to be reset
self[:current_version_uuid] = nil
end
+ # need to adjust the name after the uuid has been generated
+ add_uuid_to_make_unique_name
+ else
+ # existing record, just update the name directly.
+ add_uuid_to_name
end
-
retry
end
end
*ft[:param_out])
end
+ @_add_uuid_to_name = false
+ def add_uuid_to_make_unique_name
+ @_add_uuid_to_name = true
+ end
+
+ def add_uuid_to_name
+ # Incorporate the random part of the UUID into the name. This
+ # lets us prevent name collision but the part we add to the name
+ # is still somewhat meaningful (instead of generating a second
+ # random meaningless string).
+ #
+ # Because ArvadosModel is an abstract class and assign_uuid is
+ # part of HasUuid (which is included by the other concrete
+ # classes) the assign_uuid hook gets added (and run) after this
+ # one. So we need to call assign_uuid here to make sure we have a
+ # uuid.
+ assign_uuid
+ self.name = "#{self.name[0..236]} (#{self.uuid[-15..-1]})"
+ end
+
protected
def self.deep_sort_hash(x)
before_update :apply_max_overlapping_permissions
before_create :apply_max_overlapping_permissions
after_update :delete_overlapping_permissions
- after_update :call_update_permissions
- after_create :call_update_permissions
+ after_update :call_update_permissions, :if => Proc.new { @need_update_permissions }
+ after_create :call_update_permissions, :if => Proc.new { @need_update_permissions }
before_destroy :clear_permissions
after_destroy :delete_overlapping_permissions
after_destroy :check_permissions
+ before_save :check_need_update_permissions
api_accessible :user, extend: :common do |t|
t.add :tail_uuid
'can_manage' => 3,
}
+ def check_need_update_permissions
+ @need_update_permissions = self.link_class == 'permission' && (name != name_was || new_record?)
+ end
+
def call_update_permissions
- if self.link_class == 'permission'
update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
current_user.forget_cached_group_perms
- end
end
def clear_permissions
update!(redirect_to_user_uuid: new_user.uuid, username: nil)
end
skip_check_permissions_against_full_refresh do
- update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM
- update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM
- update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM
+ update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
end
- update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM, nil, true
end
end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-module SimulateJobLog
- include CurrentApiClient
- def replay(filename, multiplier = 1, simulated_job_uuid = nil)
- raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
-
- multiplier = multiplier.to_f
- multiplier = 1.0 if multiplier <= 0
-
- actual_start_time = Time.now
- log_start_time = nil
-
- if simulated_job_uuid and (job = Job.where(uuid: simulated_job_uuid).first)
- job_owner_uuid = job.owner_uuid
- else
- job_owner_uuid = system_user_uuid
- end
-
- act_as_system_user do
- File.open(filename).each.with_index do |line, index|
- cols = {}
- cols[:timestamp], rest_of_line = line.split(' ', 2)
- begin
- cols[:timestamp] = Time.strptime( cols[:timestamp], "%Y-%m-%d_%H:%M:%S" )
- rescue ArgumentError
- if line =~ /^((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (.*)/
- # Wed Nov 19 07:12:39 2014
- cols[:timestamp] = Time.strptime( $1, "%a %b %d %H:%M:%S %Y" )
- rest_of_line = $2
- else
- STDERR.puts "Ignoring log line because of unknown time format: #{line}"
- end
- end
- cols[:job_uuid], cols[:pid], cols[:task], cols[:event_type], cols[:message] = rest_of_line.split(' ', 5)
- # Override job uuid with a simulated one if specified
- cols[:job_uuid] = simulated_job_uuid || cols[:job_uuid]
- # determine when we want to simulate this log being created, based on the time multiplier
- log_start_time = cols[:timestamp] if log_start_time.nil?
- log_time = cols[:timestamp]
- actual_elapsed_time = Time.now - actual_start_time
- log_elapsed_time = log_time - log_start_time
- modified_elapsed_time = log_elapsed_time / multiplier
- pause_time = modified_elapsed_time - actual_elapsed_time
- sleep pause_time if pause_time > 0
-
- Log.new({
- owner_uuid: job_owner_uuid,
- event_at: Time.zone.local_to_utc(cols[:timestamp]),
- object_uuid: cols[:job_uuid],
- event_type: cols[:event_type],
- properties: { 'text' => line }
- }).save!
- end
- end
-
- end
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'simulate_job_log'
-desc 'Simulate job logging from a file. Three arguments: log filename, time multipler (optional), simulated job uuid (optional). E.g. (use quotation marks if using spaces between args): rake "replay_job_log[log.txt, 2.0, qr1hi-8i9sb-nf3qk0xzwwz3lre]"'
-task :replay_job_log, [:filename, :multiplier, :uuid] => :environment do |t, args|
- include SimulateJobLog
- abort("No filename specified.") if args[:filename].blank?
- replay( args[:filename], args[:multiplier].to_f, args[:uuid] )
-end
REVOKE_PERM = 0
CAN_MANAGE_PERM = 3
-def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil, update_all_users=false
return if Thread.current[:suppress_update_permissions]
#
# Disable merge join for just this query (also local for this transaction), then reenable it.
ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
+ if perm_origin_uuid[5..11] == '-tpzed-' && !update_all_users
+ # Modifying permission granted to a user, recompute the all permissions for that user
+
+ ActiveRecord::Base.connection.exec_query %{
+with origin_user_perms as (
+ select pq.origin_uuid as user_uuid, target_uuid, pq.val, pq.traverse_owned from (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select '#{perm_origin_uuid}'::varchar(255), '#{perm_origin_uuid}'::varchar(255), 3, true, true
+ where exists (select uuid from users where uuid='#{perm_origin_uuid}')
+},
+:edge_perm => %{
+case (edges.edge_id = '#{edge_id}')
+ when true then #{perm_level}
+ else edges.val
+ end
+}
+} }) as pq),
+
+/*
+ Because users always have permission on themselves, this
+ query also makes sure those permission rows are always
+ returned.
+*/
+temptable_perms as (
+ select * from origin_user_perms
+ union all
+ select target_uuid as user_uuid, target_uuid, 3, true
+ from origin_user_perms
+ where origin_user_perms.target_uuid like '_____-tpzed-_______________' and
+ origin_user_perms.target_uuid != '#{perm_origin_uuid}'
+),
+
+/*
+ Now that we have recomputed a set of permissions, delete any
+ rows from the materialized_permissions table where (target_uuid,
+ user_uuid) is not present or has perm_level=0 in the recomputed
+ set.
+*/
+delete_rows as (
+ delete from #{PERMISSION_VIEW} where
+ user_uuid='#{perm_origin_uuid}' and
+ not exists (select 1 from temptable_perms
+ where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+ user_uuid='#{perm_origin_uuid}' and
+ val>0)
+)
+
+/*
+ Now insert-or-update permissions in the recomputed set. The
+ WHERE clause is important to avoid redundantly updating rows
+ that haven't actually changed.
+*/
+insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
+ select user_uuid, target_uuid, val as perm_level, traverse_owned from temptable_perms where val>0
+on conflict (user_uuid, target_uuid) do update
+set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned
+where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
+ #{PERMISSION_VIEW}.target_uuid=EXCLUDED.target_uuid and
+ (#{PERMISSION_VIEW}.perm_level != EXCLUDED.perm_level or
+ #{PERMISSION_VIEW}.traverse_owned != EXCLUDED.traverse_owned);
+
+}
+ else
+ # Modifying permission granted to a group, recompute permissions for everything accessible through that group
ActiveRecord::Base.connection.exec_query %{
with temptable_perms as (
select * from compute_permission_subgraph($1, $2, $3, $4)),
starting_uuid,
perm_level,
edge_id]
+ end
if perm_level>0
check_permissions_against_full_refresh
ensure_unique_name: true
}
assert_response :success
- assert_match /^owned_by_active \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
end
assert_equal false, json_response['is_trashed']
assert_nil json_response['trash_at']
assert_nil json_response['delete_at']
- assert_match /^same name for trashed and persisted collections \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test 'cannot show collection in trashed subproject' do
assert_not_equal(new_project['uuid'],
groups(:aproject).uuid,
"create returned same uuid as existing project")
- assert_match(/^A Project \(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\)$/,
+ assert_match(/^A Project \(#{new_project['uuid'][-15..-1]}\)$/,
new_project['name'])
end
ensure_unique_name: true
}
assert_response :success
- assert_match /^trashed subproject 3 \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^trashed subproject 3 \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test "move trashed subproject to new owner #{auth}" do
assert_equal ContainerRequest::Final, cr.state
output_coll = Collection.find_by_uuid(cr.output_uuid)
# Make sure the resulting output collection name include the original name
- # plus the date
+ # plus the last 15 characters of uuid
assert_not_equal output_name, output_coll.name,
"more than one collection with the same owner and name"
assert output_coll.name.include?(output_name),
"New name should include original name"
- assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
- "New name should include ISO8601 date"
+ assert_match /#{output_coll.uuid[-15..-1]}/, output_coll.name,
+ "New name should include last 15 characters of uuid"
end
[[0, :check_output_ttl_0],
// using the new fs).
sess.inuse.Lock()
if !sess.userLoaded || refresh {
- err := sess.client.RequestAndDecode(&sess.user, "GET", "/arvados/v1/users/current", nil, nil)
+ err := sess.client.RequestAndDecode(&sess.user, "GET", "arvados/v1/users/current", nil, nil)
if he := errorWithHTTPStatus(nil); errors.As(err, &he) && he.HTTPStatus() == http.StatusForbidden {
// token is OK, but "get user id" api is out
// of scope -- use existing/expired info if
s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
}
func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
- r.Header.Add("Authorization", "Bearer "+tok)
+ r.Header.Add("Authorization", "OAuth2 "+tok)
return http.StatusUnauthorized
}
+
func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
s.doVhostRequests(c, authzViaAuthzHeaderBearer)
}
return http.StatusUnauthorized
}
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
+ s.doVhostRequests(c, authzViaHTTPBasicAuth)
+}
+func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(tok)),
+ })
+ return http.StatusUnauthorized
+}
+
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
+ s.doVhostRequests(c, func(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
+ })
+ return http.StatusUnauthorized
+ })
+}
+
func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
s.doVhostRequests(c, authzViaPath)
}
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
events:
worker_connections: {{ max_reqs * 3 + 1 }}
- ### SNIPPETS
- snippets:
- # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
- ssl_hardening_default.conf:
- - ssl_session_timeout: 1d
- - ssl_session_cache: 'shared:arvadosSSL:10m'
- - ssl_session_tickets: 'off'
-
- # intermediate configuration
- - ssl_protocols: TLSv1.2 TLSv1.3
- - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
- - ssl_prefer_server_ciphers: 'off'
-
- # HSTS (ngx_http_headers_module is required) (63072000 seconds)
- - add_header: 'Strict-Transport-Security "max-age=63072000" always'
-
- # OCSP stapling
- - ssl_stapling: 'on'
- - ssl_stapling_verify: 'on'
-
- # verify chain of trust of OCSP response using Root CA and Intermediate certs
- # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
-
- # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
- # - ssl_dhparam: /path/to/dhparam
-
- # replace with the IP address of your resolver
- # - resolver: 127.0.0.1
-
### SITES
servers:
managed:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SNIPPETS
+ snippets:
+ # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+ ssl_hardening_default.conf:
+ - ssl_session_timeout: 1d
+ - ssl_session_cache: 'shared:arvadosSSL:10m'
+ - ssl_session_tickets: 'off'
+
+ # intermediate configuration
+ - ssl_protocols: TLSv1.2 TLSv1.3
+ - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ - ssl_prefer_server_ciphers: 'off'
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+ # OCSP stapling
+ - ssl_stapling: 'on'
+ - ssl_stapling_verify: 'on'
+
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+ # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+ # - ssl_dhparam: /path/to/dhparam
+
+ # replace with the IP address of your resolver
+ # - resolver: 127.0.0.1
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
;;
generate-tokens)
- for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
+ for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN DATABASE_PASSWORD; do
echo ${i}=$(
tr -dc A-Za-z0-9 </dev/urandom | head -c 32
echo ''
MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters
SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters
ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters
-WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters
DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value
LE_AWS_ACCESS_KEY_ID="FIXME"
s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
- s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g;
s#__SSL_KEY_ENCRYPTED__#${SSL_KEY_ENCRYPTED}#g;
s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION:-}#g;
s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g;
# If we want to try a specific branch of the formula
if [[ ! -z "${BRANCH:-}" && "x${BRANCH}" != "xmain" ]]; then
- ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
+ ( cd ${F_DIR}/arvados && git fetch && git checkout --quiet "${BRANCH}" || git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
elif [ "x${ARVADOS_TAG:-}" != "x" ]; then
( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
fi
echo " - postgres" >> ${STATES_TOP}
echo " - logrotate" >> ${STATES_TOP}
echo " - docker.software" >> ${STATES_TOP}
- echo " - arvados" >> ${STATES_TOP}
+ echo " - arvados.repo" >> ${STATES_TOP}
+ echo " - arvados.config" >> ${STATES_TOP}
+ echo " - arvados.ruby" >> ${STATES_TOP}
+ echo " - arvados.api" >> ${STATES_TOP}
+ echo " - arvados.controller" >> ${STATES_TOP}
+ echo " - arvados.keepstore" >> ${STATES_TOP}
+ echo " - arvados.websocket" >> ${STATES_TOP}
+ echo " - arvados.keepweb" >> ${STATES_TOP}
+ echo " - arvados.workbench2" >> ${STATES_TOP}
+ echo " - arvados.keepproxy" >> ${STATES_TOP}
+ echo " - arvados.shell" >> ${STATES_TOP}
+ echo " - arvados.dispatcher" >> ${STATES_TOP}
echo " - extra.shell_sudo_passwordless" >> ${STATES_TOP}
echo " - extra.shell_cron_add_login_sync" >> ${STATES_TOP}
echo " - extra.passenger_rvm" >> ${STATES_TOP}
for SVC in grafana prometheus; do
grep -q "nginx_${SVC}_configuration" ${PILLARS_TOP} || echo " - nginx_${SVC}_configuration" >> ${PILLARS_TOP}
done
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
if [ "${SSL_MODE}" = "lets-encrypt" ]; then
grep -q "letsencrypt" ${PILLARS_TOP} || echo " - letsencrypt" >> ${PILLARS_TOP}
for SVC in grafana prometheus; do
grep -q "aws_credentials" ${PILLARS_TOP} || echo " - aws_credentials" >> ${PILLARS_TOP}
grep -q "postgresql" ${PILLARS_TOP} || echo " - postgresql" >> ${PILLARS_TOP}
grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP}
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
grep -q "nginx_api_configuration" ${PILLARS_TOP} || echo " - nginx_api_configuration" >> ${PILLARS_TOP}
grep -q "nginx_controller_configuration" ${PILLARS_TOP} || echo " - nginx_controller_configuration" >> ${PILLARS_TOP}
;;
"websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
### States ###
- if [ "${R}" = "workbench" ]; then
- grep -q " - logrotate" ${STATES_TOP} || echo " - logrotate" >> ${STATES_TOP}
- NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
- if grep -q " - nginx$" ${STATES_TOP}; then
- sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${STATES_TOP}
- else
- echo " - nginx.passenger" >> ${STATES_TOP}
- fi
- else
- grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP}
- fi
+ grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP}
if [ "${SSL_MODE}" = "lets-encrypt" ]; then
if [ "x${USE_LETSENCRYPT_ROUTE53:-}" = "xyes" ]; then
fi
# webshell role is just a nginx vhost, so it has no state
- if [ "${R}" != "webshell" ]; then
+ # workbench role is deprecated since 2.7.0
+ if [[ "${R}" != "webshell" && "${R}" != "workbench" ]]; then
grep -q "arvados.${R}" ${STATES_TOP} || echo " - arvados.${R}" >> ${STATES_TOP}
fi
### Pillars ###
- if [ "${R}" = "workbench" ]; then
- grep -q "logrotate_wb1" ${PILLARS_TOP} || echo " - logrotate_wb1" >> ${PILLARS_TOP}
- fi
- grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP}
grep -q "nginx_${R}_configuration" ${PILLARS_TOP} || echo " - nginx_${R}_configuration" >> ${PILLARS_TOP}
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
grep -q "nginx_download_configuration" ${PILLARS_TOP} || echo " - nginx_download_configuration" >> ${PILLARS_TOP}
projects / arvados.git / commitdiff