# SPDX-License-Identifier: AGPL-3.0
SHELL := '/bin/bash'
-all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
+all: centos7/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -f -rlt centos7/generated common-generated/*
+all: debian10/generated
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
cp -f -rlt debian10/generated common-generated/*
+all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
cp -f -rlt debian11/generated common-generated/*
+all: debian12/generated
+debian12/generated: common-generated-all
+ test -d debian12/generated || mkdir debian12/generated
+ cp -f -rlt debian12/generated common-generated/*
+
+all: rocky8/generated
rocky8/generated: common-generated-all
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
+all: ubuntu1804/generated
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -f -rlt ubuntu1804/generated common-generated/*
+all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
+all: ubuntu2204/generated
+ubuntu2204/generated: common-generated-all
+ test -d ubuntu2204/generated || mkdir ubuntu2204/generated
+ cp -f -rlt ubuntu2204/generated common-generated/*
+
GOTARBALL_=DOES_NOT_EXIST
NODETARBALL_=DOES_NOT_EXIST
GOVERSION=$(shell grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $$2}')
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN cd /usr/local/src \
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ARG HOSTTYPE
+ARG BRANCH
+ARG GOVERSION
+
+## dont use debian:12 here since the word 'bookworm' is used for rvm precompiled binaries
+FROM debian:bookworm as build_x86_64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64
+
+FROM debian:bookworm as build_aarch64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+RUN echo HOSTTYPE ${HOSTTYPE}
+
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+SHELL ["/bin/bash", "-c"]
+# Install dependencies.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-virtualenv python3-dev libpam-dev equivs
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
+
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs 8"
+
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian12"]
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ARG HOSTTYPE
+ARG BRANCH
+ARG GOVERSION
+
+FROM ubuntu:jammy as build_x86_64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64, and that requires some sources.list mangling
+ONBUILD RUN /bin/sed -i 's/deb http/deb [ arch=amd64 ] http/' /etc/apt/sources.list
+ONBUILD ADD ports.list /etc/apt/sources.list.d/
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64
+
+FROM ubuntu:jammy as build_aarch64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
+LABEL org.opencontainers.image.authors="Arvados Package Maintainers <packaging@arvados.org>"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+SHELL ["/bin/bash", "-c"]
+# Install dependencies.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
+
+# Install virtualenv
+RUN /usr/bin/pip3 install 'virtualenv<20'
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
+ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
+
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs 8"
+
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados && \
+ go mod download
+
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2204"]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse
+deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
-
+all: centos7/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -f -rlt centos7/generated common-generated/*
+all: debian10/generated
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
cp -f -rlt debian10/generated common-generated/*
+all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
cp -f -rlt debian11/generated common-generated/*
+all: debian12/generated
+debian12/generated: common-generated-all
+ test -d debian12/generated || mkdir debian12/generated
+ cp -f -rlt debian12/generated common-generated/*
+
+all: rocky8/generated
rocky8/generated: common-generated-all
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
+all: ubuntu1804/generated
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -f -rlt ubuntu1804/generated common-generated/*
+all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
+all: ubuntu2204/generated
+ubuntu2204/generated: common-generated-all
+ test -d ubuntu2204/generated || mkdir ubuntu2204/generated
+ cp -f -rlt ubuntu2204/generated common-generated/*
+
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM debian:bookworm
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies
+RUN apt-get update && \
+ apt-get -y install --no-install-recommends curl ca-certificates gpg procps gpg-agent
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ echo "gem: --no-document" >> /etc/gemrc && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian12/ /" >>/etc/apt/sources.list
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM ubuntu:jammy
+LABEL org.opencontainers.image.authors="Arvados Package Maintainers <packaging@arvados.org>"
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies
+RUN apt-get update && \
+ apt-get -y install --no-install-recommends curl ca-certificates gnupg2
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+ gpg --import --no-tty /tmp/pkuczynski.asc && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu2204/ /" >>/etc/apt/sources.list
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
declare -A dd
dd[debian10]=buster
dd[debian11]=bullseye
+ dd[debian12]=bookworm
dd[ubuntu1804]=bionic
dd[ubuntu2004]=focal
+ dd[ubuntu2204]=jammy
D=${dd[$TARGET]}
if [ ${pkgname:0:3} = "lib" ]; then
repo_subdir=${pkgname:0:4}
rm -rf dist/*
- # Get the latest setuptools
- if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ # Get the latest setuptools.
+ #
+ # Note "pip3 install setuptools" fails on debian12 ("error:
+ # externally-managed-environment") even if that requirement is
+ # already satisfied, so we parse "pip3 list" output instead to check
+ # whether we need to do anything.
+ if [[ "$($pip list | grep -P -o '^setuptools\s+\K[0-9]+')" -ge 66 ]]; then
+ : # OK, already installed
+ elif ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'; then
echo "Error, unable to upgrade setuptools with"
- echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'"
exit 1
fi
# filter a useless warning (when building the cwltest package) from the stderr output
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.6 or newer is required.
+Ruby 2.7 or newer is required.
* "Option 1: Install from packages":#packages
* "Option 2: Install with RVM":#rvm
-* "Option 3: Install from source":#fromsource
h2(#packages). Option 1: Install from packages
-{% include 'notebox_begin' %}
-Future versions of Arvados may require a newer version of Ruby than is packaged with your OS. Using OS packages simplifies initial install, but may complicate upgrades that rely on a newer Ruby. If this is a concern, we recommend using "RVM":#rvm.
-{% include 'notebox_end' %}
-
h3. Alma/CentOS/Red Hat/Rocky
-The Ruby version shipped with version 7 of these distributions is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+Version 7 of these distributions does not provide a new enough Ruby version. Use "RVM":#rvm to install Ruby 2.7 or newer.
-Ruby 2.7 is available in a module with version 8 of these distributions. You can install it by running:
+Version 8 of these distributions provides Ruby 2.7. You can install it by running:
<notextile>
<pre><code># <span class="userinput">dnf module enable ruby:2.7</span>
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install Ruby 2.7 or newer.
Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
h2(#rvm). Option 2: Install with RVM
+{% include 'notebox_begin_warning' %}
+We do not recommend using RVM unless the Ruby version provided by your OS distribution is older than 2.7.
+{% include 'notebox_end' %}
+
h3. Install gpg and curl
-h4. Alma/CentOS/Red Hat/Rocky
+h4. CentOS/Red Hat 7
+
+<pre>
+yum install gpg curl which findutils procps
+</pre>
+
+{% comment %}
+To build ruby 3.2.2 on CentOS 7, add: "yum --enablerepo=powertools install libyaml-devel"
+{% endcomment %}
+
+h4. Alma/CentOS/Red Hat/Rocky 8+
<pre>
-dnf install gpg curl which
+dnf install gpg curl which findutils procps
</pre>
h4. Debian and Ubuntu
<pre>
-apt-get --no-install-recommends install gpg curl
+apt-get --no-install-recommends install gpg curl ca-certificates dirmngr procps
</pre>
h3. Install RVM, Ruby and Bundler
<notextile>
-<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
+<pre><code><span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7.7
</span></code></pre></notextile>
-This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the Ruby 2.7.7 release, as well as the @gem@ and @bundle@ commands.
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
<pre><code><span class="userinput">rvm-exec default ruby -v
</span></code></pre></notextile>
-
-h2(#fromsource). Option 3: Install from source
-
-Install prerequisites for Debian 10, Ubuntu 18.04 and Ubuntu 20.04:
-
-<notextile>
-<pre><code><span class="userinput">sudo apt-get install \
- bison build-essential gettext libcurl4 \
- libcurl4-openssl-dev libpcre3-dev libreadline-dev \
- libssl-dev libxslt1.1 zlib1g-dev
-</span></code></pre></notextile>
-
-Install prerequisites for Alma/CentOS/Red Hat/Rocky:
-
-<notextile>
-<pre><code><span class="userinput">sudo dnf install \
- libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel \
- patch readline-devel zlib-devel libffi-devel openssl-devel \
- make automake libtool bison sqlite-devel tar
-</span></code></pre></notextile>
-
-Build and install Ruby:
-
-<notextile>
-<pre><code><span class="userinput">mkdir -p ~/src
-cd ~/src
-curl -f https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz | tar xz
-cd ruby-3.2.2
-./configure --disable-install-static-library --enable-shared --disable-install-doc
-make -j8
-sudo make install
-
-# Make sure the post install script can find the gem and ruby executables
-sudo ln -s /usr/local/bin/gem /usr/bin/gem
-sudo ln -s /usr/local/bin/ruby /usr/bin/ruby
-# Install bundler
-sudo -i gem install bundler --no-document</span>
-</code></pre></notextile>
"previous: Upgrading to 2.7.0":#v2_7_0
+h3. Remove Workbench1 packages after upgrading the salt installer
+
+If you installed a previous version of Arvados with the Salt installer, and you upgrade your installer to upgrade the cluster, you should uninstall the @arvados-workbench@ package from the workbench instance afterwards.
+
h3. Remove Workbench1 packages and configuration
The Workbench1 application has been removed from the Arvados distribution. We recommend the following follow-up steps.
# KEEPSTORE nodes (at least 1 if using S3 as a Keep backend, else 2)
## arvados keepstore (recommendend hostnames @keep0.${DOMAIN}@ and @keep1.${DOMAIN}@)
# WORKBENCH node
-## arvados workbench (recommendend hostname @workbench.${DOMAIN}@)
-## arvados workbench2 (recommendend hostname @workbench2.${DOMAIN}@)
-## arvados webshell (recommendend hostname @webshell.${DOMAIN}@)
-## arvados websocket (recommendend hostname @ws.${DOMAIN}@)
+## arvados legacy workbench URLs (recommendend hostname @workbench.${DOMAIN}@)
+## arvados workbench2 (recommendend hostname @workbench2.${DOMAIN}@)
+## arvados webshell (recommendend hostname @webshell.${DOMAIN}@)
+## arvados websocket (recommendend hostname @ws.${DOMAIN}@)
## arvados cloud dispatcher
## arvados keepbalance
## arvados keepproxy (recommendend hostname @keep.${DOMAIN}@)
MANAGEMENT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
SYSTEM_ROOT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
ANONYMOUS_USER_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-WORKBENCH_SECRET_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
DATABASE_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
</code></pre>
# Set @DATABASE_PASSWORD@ to a random string (unless you "already have a database":#ext-database then you should set it to that database's password)
Most service logs go to @/var/log/syslog@.
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance(s).
Workbench 2 is a client-side Javascript application. If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
If you are using multiple hostnames, determine the base domain for the cluster. This will be referred to as @${DOMAIN}@.
-For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@" means @controller.xarv1.example.com@.
+For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@ means @controller.xarv1.example.com@.
h3. Machine specification
Most service logs go to @/var/log/syslog@.
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance.
Workbench 2 is a client-side Javascript application. If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
public class ExternalConfigProvider implements ConfigProvider {
+ private static final int DEFAULT_CONNECTION_TIMEOUT = 60000;
+ private static final int DEFAULT_READ_TIMEOUT = 60000;
+ private static final int DEFAULT_WRITE_TIMEOUT = 60000;
+
private boolean apiHostInsecure;
private String keepWebHost;
private int keepWebPort;
this.fileSplitDirectory = fileSplitDirectory;
this.numberOfCopies = numberOfCopies;
this.numberOfRetries = numberOfRetries;
- this.connectTimeout = 60000;
- this.readTimeout = 60000;
- this.writeTimeout = 60000;
+ this.connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+ this.readTimeout = DEFAULT_READ_TIMEOUT;
+ this.writeTimeout = DEFAULT_WRITE_TIMEOUT;
}
ExternalConfigProvider(boolean apiHostInsecure, String keepWebHost, int keepWebPort, String apiHost, int apiPort,
private File fileSplitDirectory;
private int numberOfCopies;
private int numberOfRetries;
+ private int connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+ private int readTimeout = DEFAULT_READ_TIMEOUT;
+ private int writeTimeout = DEFAULT_WRITE_TIMEOUT;
ExternalConfigProviderBuilder() {
}
return this;
}
+ public ExternalConfigProvider.ExternalConfigProviderBuilder connectTimeout(int connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ return this;
+ }
+
+ public ExternalConfigProvider.ExternalConfigProviderBuilder readTimeout(int readTimeout) {
+ this.readTimeout = readTimeout;
+ return this;
+ }
+
+ public ExternalConfigProvider.ExternalConfigProviderBuilder writeTimeout(int writeTimeout) {
+ this.writeTimeout = writeTimeout;
+ return this;
+ }
+
public ExternalConfigProvider build() {
- return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries);
+ return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries, connectTimeout, readTimeout, writeTimeout);
}
}
.fileSplitDirectory(CONFIG.getFileSplitDirectory())
.numberOfCopies(CONFIG.getNumberOfCopies())
.numberOfRetries(CONFIG.getNumberOfRetries())
+ .connectTimeout(CONFIG.getConnectTimeout())
+ .readTimeout(CONFIG.getReadTimeout())
+ .writeTimeout(CONFIG.getWriteTimeout())
.build();
}
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados Python SDK
+
+This module provides the entire Python SDK for Arvados. The most useful modules
+include:
+
+* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
+ `arvados.api` to construct a client object.
+
+* arvados.collection - The `arvados.collection.Collection` class provides a
+ high-level interface to read and write collections. It coordinates sending
+ data to and from Keep, and synchronizing updates with the collection object.
+
+* arvados.util - Utility functions to use mostly in conjunction with the API
+ client object and the results it returns.
+
+Other submodules provide lower-level functionality.
+"""
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-import bz2
-import fcntl
-import hashlib
-import http.client
-import httplib2
-import json
import logging as stdliblog
import os
-import pprint
-import re
-import string
import sys
-import time
import types
-import zlib
-if sys.version_info >= (3, 0):
- from collections import UserDict
-else:
- from UserDict import UserDict
+from collections import UserDict
from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
client constructors are `api` and `api_from_config`.
"""
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
import collections
-import http.client
import httplib2
import json
import logging
import time
import types
+from typing import (
+ Any,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+)
+
import apiclient
import apiclient.http
from apiclient import discovery as apiclient_discovery
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type):
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
"""Set up an HTTP file cache
This function constructs and returns an `arvados.cache.SafeHTTPCache`
return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
- version,
- discoveryServiceUrl,
- token,
+ version: str,
+ discoveryServiceUrl: str,
+ token: str,
*,
- cache=True,
- http=None,
- insecure=False,
- num_retries=10,
- request_id=None,
- timeout=5*60,
- **kwargs,
-):
+ cache: bool=True,
+ http: Optional[httplib2.Http]=None,
+ insecure: bool=False,
+ num_retries: int=10,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ **kwargs: Any,
+) -> apiclient_discovery.Resource:
"""Build an Arvados API client
This function returns a `googleapiclient.discovery.Resource` object
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
-
"""
if http is None:
http = httplib2.Http(
return svc
def normalize_api_kwargs(
- version=None,
- discoveryServiceUrl=None,
- host=None,
- token=None,
- **kwargs,
-):
+ version: Optional[str]=None,
+ discoveryServiceUrl: Optional[str]=None,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ **kwargs: Any,
+) -> Dict[str, Any]:
"""Validate kwargs from `api` and build kwargs for `api_client`
This method takes high-level keyword arguments passed to the `api`
**kwargs,
}
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> Dict[str, Any]:
"""Build `api_client` keyword arguments from configuration
This function accepts a mapping with Arvados configuration settings like
**kwargs,
)
-def api(version=None, cache=True, host=None, token=None, insecure=False,
- request_id=None, timeout=5*60, *,
- discoveryServiceUrl=None, **kwargs):
+def api(
+ version: Optional[str]=None,
+ cache: bool=True,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ insecure: bool=False,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ *,
+ discoveryServiceUrl: Optional[str]=None,
+ **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
from .safeapi import ThreadSafeApiCache
return ThreadSafeApiCache({}, {}, kwargs, version)
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
#
# SPDX-License-Identifier: Apache-2.0
-from builtins import range
-from builtins import object
import functools
import inspect
import pycurl
import time
from collections import deque
+from typing import (
+ Callable,
+ Generic,
+ Optional,
+ TypeVar,
+)
import arvados.errors
_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504])
-class RetryLoop(object):
+CT = TypeVar('CT', bound=Callable)
+T = TypeVar('T')
+
+class RetryLoop(Generic[T]):
"""Coordinate limited retries of code.
`RetryLoop` coordinates a loop that runs until it records a
it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- * success_check: Callable --- This is a function that will be called
- each time the loop saves a result. The function should return `True`
- if the result indicates the code succeeded, `False` if it represents a
- permanent failure, and `None` if it represents a temporary failure.
- If no function is provided, the loop will end after any result is
- saved.
+ * success_check: Callable[[T], bool | None] --- This is a function that
+ will be called each time the loop saves a result. The function should
+ return `True` if the result indicates the code succeeded, `False` if
+ it represents a permanent failure, and `None` if it represents a
+ temporary failure. If no function is provided, the loop will end
+ after any result is saved.
* backoff_start: float --- The number of seconds that must pass before
the loop's second iteration. Default 0, which disables all waiting.
* max_wait: float --- Maximum number of seconds to wait between
retries. Default 60.
"""
- def __init__(self, num_retries, success_check=lambda r: True,
- backoff_start=0, backoff_growth=2, save_results=1,
- max_wait=60):
+ def __init__(
+ self,
+ num_retries: int,
+ success_check: Callable[[T], Optional[bool]]=lambda r: True,
+ backoff_start: float=0,
+ backoff_growth: float=2,
+ save_results: int=1,
+ max_wait: float=60
+ ) -> None:
self.tries_left = num_retries + 1
self.check_result = success_check
self.backoff_wait = backoff_start
self._running = None
self._success = None
- def __iter__(self):
+ def __iter__(self) -> 'RetryLoop':
"""Return an iterator of retries."""
return self
- def running(self):
+ def running(self) -> Optional[bool]:
"""Return whether this loop is running.
Returns `None` if the loop has never run, `True` if it is still running,
"""
return self._running and (self._success is None)
- def __next__(self):
+ def __next__(self) -> int:
"""Record a loop attempt.
If the loop is still running, decrements the number of tries left and
self.tries_left -= 1
return self.tries_left
- def save_result(self, result):
+ def save_result(self, result: T) -> None:
"""Record a loop result.
Save the given result, and end the loop if it indicates
Arguments:
- * result: Any --- The result from this loop attempt to check and
- save.
+ * result: T --- The result from this loop attempt to check and save.
"""
if not self.running():
raise arvados.errors.AssertionError(
self._success = self.check_result(result)
self._attempts += 1
- def success(self):
+ def success(self) -> Optional[bool]:
"""Return the loop's end state.
Returns `True` if the loop recorded a successful result, `False` if it
"""
return self._success
- def last_result(self):
+ def last_result(self) -> T:
"""Return the most recent result the loop saved.
Raises `arvados.errors.AssertionError` if called before any result has
raise arvados.errors.AssertionError(
"queried loop results before any were recorded")
- def attempts(self):
+ def attempts(self) -> int:
"""Return the number of results that have been saved.
This count includes all kinds of results: success, permanent failure,
"""
return self._attempts
- def attempts_str(self):
+ def attempts_str(self) -> str:
"""Return a human-friendly string counting saved results.
This method returns '1 attempt' or 'N attempts', where the number
return '{} attempts'.format(self._attempts)
-def check_http_response_success(status_code):
+def check_http_response_success(status_code: int) -> Optional[bool]:
"""Convert a numeric HTTP status code to a loop control flag.
This method takes a numeric HTTP status code and returns `True` if
else:
return None # Get well soon, server.
-def retry_method(orig_func):
+def retry_method(orig_func: CT) -> CT:
"""Provide a default value for a method's num_retries argument.
This is a decorator for instance and class methods that accept a
Arvados API client.
"""
-from __future__ import absolute_import
-
-from builtins import object
import sys
import threading
+from typing import (
+ Any,
+ Mapping,
+ Optional,
+)
+
from . import config
from . import keep
from . import util
Arguments:
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`,
- and optionally `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, uses
`arvados.config.settings` to get these parameters from user
configuration. You can pass an empty mapping to build the client
solely from `api_params`.
- keep_params: Mapping[str, Any]
- : Keyword arguments used to construct an associated
- `arvados.keep.KeepClient`.
+ * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+ an associated `arvados.keep.KeepClient`.
- api_params: Mapping[str, Any]
- : Keyword arguments used to construct each thread's API client. These
- have the same meaning as in the `arvados.api.api` function.
+ * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+ each thread's API client. These have the same meaning as in the
+ `arvados.api.api` function.
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ `'v1'`.
"""
-
- def __init__(self, apiconfig=None, keep_params={}, api_params={}, version=None):
+ def __init__(
+ self,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ keep_params: Optional[Mapping[str, Any]]={},
+ api_params: Optional[Mapping[str, Any]]={},
+ version: Optional[str]=None,
+ ) -> None:
if apiconfig or apiconfig is None:
self._api_kwargs = api.api_kwargs_from_config(version, apiconfig, **api_params)
else:
self.local = threading.local()
self.keep = keep.KeepClient(api_client=self, **keep_params)
- def localapi(self):
+ def localapi(self) -> 'googleapiclient.discovery.Resource':
try:
client = self.local.api
except AttributeError:
self.local.api = client
return client
- def __getattr__(self, name):
+ def __getattr__(self, name: str) -> Any:
# Proxy nonexistent attributes to the thread-local API client.
return getattr(self.localapi(), name)
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados utilities
-from __future__ import division
-from builtins import range
+This module provides functions and constants that are useful across a variety
+of Arvados resource types, or extend the Arvados API client (see `arvados.api`).
+"""
+import errno
import fcntl
import functools
import hashlib
import random
import re
import subprocess
-import errno
import sys
import warnings
import arvados.errors
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ TypeVar,
+ Union,
+)
+
+T = TypeVar('T')
+
HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+"""Regular expression to match a hexadecimal string (case-insensitive)"""
CR_UNCOMMITTED = 'Uncommitted'
+"""Constant `state` value for uncommited container requests"""
CR_COMMITTED = 'Committed'
+"""Constant `state` value for committed container requests"""
CR_FINAL = 'Final'
+"""Constant `state` value for finalized container requests"""
keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+"""Regular expression to match any Keep block locator"""
signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+"""Regular expression to match any Keep block locator with an access token hint"""
portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
+"""Regular expression to match any collection portable data hash"""
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+"""Regular expression to match an Arvados collection manifest text"""
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a file path from a collection identified by portable data hash"""
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a `keep:` URI with a collection identified by portable data hash"""
+
uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+"""Regular expression to match any Arvados object UUID"""
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+"""Regular expression to match any Arvados collection UUID"""
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
+"""Regular expression to match any Arvados container UUID"""
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
-user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados group UUID"""
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+"""Regular expression to match any Arvados link UUID"""
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados user UUID"""
job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
-keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
-keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match any Arvados job UUID
+
+.. WARNING:: Deprecated
+ Arvados job resources are deprecated and will be removed in a future
+ release. Prefer the containers API instead.
+"""
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
If the following arguments are given, they'll be included in the
notices:
- preferred: str | None
- : The name of an alternative that users should use instead.
+ * preferred: str | None --- The name of an alternative that users should
+ use instead.
- version: str | None
- : The version of Arvados when the callable is scheduled to be
- removed.
+ * version: str | None --- The version of Arvados when the callable is
+ scheduled to be removed.
"""
if version is None:
version = ''
return deprecated_wrapper
return deprecated_decorator
+def is_hex(s: str, *length_args: int) -> bool:
+ """Indicate whether a string is a hexadecimal number
+
+ This method returns true if all characters in the string are hexadecimal
+ digits. It is case-insensitive.
+
+ You can also pass optional length arguments to check that the string has
+ the expected number of digits. If you pass one integer, the string must
+ have that length exactly, otherwise the method returns False. If you
+ pass two integers, the string's length must fall within that minimum and
+ maximum (inclusive), otherwise the method returns False.
+
+ Arguments:
+
+ * s: str --- The string to check
+
+ * length_args: int --- Optional length limit(s) for the string to check
+ """
+ num_length_args = len(length_args)
+ if num_length_args > 2:
+ raise arvados.errors.ArgumentError(
+ "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
+ elif num_length_args == 2:
+ good_len = (length_args[0] <= len(s) <= length_args[1])
+ elif num_length_args == 1:
+ good_len = (len(s) == length_args[0])
+ else:
+ good_len = True
+ return bool(good_len and HEX_RE.match(s))
+
+def keyset_list_all(
+ fn: Callable[..., 'arvados.api_resources.ArvadosAPIRequest'],
+ order_key: str="created_at",
+ num_retries: int=0,
+ ascending: bool=True,
+ **kwargs: Any,
+) -> Iterator[Dict[str, Any]]:
+ """Iterate all Arvados resources from an API list call
+
+ This method takes a method that represents an Arvados API list call, and
+ iterates the objects returned by the API server. It can make multiple API
+ calls to retrieve and iterate all objects available from the API server.
+
+ Arguments:
+
+ * fn: Callable[..., arvados.api_resources.ArvadosAPIRequest] --- A
+ function that wraps an Arvados API method that returns a list of
+ objects. If you have an Arvados API client named `arv`, examples
+ include `arv.collections().list` and `arv.groups().contents`. Note
+ that you should pass the function *without* calling it.
+
+ * order_key: str --- The name of the primary object field that objects
+ should be sorted by. This name is used to build an `order` argument
+ for `fn`. Default `'created_at'`.
+
+ * num_retries: int --- This argument is passed through to
+ `arvados.api_resources.ArvadosAPIRequest.execute` for each API call. See
+ that method's docstring for details. Default 0 (meaning API calls will
+ use the `num_retries` value set when the Arvados API client was
+ constructed).
+
+ * ascending: bool --- Used to build an `order` argument for `fn`. If True,
+ all fields will be sorted in `'asc'` (ascending) order. Otherwise, all
+ fields will be sorted in `'desc'` (descending) order.
+
+ Additional keyword arguments will be passed directly to `fn` for each API
+ call. Note that this function sets `count`, `limit`, and `order` as part of
+ its work.
+ """
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ asc = "asc" if ascending else "desc"
+ kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
+ other_filters = kwargs.get("filters", [])
+
+ try:
+ select = set(kwargs['select'])
+ except KeyError:
+ pass
+ else:
+ select.add(order_key)
+ select.add('uuid')
+ kwargs['select'] = list(select)
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+def ca_certs_path(fallback: T=httplib2.CA_CERTS) -> Union[str, T]:
+ """Return the path of the best available source of CA certificates
+
+ This function checks various known paths that provide trusted CA
+ certificates, and returns the first one that exists. It checks:
+
+ * the path in the `SSL_CERT_FILE` environment variable (used by OpenSSL)
+ * `/etc/arvados/ca-certificates.crt`, respected by all Arvados software
+ * `/etc/ssl/certs/ca-certificates.crt`, the default store on Debian-based
+ distributions
+ * `/etc/pki/tls/certs/ca-bundle.crt`, the default store on Red Hat-based
+ distributions
+
+ If none of these paths exist, this function returns the value of `fallback`.
+
+ Arguments:
+
+ * fallback: T --- The value to return if none of the known paths exist.
+ The default value is the certificate store of Mozilla's trusted CAs
+ included with the Python [certifi][] package.
+
+ [certifi]: https://pypi.org/project/certifi/
+ """
+ for ca_certs_path in [
+ # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+ # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+ os.environ.get('SSL_CERT_FILE'),
+ # Arvados specific:
+ '/etc/arvados/ca-certificates.crt',
+ # Debian:
+ '/etc/ssl/certs/ca-certificates.crt',
+ # Red Hat:
+ '/etc/pki/tls/certs/ca-bundle.crt',
+ ]:
+ if ca_certs_path and os.path.exists(ca_certs_path):
+ return ca_certs_path
+ return fallback
+
+def new_request_id() -> str:
+ """Return a random request ID
+
+ This function generates and returns a random string suitable for use as a
+ `X-Request-Id` header value in the Arvados API.
+ """
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
+
+def get_config_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's configuration, with caching
+
+ This function gets and returns the Arvados configuration from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster configuration.
+ """
+ if not svc._rootDesc.get('resources').get('configs', False):
+ # Old API server version, no config export endpoint
+ return {}
+ if not hasattr(svc, '_cached_config'):
+ svc._cached_config = svc.configs().get().execute()
+ return svc._cached_config
+
+def get_vocabulary_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's vocabulary, with caching
+
+ This function gets and returns the Arvados vocabulary from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ .. HINT:: Low-level method
+ This is a relatively low-level wrapper around the Arvados API. Most
+ users will prefer to use `arvados.vocabulary.load_vocabulary`.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster vocabulary.
+ """
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
+
+def trim_name(collectionname: str) -> str:
+ """Limit the length of a name to fit within Arvados API limits
+
+ This function ensures that a string is short enough to use as an object
+ name in the Arvados API, leaving room for text that may be added by the
+ `ensure_unique_name` argument. If the source name is short enough, it is
+ returned unchanged. Otherwise, this function returns a string with excess
+ characters removed from the middle of the source string and replaced with
+ an ellipsis.
+
+ Arguments:
+
+ * collectionname: str --- The desired source name
+ """
+ max_name_len = 254 - 28
+
+ if len(collectionname) > max_name_len:
+ over = len(collectionname) - max_name_len
+ split = int(max_name_len/2)
+ collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
+
+ return collectionname
+
+@_deprecated('3.0', 'arvados.util.keyset_list_all')
+def list_all(fn, num_retries=0, **kwargs):
+ # Default limit to (effectively) api server's MAX_LIMIT
+ kwargs.setdefault('limit', sys.maxsize)
+ items = []
+ offset = 0
+ items_available = sys.maxsize
+ while len(items) < items_available:
+ c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
+ items += c['items']
+ items_available = c['items_available']
+ offset = c['offset'] + len(c['items'])
+ return items
+
@_deprecated('3.0')
def clear_tmpdir(path=None):
"""
else:
allfiles += [ent_base]
return allfiles
-
-def is_hex(s, *length_args):
- """is_hex(s[, length[, max_length]]) -> boolean
-
- Return True if s is a string of hexadecimal digits.
- If one length argument is given, the string must contain exactly
- that number of digits.
- If two length arguments are given, the string must contain a number of
- digits between those two lengths, inclusive.
- Return False otherwise.
- """
- num_length_args = len(length_args)
- if num_length_args > 2:
- raise arvados.errors.ArgumentError(
- "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
- elif num_length_args == 2:
- good_len = (length_args[0] <= len(s) <= length_args[1])
- elif num_length_args == 1:
- good_len = (len(s) == length_args[0])
- else:
- good_len = True
- return bool(good_len and HEX_RE.match(s))
-
-@_deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
- # Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxsize)
- items = []
- offset = 0
- items_available = sys.maxsize
- while len(items) < items_available:
- c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
- items += c['items']
- items_available = c['items_available']
- offset = c['offset'] + len(c['items'])
- return items
-
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
- pagesize = 1000
- kwargs["limit"] = pagesize
- kwargs["count"] = 'none'
- asc = "asc" if ascending else "desc"
- kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
- other_filters = kwargs.get("filters", [])
-
- try:
- select = set(kwargs['select'])
- except KeyError:
- pass
- else:
- select.add(order_key)
- select.add('uuid')
- kwargs['select'] = list(select)
-
- nextpage = []
- tot = 0
- expect_full_page = True
- seen_prevpage = set()
- seen_thispage = set()
- lastitem = None
- prev_page_all_same_order_key = False
-
- while True:
- kwargs["filters"] = nextpage+other_filters
- items = fn(**kwargs).execute(num_retries=num_retries)
-
- if len(items["items"]) == 0:
- if prev_page_all_same_order_key:
- nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
- prev_page_all_same_order_key = False
- continue
- else:
- return
-
- seen_prevpage = seen_thispage
- seen_thispage = set()
-
- for i in items["items"]:
- # In cases where there's more than one record with the
- # same order key, the result could include records we
- # already saw in the last page. Skip them.
- if i["uuid"] in seen_prevpage:
- continue
- seen_thispage.add(i["uuid"])
- yield i
-
- firstitem = items["items"][0]
- lastitem = items["items"][-1]
-
- if firstitem[order_key] == lastitem[order_key]:
- # Got a page where every item has the same order key.
- # Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
- prev_page_all_same_order_key = True
- else:
- # Start from the last order key seen, but skip the last
- # known uuid to avoid retrieving the same row twice. If
- # there are multiple rows with the same order key it is
- # still likely we'll end up retrieving duplicate rows.
- # That's handled by tracking the "seen" rows for each page
- # so they can be skipped if they show up on the next page.
- nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
- prev_page_all_same_order_key = False
-
-def ca_certs_path(fallback=httplib2.CA_CERTS):
- """Return the path of the best available CA certs source.
-
- This function searches for various distribution sources of CA
- certificates, and returns the first it finds. If it doesn't find any,
- it returns the value of `fallback` (httplib2's CA certs by default).
- """
- for ca_certs_path in [
- # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
- # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
- os.environ.get('SSL_CERT_FILE'),
- # Arvados specific:
- '/etc/arvados/ca-certificates.crt',
- # Debian:
- '/etc/ssl/certs/ca-certificates.crt',
- # Red Hat:
- '/etc/pki/tls/certs/ca-bundle.crt',
- ]:
- if ca_certs_path and os.path.exists(ca_certs_path):
- return ca_certs_path
- return fallback
-
-def new_request_id():
- rid = "req-"
- # 2**104 > 36**20 > 2**103
- n = random.getrandbits(104)
- for _ in range(20):
- c = n % 36
- if c < 10:
- rid += chr(c+ord('0'))
- else:
- rid += chr(c+ord('a')-10)
- n = n // 36
- return rid
-
-def get_config_once(svc):
- if not svc._rootDesc.get('resources').get('configs', False):
- # Old API server version, no config export endpoint
- return {}
- if not hasattr(svc, '_cached_config'):
- svc._cached_config = svc.configs().get().execute()
- return svc._cached_config
-
-def get_vocabulary_once(svc):
- if not svc._rootDesc.get('resources').get('vocabularies', False):
- # Old API server version, no vocabulary export endpoint
- return {}
- if not hasattr(svc, '_cached_vocabulary'):
- svc._cached_vocabulary = svc.vocabularies().get().execute()
- return svc._cached_vocabulary
-
-def trim_name(collectionname):
- """
- trim_name takes a record name (collection name, project name, etc)
- and trims it to fit the 255 character name limit, with additional
- space for the timestamp added by ensure_unique_name, by removing
- excess characters from the middle and inserting an ellipse
- """
-
- max_name_len = 254 - 28
-
- if len(collectionname) > max_name_len:
- over = len(collectionname) - max_name_len
- split = int(max_name_len/2)
- collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
-
- return collectionname
#
# SPDX-License-Identifier: Apache-2.0
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
import argparse
import atexit
import errno
import shutil
import signal
import socket
-import string
import subprocess
import sys
import tempfile
import unittest
import yaml
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
+from urllib.parse import urlparse
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
if __name__ == '__main__' and os.path.exists(
before_destroy :ensure_owner_uuid_is_permitted
before_destroy :ensure_permission_to_destroy
before_create :update_modified_by_fields
+ before_create :add_uuid_to_name, :if => Proc.new { @_add_uuid_to_name }
before_update :maybe_update_modified_by_fields
after_create :log_create
after_update :log_update
end
def save_with_unique_name!
- uuid_was = uuid
- name_was = name
max_retries = 2
transaction do
conn = ActiveRecord::Base.connection
conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- if new_name == name
- # If the database is fast enough to do two attempts in the
- # same millisecond, we need to wait to ensure we try a
- # different timestamp on each attempt.
- sleep 0.002
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- end
-
- self[:name] = new_name
- if uuid_was.nil? && !uuid.nil?
+ if uuid_was.nil?
+ # new record, the uuid caused a name collision (very
+ # unlikely but possible), so generate new uuid
self[:uuid] = nil
if self.is_a? Collection
- # Reset so that is assigned to the new UUID
+ # Also needs to be reset
self[:current_version_uuid] = nil
end
+ # need to adjust the name after the uuid has been generated
+ add_uuid_to_make_unique_name
+ else
+ # existing record, just update the name directly.
+ add_uuid_to_name
end
-
retry
end
end
*ft[:param_out])
end
+ @_add_uuid_to_name = false
+ def add_uuid_to_make_unique_name
+ @_add_uuid_to_name = true
+ end
+
+ def add_uuid_to_name
+ # Incorporate the random part of the UUID into the name. This
+ # lets us prevent name collision but the part we add to the name
+ # is still somewhat meaningful (instead of generating a second
+ # random meaningless string).
+ #
+ # Because ArvadosModel is an abstract class and assign_uuid is
+ # part of HasUuid (which is included by the other concrete
+ # classes) the assign_uuid hook gets added (and run) after this
+ # one. So we need to call assign_uuid here to make sure we have a
+ # uuid.
+ assign_uuid
+ self.name = "#{self.name[0..236]} (#{self.uuid[-15..-1]})"
+ end
+
protected
def self.deep_sort_hash(x)
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-module SimulateJobLog
- include CurrentApiClient
- def replay(filename, multiplier = 1, simulated_job_uuid = nil)
- raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
-
- multiplier = multiplier.to_f
- multiplier = 1.0 if multiplier <= 0
-
- actual_start_time = Time.now
- log_start_time = nil
-
- if simulated_job_uuid and (job = Job.where(uuid: simulated_job_uuid).first)
- job_owner_uuid = job.owner_uuid
- else
- job_owner_uuid = system_user_uuid
- end
-
- act_as_system_user do
- File.open(filename).each.with_index do |line, index|
- cols = {}
- cols[:timestamp], rest_of_line = line.split(' ', 2)
- begin
- cols[:timestamp] = Time.strptime( cols[:timestamp], "%Y-%m-%d_%H:%M:%S" )
- rescue ArgumentError
- if line =~ /^((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (.*)/
- # Wed Nov 19 07:12:39 2014
- cols[:timestamp] = Time.strptime( $1, "%a %b %d %H:%M:%S %Y" )
- rest_of_line = $2
- else
- STDERR.puts "Ignoring log line because of unknown time format: #{line}"
- end
- end
- cols[:job_uuid], cols[:pid], cols[:task], cols[:event_type], cols[:message] = rest_of_line.split(' ', 5)
- # Override job uuid with a simulated one if specified
- cols[:job_uuid] = simulated_job_uuid || cols[:job_uuid]
- # determine when we want to simulate this log being created, based on the time multiplier
- log_start_time = cols[:timestamp] if log_start_time.nil?
- log_time = cols[:timestamp]
- actual_elapsed_time = Time.now - actual_start_time
- log_elapsed_time = log_time - log_start_time
- modified_elapsed_time = log_elapsed_time / multiplier
- pause_time = modified_elapsed_time - actual_elapsed_time
- sleep pause_time if pause_time > 0
-
- Log.new({
- owner_uuid: job_owner_uuid,
- event_at: Time.zone.local_to_utc(cols[:timestamp]),
- object_uuid: cols[:job_uuid],
- event_type: cols[:event_type],
- properties: { 'text' => line }
- }).save!
- end
- end
-
- end
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'simulate_job_log'
-desc 'Simulate job logging from a file. Three arguments: log filename, time multipler (optional), simulated job uuid (optional). E.g. (use quotation marks if using spaces between args): rake "replay_job_log[log.txt, 2.0, qr1hi-8i9sb-nf3qk0xzwwz3lre]"'
-task :replay_job_log, [:filename, :multiplier, :uuid] => :environment do |t, args|
- include SimulateJobLog
- abort("No filename specified.") if args[:filename].blank?
- replay( args[:filename], args[:multiplier].to_f, args[:uuid] )
-end
ensure_unique_name: true
}
assert_response :success
- assert_match /^owned_by_active \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
end
assert_equal false, json_response['is_trashed']
assert_nil json_response['trash_at']
assert_nil json_response['delete_at']
- assert_match /^same name for trashed and persisted collections \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test 'cannot show collection in trashed subproject' do
assert_not_equal(new_project['uuid'],
groups(:aproject).uuid,
"create returned same uuid as existing project")
- assert_match(/^A Project \(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\)$/,
+ assert_match(/^A Project \(#{new_project['uuid'][-15..-1]}\)$/,
new_project['name'])
end
ensure_unique_name: true
}
assert_response :success
- assert_match /^trashed subproject 3 \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^trashed subproject 3 \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test "move trashed subproject to new owner #{auth}" do
assert_equal ContainerRequest::Final, cr.state
output_coll = Collection.find_by_uuid(cr.output_uuid)
# Make sure the resulting output collection name include the original name
- # plus the date
+ # plus the last 15 characters of uuid
assert_not_equal output_name, output_coll.name,
"more than one collection with the same owner and name"
assert output_coll.name.include?(output_name),
"New name should include original name"
- assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
- "New name should include ISO8601 date"
+ assert_match /#{output_coll.uuid[-15..-1]}/, output_coll.name,
+ "New name should include last 15 characters of uuid"
end
[[0, :check_output_ttl_0],
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
events:
worker_connections: {{ max_reqs * 3 + 1 }}
- ### SNIPPETS
- snippets:
- # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
- ssl_hardening_default.conf:
- - ssl_session_timeout: 1d
- - ssl_session_cache: 'shared:arvadosSSL:10m'
- - ssl_session_tickets: 'off'
-
- # intermediate configuration
- - ssl_protocols: TLSv1.2 TLSv1.3
- - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
- - ssl_prefer_server_ciphers: 'off'
-
- # HSTS (ngx_http_headers_module is required) (63072000 seconds)
- - add_header: 'Strict-Transport-Security "max-age=63072000" always'
-
- # OCSP stapling
- - ssl_stapling: 'on'
- - ssl_stapling_verify: 'on'
-
- # verify chain of trust of OCSP response using Root CA and Intermediate certs
- # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
-
- # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
- # - ssl_dhparam: /path/to/dhparam
-
- # replace with the IP address of your resolver
- # - resolver: 127.0.0.1
-
### SITES
servers:
managed:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SNIPPETS
+ snippets:
+ # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+ ssl_hardening_default.conf:
+ - ssl_session_timeout: 1d
+ - ssl_session_cache: 'shared:arvadosSSL:10m'
+ - ssl_session_tickets: 'off'
+
+ # intermediate configuration
+ - ssl_protocols: TLSv1.2 TLSv1.3
+ - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ - ssl_prefer_server_ciphers: 'off'
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+ # OCSP stapling
+ - ssl_stapling: 'on'
+ - ssl_stapling_verify: 'on'
+
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+ # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+ # - ssl_dhparam: /path/to/dhparam
+
+ # replace with the IP address of your resolver
+ # - resolver: 127.0.0.1
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
# - ruby-dev
# - zlib1g-dev
- # config:
+ config:
+ check_command: /usr/bin/arvados-server config-check -strict=false -config
# file: /etc/arvados/config.yml
# user: root
## IMPORTANT!!!!!
### KEYS
secrets:
blob_signing_key: __BLOB_SIGNING_KEY__
- workbench_secret_key: __WORKBENCH_SECRET_KEY__
+ workbench_secret_key: "deprecated"
Login:
Test:
;;
generate-tokens)
- for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
+ for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN DATABASE_PASSWORD; do
echo ${i}=$(
tr -dc A-Za-z0-9 </dev/urandom | head -c 32
echo ''
MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters
SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters
ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters
-WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters
DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value
LE_AWS_ACCESS_KEY_ID="FIXME"
s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
- s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g;
s#__SSL_KEY_ENCRYPTED__#${SSL_KEY_ENCRYPTED}#g;
s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION:-}#g;
s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g;
# If we want to try a specific branch of the formula
if [[ ! -z "${BRANCH:-}" && "x${BRANCH}" != "xmain" ]]; then
- ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
+ ( cd ${F_DIR}/arvados && git fetch && git checkout --quiet "${BRANCH}" || git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
elif [ "x${ARVADOS_TAG:-}" != "x" ]; then
( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
fi
echo " - postgres" >> ${STATES_TOP}
echo " - logrotate" >> ${STATES_TOP}
echo " - docker.software" >> ${STATES_TOP}
- echo " - arvados" >> ${STATES_TOP}
+ echo " - arvados.repo" >> ${STATES_TOP}
+ echo " - arvados.config" >> ${STATES_TOP}
+ echo " - arvados.ruby" >> ${STATES_TOP}
+ echo " - arvados.api" >> ${STATES_TOP}
+ echo " - arvados.controller" >> ${STATES_TOP}
+ echo " - arvados.keepstore" >> ${STATES_TOP}
+ echo " - arvados.websocket" >> ${STATES_TOP}
+ echo " - arvados.keepweb" >> ${STATES_TOP}
+ echo " - arvados.workbench2" >> ${STATES_TOP}
+ echo " - arvados.keepproxy" >> ${STATES_TOP}
+ echo " - arvados.shell" >> ${STATES_TOP}
+ echo " - arvados.dispatcher" >> ${STATES_TOP}
echo " - extra.shell_sudo_passwordless" >> ${STATES_TOP}
echo " - extra.shell_cron_add_login_sync" >> ${STATES_TOP}
echo " - extra.passenger_rvm" >> ${STATES_TOP}
for SVC in grafana prometheus; do
grep -q "nginx_${SVC}_configuration" ${PILLARS_TOP} || echo " - nginx_${SVC}_configuration" >> ${PILLARS_TOP}
done
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
if [ "${SSL_MODE}" = "lets-encrypt" ]; then
grep -q "letsencrypt" ${PILLARS_TOP} || echo " - letsencrypt" >> ${PILLARS_TOP}
for SVC in grafana prometheus; do
grep -q "aws_credentials" ${PILLARS_TOP} || echo " - aws_credentials" >> ${PILLARS_TOP}
grep -q "postgresql" ${PILLARS_TOP} || echo " - postgresql" >> ${PILLARS_TOP}
grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP}
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
grep -q "nginx_api_configuration" ${PILLARS_TOP} || echo " - nginx_api_configuration" >> ${PILLARS_TOP}
grep -q "nginx_controller_configuration" ${PILLARS_TOP} || echo " - nginx_controller_configuration" >> ${PILLARS_TOP}
;;
"websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
### States ###
- if [ "${R}" = "workbench" ]; then
- grep -q " - logrotate" ${STATES_TOP} || echo " - logrotate" >> ${STATES_TOP}
- NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
- if grep -q " - nginx$" ${STATES_TOP}; then
- sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${STATES_TOP}
- else
- echo " - nginx.passenger" >> ${STATES_TOP}
- fi
- else
- grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP}
- fi
+ grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP}
if [ "${SSL_MODE}" = "lets-encrypt" ]; then
if [ "x${USE_LETSENCRYPT_ROUTE53:-}" = "xyes" ]; then
fi
# webshell role is just a nginx vhost, so it has no state
- if [ "${R}" != "webshell" ]; then
+ # workbench role is deprecated since 2.7.0
+ if [[ "${R}" != "webshell" && "${R}" != "workbench" ]]; then
grep -q "arvados.${R}" ${STATES_TOP} || echo " - arvados.${R}" >> ${STATES_TOP}
fi
### Pillars ###
- if [ "${R}" = "workbench" ]; then
- grep -q "logrotate_wb1" ${PILLARS_TOP} || echo " - logrotate_wb1" >> ${PILLARS_TOP}
- fi
- grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP}
grep -q "nginx_${R}_configuration" ${PILLARS_TOP} || echo " - nginx_${R}_configuration" >> ${PILLARS_TOP}
+ grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP}
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
grep -q "nginx_download_configuration" ${PILLARS_TOP} || echo " - nginx_download_configuration" >> ${PILLARS_TOP}
projects / arvados.git / commitdiff