Quickstart
==========
-Build and test all the packages for debian10 on your architecture by
+Build and test all the packages for a distribution on your architecture by
running:
- ./run-build-test-packages-one-target.sh
+ ./run-build-test-packages-one-target.sh --target DISTRO
-This will build package build and test Docker images for debian10, build all
-packages in a build container, then test all packages in a test container.
-
-Use a different distro by adding the `--target TARGET` option.
+This will build package build and test Docker images for the named target
+distribution, build all packages in a build container, then test all
+packages in a test container.
Limit the build to a single architecture by adding the `--arch ARCH`
option. Supported architectures are amd64 and arm64. Note cross-compilation
SHELL := '/bin/bash'
-all: centos7/generated
-centos7/generated: common-generated-all
- test -d centos7/generated || mkdir centos7/generated
- cp -f -rlt centos7/generated common-generated/*
-
-all: debian10/generated
-debian10/generated: common-generated-all
- test -d debian10/generated || mkdir debian10/generated
- cp -f -rlt debian10/generated common-generated/*
-
all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
-all: ubuntu1804/generated
-ubuntu1804/generated: common-generated-all
- test -d ubuntu1804/generated || mkdir ubuntu1804/generated
- cp -f -rlt ubuntu1804/generated common-generated/*
-
all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-ARG HOSTTYPE
-ARG BRANCH
-ARG GOVERSION
-
-FROM centos:7 as build_x86_64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-
-FROM centos:7 as build_aarch64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-
-FROM build_${HOSTTYPE}
-
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-SHELL ["/bin/bash", "-c"]
-# Install dependencies.
-RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel fuse-devel xz-libs git wget pam-devel
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- echo "gem: --no-document" >> ~/.gemrc && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
-
-# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
-RUN cd /usr/local/src \
-&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz \
-&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz.sig \
-&& tar xzf bash-4.4.12.tar.gz \
-&& cd bash-4.4.12 \
-&& ./configure --prefix=/usr/local/$( basename $( pwd ) ) \
-&& make \
-&& make install \
-&& ln -sf /usr/local/src/bash-4.4.12/bash /bin/bash
-
-# Need to "touch" RPM database to workaround bug in interaction between
-# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install python3 python3-pip python3-devel
-
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
-RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
-# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
-ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-
-# Preseed the go module cache and the ruby gems, using the currently checked
-# out branch of the source tree. This avoids potential compatibility issues
-# between the version of Ruby and certain gems.
-RUN git clone git://git.arvados.org/arvados.git /tmp/arvados && \
- cd /tmp/arvados && \
- if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
- cd /tmp/arvados/services/api && \
- /usr/local/rvm/bin/rvm-exec default bundle install && \
- cd /tmp/arvados && \
- go mod download
-
-# The version of setuptools that comes with CentOS is way too old
-RUN pip3 install 'setuptools<45'
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "centos7"]
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-ARG HOSTTYPE
-ARG BRANCH
-ARG GOVERSION
-
-## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
-FROM debian:buster as build_x86_64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-# No cross compilation support for debian10 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
-
-FROM debian:buster as build_aarch64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-
-FROM build_${HOSTTYPE}
-
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-SHELL ["/bin/bash", "-c"]
-# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
-
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- echo "gem: --no-document" >> ~/.gemrc && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
-
-RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
-# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
-ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-
-# Preseed the go module cache and the ruby gems, using the currently checked
-# out branch of the source tree. This avoids potential compatibility issues
-# between the version of Ruby and certain gems.
-RUN git clone git://git.arvados.org/arvados.git /tmp/arvados && \
- cd /tmp/arvados && \
- if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
- cd /tmp/arvados/services/api && \
- /usr/local/rvm/bin/rvm-exec default bundle install && \
- cd /tmp/arvados && \
- go mod download
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian10"]
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-ARG HOSTTYPE
-ARG BRANCH
-ARG GOVERSION
-
-FROM ubuntu:bionic as build_x86_64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
-# No cross compilation support for ubuntu1804 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
-
-FROM ubuntu:bionic as build_aarch64
-ONBUILD ARG BRANCH
-# Install go
-ONBUILD ARG GOVERSION
-ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
-ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-# Install nodejs and npm
-ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-ONBUILD RUN npm install -g yarn
-ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
-
-FROM build_${HOSTTYPE}
-
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-SHELL ["/bin/bash", "-c"]
-# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3.8 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3.8-venv python3.8-dev libpam-dev equivs
-
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- echo "gem: --no-document" >> ~/.gemrc && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
- /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1
-
-RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
-# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
-ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-
-# Preseed the go module cache and the ruby gems, using the currently checked
-# out branch of the source tree. This avoids potential compatibility issues
-# between the version of Ruby and certain gems.
-RUN git clone git://git.arvados.org/arvados.git /tmp/arvados && \
- cd /tmp/arvados && \
- if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
- cd /tmp/arvados/services/api && \
- /usr/local/rvm/bin/rvm-exec default bundle install && \
- cd /tmp/arvados && \
- go mod download
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated
-centos7/generated: common-generated-all
- test -d centos7/generated || mkdir centos7/generated
- cp -f -rlt centos7/generated common-generated/*
-
-all: debian10/generated
-debian10/generated: common-generated-all
- test -d debian10/generated || mkdir debian10/generated
- cp -f -rlt debian10/generated common-generated/*
-
all: debian11/generated
debian11/generated: common-generated-all
test -d debian11/generated || mkdir debian11/generated
test -d rocky8/generated || mkdir rocky8/generated
cp -f -rlt rocky8/generated common-generated/*
-all: ubuntu1804/generated
-ubuntu1804/generated: common-generated-all
- test -d ubuntu1804/generated || mkdir ubuntu1804/generated
- cp -f -rlt ubuntu1804/generated common-generated/*
-
all: ubuntu2004/generated
ubuntu2004/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM centos:7
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-# Install dependencies.
-RUN yum -q -y install scl-utils centos-release-scl which tar wget
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN touch /var/lib/rpm/* && \
- gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
-
-# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
-RUN cd /usr/local/src \
-&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz \
-&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz.sig \
-&& tar xzf bash-4.4.12.tar.gz \
-&& cd bash-4.4.12 \
-&& ./configure --prefix=/usr/local/$( basename $( pwd ) ) \
-&& make \
-&& make install \
-&& ln -sf /usr/local/src/bash-4.4.12/bash /bin/bash
-
-# Add epel, we need it for the python-pam dependency
-RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-RUN rpm -ivh epel-release-latest-7.noarch.rpm
-
-COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
+++ /dev/null
-[localrepo]
-name=Arvados Test
-baseurl=file:///arvados/packages/centos7
-gpgcheck=0
-enabled=1
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM debian:buster
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates gpg procps gpg-agent
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian10/ /" >>/etc/apt/sources.list
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM ubuntu:bionic
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates gnupg2
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu1804/ /" >>/etc/apt/sources.list
-
-# Add preferences file for the Arvados packages. This pins Arvados
-# packages at priority 501, so that older python dependency versions
-# are preferred in those cases where we need them
-ADD etc-apt-preferences.d-arvados /etc/apt/preferences.d/arvados
+++ /dev/null
-Package: *
-Pin: release o=Arvados
-Pin-Priority: 501
apt-get install -y nginx
dpkg-reconfigure "$PACKAGE_NAME"
;;
- centos*)
- yum install --assumeyes httpd
- yum reinstall --assumeyes "$PACKAGE_NAME"
- ;;
rocky*)
microdnf --assumeyes install httpd
microdnf --assumeyes reinstall "$PACKAGE_NAME"
target="$(basename "$0" .sh)"
target="${target##*-}"
-case "$target" in
- centos*) yum -q clean all ;;
- rocky*) microdnf --assumeyes clean all ;;
-esac
+microdnf --assumeyes clean all
touch /var/lib/rpm/*
export ARV_PACKAGES_DIR="/arvados/packages/$target"
rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
-
-case "$target" in
- centos*) yum install --assumeyes -e 0 $1 ;;
- rocky*) microdnf --assumeyes install $1 ;;
-esac
-
+microdnf --assumeyes install "$1"
rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
-
diff "$ARV_PACKAGES_DIR/$1".{before,after} >"$ARV_PACKAGES_DIR/$1.diff" || true
-# Enable any Software Collections that the package depended on.
-if [[ -d /opt/rh ]]; then
- # We have to stage the list to a file, because `ls | while read` would
- # make a subshell, causing the `source` lines to have no effect.
- scl_list=$(mktemp)
- ls /opt/rh >"$scl_list"
-
- # SCL scripts aren't designed to run with -eu.
- set +eu
- while read scl; do
- source scl_source enable "$scl"
- done <"$scl_list"
- set -eu
- rm "$scl_list"
-fi
-
mkdir -p /tmp/opts
cd /tmp/opts
$(basename $0): Orchestrate run-build-packages.sh for one target
Syntax:
- WORKSPACE=/path/to/arvados $(basename $0) [options]
+ WORKSPACE=/path/to/arvados $(basename $0) --target <target> [options]
--target <target>
- Distribution to build packages for (default: debian10)
+ Distribution to build packages for
--command
Build command to execute (default: use built-in Docker image command)
--test-packages
exit 1
fi
-TARGET=debian10
FORCE_BUILD=0
COMMAND=
DEBUG=
+TARGET=
eval set -- "$PARSEDOPTS"
while [ $# -gt 0 ]; do
set -e
orig_umask="$(umask)"
+if [[ -z "$TARGET" ]]; then
+ echo "FATAL: --target must be specified" >&2
+ exit 2
+elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then
+ echo "FATAL: unknown build target '$TARGET'" >&2
+ exit 2
+fi
+
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
fi
$(basename "$0"): Build Arvados packages
Syntax:
- WORKSPACE=/path/to/arvados $(basename "$0") [options]
+ WORKSPACE=/path/to/arvados $(basename "$0") --target <target> [options]
Options:
--debug
Output debug information (default: false)
--target <target>
- Distribution to build packages for (default: debian10)
+ Distribution to build packages for
--only-build <package>
Build only a specific package (or ONLY_BUILD from environment)
--arch <arch>
DEBUG=${ARVADOS_DEBUG:-0}
FORCE_BUILD=${FORCE_BUILD:-0}
EXITCODE=0
-TARGET=debian10
COMMAND=
+TARGET=
PARSEDOPTS=$(getopt --name "$0" --longoptions \
help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
shift
done
+if [[ -z "$TARGET" ]]; then
+ echo "FATAL: --target must be specified" >&2
+ exit 2
+elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then
+ echo "FATAL: unknown build target '$TARGET'" >&2
+ exit 2
+fi
+
if [[ "$COMMAND" != "" ]]; then
COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
fi
$(basename $0): Build, test and (optionally) upload packages for one target
Syntax:
- WORKSPACE=/path/to/arvados $(basename $0) [options]
+ WORKSPACE=/path/to/arvados $(basename $0) --target <target> [options]
--target <target>
- Distribution to build packages for (default: debian10)
+ Distribution to build packages for
--only-build <package>
Build only a specific package (or ONLY_BUILD from environment)
--arch <arch>
exit 1
fi
-TARGET=debian10
UPLOAD=0
RC=0
DEBUG=
+TARGET=
declare -a build_args=()
shift
done
+if [[ -z "$TARGET" ]]; then
+ echo "FATAL: --target must be specified" >&2
+ exit 2
+elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then
+ echo "FATAL: unknown build target '$TARGET'" >&2
+ exit 2
+fi
+
build_args+=(--target "$TARGET")
if [[ -n "$ONLY_BUILD" ]]; then
fi
case "$package_format-$TARGET" in
- # Older Debian/Ubuntu do not support cross compilation because the
+ # Ubuntu 20.04 does not support cross compilation because the
# libfuse package does not support multiarch. See
# <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477>.
# Red Hat-based distributions do not support native cross compilation at
# all (they use a qemu-based solution we haven't implemented yet).
- deb-debian10|deb-ubuntu1804|deb-ubuntu2004|rpm-*)
+ deb-ubuntu2004|rpm-*)
cross_compilation=0
if [[ "$native_arch" == "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
echo "Error: no cross compilation support for Go on $native_arch for $TARGET, can not build $prog for $target_arch"
echo "Package $full_pkgname build forced with --force-build, building"
elif [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian10]=buster
dd[debian11]=bullseye
dd[debian12]=bookworm
- dd[ubuntu1804]=bionic
dd[ubuntu2004]=focal
dd[ubuntu2204]=jammy
D=${dd[$TARGET]}
else
local rpm_root
case "$TARGET" in
- centos7) rpm_root="CentOS/7/dev" ;;
rocky8) rpm_root="CentOS/8/dev" ;;
*)
echo "FIXME: Don't know RPM URL path for $TARGET, building"
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
Nice=19
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
RestartPreventExitStatus=2
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
-v "${tmpdir}/arvados-server":/bin/arvados-server:ro \
-v "${tmpdir}/zzzzz.yml":/etc/arvados/config.yml:ro \
-v $(realpath "${PWD}/../../.."):/arvados:ro \
- debian:10 \
+ debian:11 \
bash -c "${setup_pam_ldap:-true} && arvados-server controller"
docker logs --follow ${ctrlctr} 2>$debug >$debug &
ctrlhostports=$(docker port ${ctrlctr} 9999/tcp)
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/config.yml
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
LimitNOFILE=65536
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
#!/bin/bash
+#
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
set -ex -o pipefail
ctrname=arvadostest
ctrbase=${ctrname}
if [[ "${1}" != "--update" ]] || ! docker images --format={{.Repository}} | grep -x ${ctrbase}; then
- ctrbase=debian:10
+ ctrbase=debian:11
fi
if docker ps -a --format={{.Names}} | grep -x ${ctrname}; then
pkgs = append(pkgs, "g++", "libcurl4", "libcurl4-openssl-dev")
case osv.Debian || osv.Ubuntu:
pkgs = append(pkgs, "g++", "libcurl3", "libcurl3-openssl-dev")
- case osv.Centos:
+ case osv.RedHat:
pkgs = append(pkgs, "gcc", "gcc-c++", "libcurl-devel", "postgresql-devel")
}
cmd := exec.CommandContext(ctx, "apt-get")
} else if osv.Debian {
var codename string
switch osv.Major {
- case 10:
- codename = "buster"
case 11:
codename = "bullseye"
case 12:
type osversion struct {
Debian bool
Ubuntu bool
- Centos bool
+ RedHat bool
Major int
}
osv.Ubuntu = true
case "debian":
osv.Debian = true
- case "centos":
- osv.Centos = true
default:
- return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
+ idLikeMatched := false
+ for _, idLike := range strings.Split(kv["ID_LIKE"], " ") {
+ switch idLike {
+ case "debian":
+ osv.Debian = true
+ idLikeMatched = true
+ case "rhel":
+ osv.RedHat = true
+ idLikeMatched = true
+ }
+ if idLikeMatched {
+ break
+ }
+ }
+ if !idLikeMatched {
+ return osv, fmt.Errorf("no supported ID found in /etc/os-release")
+ }
}
vstr := kv["VERSION_ID"]
if i := strings.Index(vstr, "."); i > 0 {
return append(pkgs,
"mime-support", // keep-web
)
- } else if osv.Centos {
+ } else if osv.RedHat {
return append(pkgs,
"fuse-libs", // services/fuse
"mailcap", // keep-web
#!/bin/bash
+#
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
set -e -o pipefail
-# Starting with a base debian buster system, like "docker run -it
-# debian:10"...
+# Starting with a base debian bullseye system, like "docker run -it
+# debian:11"...
apt update
apt upgrade
apt install --no-install-recommends build-essential ca-certificates git golang
git clone https://git.arvados.org/arvados.git
-cd arvados
-[[ -e lib/install ]] || git checkout origin/16053-install-deps
-cd cmd/arvados-server
+cd arvados/cmd/arvados-server
go run ./cmd/arvados-server install -type test
-pg_isready || pg_ctlcluster 11 main start # only needed if there's no init process (as in docker)
+pg_isready || pg_ctlcluster 13 main start # only needed if there's no init process (as in docker)
build/run-tests.sh
"-v", s.tmpdir + "/pam_arvados.so:/usr/lib/pam_arvados.so:ro",
"-v", s.tmpdir + "/conffile:/usr/share/pam-configs/arvados:ro",
"-v", s.tmpdir + "/testclient:/testclient:ro",
- "debian:buster",
+ "debian:bullseye",
"/testclient"}, args...)...)
stdout = &bytes.Buffer{}
stderr = &bytes.Buffer{}
centos*|rocky*)
fpm_depends+=(libcurl-devel postgresql-devel bison make automake gcc gcc-c++ postgresql shared-mime-info)
;;
- ubuntu1804)
- fpm_depends+=(libcurl-ssl-dev libpq-dev g++ bison zlib1g-dev make postgresql-client shared-mime-info)
- fpm_conflicts+=(ruby-bundler)
- ;;
debian* | ubuntu*)
fpm_depends+=(libcurl-ssl-dev libpq-dev g++ bison zlib1g-dev make postgresql-client shared-mime-info)
;;
Description=Arvados Crunch Dispatcher for LOCAL service
Documentation=https://doc.arvados.org/
After=network.target
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
RestartSec=1
LimitNOFILE=1000000
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
[Install]
WantedBy=multi-user.target
Description=Arvados Docker Image Cleaner
Documentation=https://doc.arvados.org/
After=network.target
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
Restart=always
RestartSec=10s
RestartPreventExitStatus=2
-#
-# This unwieldy ExecStart command detects at runtime whether
-# arvados-docker-cleaner is installed with the Python 3.3 Software
-# Collection, and if so, invokes it with the "scl" wrapper.
-ExecStart=/bin/sh -c 'if [ -e /opt/rh/rh-python36/root/bin/arvados-docker-cleaner ]; then exec scl enable rh-python36 arvados-docker-cleaner; else exec arvados-docker-cleaner; fi'
-
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
+ExecStart=/usr/bin/arvados-docker-cleaner
[Install]
WantedBy=multi-user.target
# changes in the package. (i.e. example config files externally added
ITERATION?=1
-TARGETS?=centos7 rocky8 debian10 debian11 ubuntu1804 ubuntu2004
+TARGETS?=rocky8 debian11 debian12 ubuntu2004 ubuntu2204
DESCRIPTION=Arvados Workbench2 - Arvados is a free and open source platform for big data science.
MAINTAINER=Arvados Package Maintainers <packaging@arvados.org>
#
# SPDX-License-Identifier: AGPL-3.0
-FROM node:12.22.3-buster
+FROM node:12.22.12-bullseye
LABEL maintainer="Arvados Package Maintainers <packaging@arvados.org>"
-RUN echo deb http://deb.debian.org/debian buster-backports main >> /etc/apt/sources.list.d/backports.list
+RUN echo deb http://deb.debian.org/debian bullseye-backports main >> /etc/apt/sources.list.d/backports.list
RUN apt-get update && \
apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
libsecret-1-0 libsecret-1-dev rpm ruby ruby-dev rubygems build-essential \
label: {
boxSizing: 'border-box',
color: theme.palette.grey["600"],
- width: '100%'
+ width: '100%',
+ marginTop: "0.4em",
},
value: {
boxSizing: 'border-box',
</Tooltip>}
</Typography>
</Typography>);
-
| "symmetricTabs"
| "imagePlaceholder"
| "rowWithPreview"
- | "labelColumn";
+ | "labelColumn"
+ | "primaryRow";
const styles: StyleRulesCallback<CssRules> = (theme: ArvadosTheme) => ({
card: {
},
tableWrapper: {
height: "auto",
- maxHeight: `calc(100% - ${theme.spacing.unit * 4.5}px)`,
+ maxHeight: `calc(100% - ${theme.spacing.unit * 3}px)`,
overflow: "auto",
},
tableRoot: {
paddingLeft: "20px",
},
secondaryRow: {
- height: "29px",
+ height: "24px",
verticalAlign: "top",
position: "relative",
- top: "-9px",
+ top: "-4px",
},
emptyValue: {
color: theme.customs.colors.grey700,
noBorderRow: {
"& td": {
borderBottom: "none",
+ paddingTop: "2px",
+ paddingBottom: "2px",
},
+ height: "24px",
},
symmetricTabs: {
"& button": {
labelColumn: {
minWidth: "120px",
},
+ primaryRow: {
+ height: "24px",
+ "& td": {
+ paddingTop: "2px",
+ paddingBottom: "2px",
+ },
+ },
});
export enum ProcessIOCardType {
</Grid>
)}
{/* Once loaded, either raw or params may still be empty
- * Raw when all params are empty
- * Params when raw is provided by containerRequest properties but workflow mount is absent for preview
- */}
+ * Raw when all params are empty
+ * Params when raw is provided by containerRequest properties but workflow mount is absent for preview
+ */}
{!loading && (hasRaw || hasParams) && (
<>
<Tabs
{/* params will be empty on processes without workflow definitions in mounts, so we only show raw */}
{hasParams && <Tab label="Parameters" />}
{!forceShowParams && <Tab label="JSON" />}
+ {hasOutputCollecton && <Tab label="Collection" />}
</Tabs>
{mainProcTabState === 0 && params && hasParams && (
<div className={classes.tableWrapper}>
<ProcessIORaw data={raw} />
</div>
)}
+ {mainProcTabState === 2 && hasOutputCollecton && (
+ <>
+ {outputUuid && (
+ <Typography className={classes.collectionLink}>
+ Output Collection:{" "}
+ <MuiLink
+ className={classes.keepLink}
+ onClick={() => {
+ navigateTo(outputUuid || "");
+ }}
+ >
+ {outputUuid}
+ </MuiLink>
+ </Typography>
+ )}
+ <ProcessOutputCollectionFiles
+ isWritable={false}
+ currentItemUuid={outputUuid}
+ />
+ </>
+ )}
+
</>
)}
{!loading && !hasRaw && !hasParams && (
const rest = param.value.slice(1);
const mainRowClasses = {
[classes.noBorderRow]: rest.length > 0,
+ [classes.primaryRow]: true
};
return (
const rowClasses = {
[classes.noBorderRow]: i < rest.length - 1,
[classes.secondaryRow]: val.secondary,
+ [classes.primaryRow]: !val.secondary,
};
return (
<TableRow
const panelsData: MPVPanelState[] = [
{ name: "Details" },
- { name: "Command" },
{ name: "Logs", visible: true },
- { name: "Inputs" },
+ { name: "Subprocesses" },
{ name: "Outputs" },
+ { name: "Inputs" },
+ { name: "Command" },
{ name: "Resources" },
- { name: "Subprocesses" },
];
export const ProcessPanelRoot = withStyles(styles)(
}
}, [containerRequest, loadInputs, loadOutputs, loadOutputDefinitions, loadNodeJson]);
+ const maxHeight = "100%";
+
// Trigger processing output params when raw or definitions change
React.useEffect(() => {
updateOutputParams();
resumeOnHoldWorkflow={props.resumeOnHoldWorkflow}
/>
</MPVPanelContent>
- <MPVPanelContent
- forwardProps
- xs="auto"
- data-cy="process-cmd">
- <ProcessCmdCard
- onCopy={props.onCopyToClipboard}
- process={process}
- />
- </MPVPanelContent>
<MPVPanelContent
forwardProps
xs
- minHeight="50%"
+ minHeight={maxHeight}
+ maxHeight={maxHeight}
data-cy="process-logs">
<ProcessLogsCard
onCopy={props.onCopyToClipboard}
<MPVPanelContent
forwardProps
xs
- maxHeight="50%"
- data-cy="process-inputs">
- <ProcessIOCard
- label={ProcessIOCardType.INPUT}
- process={process}
- params={inputParams}
- raw={inputRaw}
- mounts={inputMounts}
- />
+ maxHeight={maxHeight}
+ data-cy="process-children">
+ <SubprocessPanel process={process} />
</MPVPanelContent>
<MPVPanelContent
forwardProps
xs
- maxHeight="50%"
+ maxHeight={maxHeight}
data-cy="process-outputs">
<ProcessIOCard
label={ProcessIOCardType.OUTPUT}
<MPVPanelContent
forwardProps
xs
- data-cy="process-resources">
- <ProcessResourceCard
+ maxHeight={maxHeight}
+ data-cy="process-inputs">
+ <ProcessIOCard
+ label={ProcessIOCardType.INPUT}
+ process={process}
+ params={inputParams}
+ raw={inputRaw}
+ mounts={inputMounts}
+ />
+ </MPVPanelContent>
+ <MPVPanelContent
+ forwardProps
+ xs="auto"
+ data-cy="process-cmd">
+ <ProcessCmdCard
+ onCopy={props.onCopyToClipboard}
process={process}
- nodeInfo={nodeInfo}
/>
</MPVPanelContent>
<MPVPanelContent
forwardProps
xs
- maxHeight="50%"
- data-cy="process-children">
- <SubprocessPanel process={process} />
+ data-cy="process-resources">
+ <ProcessResourceCard
+ process={process}
+ nodeInfo={nodeInfo}
+ />
</MPVPanelContent>
</MPVContainer>
) : (
dockerversion=5:20.10.13~3-0
if [[ "$DIST" =~ ^debian ]]; then
family="debian"
- if [ "$DIST" == "debian10" ]; then
- distro="buster"
- elif [ "$DIST" == "debian11" ]; then
+ if [ "$DIST" == "debian11" ]; then
distro="bullseye"
+ elif [ "$DIST" == "debian12" ]; then
+ distro="bookworm"
fi
elif [[ "$DIST" =~ ^ubuntu ]]; then
family="ubuntu"
- if [ "$DIST" == "ubuntu1804" ]; then
- distro="bionic"
- elif [ "$DIST" == "ubuntu2004" ]; then
+ if [ "$DIST" == "ubuntu2004" ]; then
distro="focal"
+ elif [ "$DIST" == "ubuntu2204" ]; then
+ distro="jammy"
fi
else
echo "Unsupported distribution $DIST"
$SUDO apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/3bf863cc.pub
$SUDO apt-get -y install software-properties-common
$SUDO add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/ /"
- # Ubuntu 18.04's add-apt-repository does not understand 'contrib'
- $SUDO add-apt-repository contrib || true
+ $SUDO add-apt-repository contrib
$SUDO apt-get update
$SUDO apt-get -y install cuda
user: {{ database_user }}
extra_conn_params:
client_encoding: UTF8
- # Centos7 does not enable SSL by default, so we disable
- # it here just for testing of the formula purposes only.
- # You should not do this in production, and should
- # configure Postgres certificates correctly
- {%- if grains.os_family in ('RedHat',) %}
- sslmode: disable
- {%- endif %}
tls:
# certificate: ''
### POSTGRESQL
postgres:
- # Centos-7's postgres package is too old, so we need to force using upstream's
- # This is not required in Debian's family as they already ship with PG +11
- {%- if salt['grains.get']('os_family') == 'RedHat' %}
- use_upstream_repo: true
- version: '12'
-
- pkgs_deps:
- - libicu
- - libxslt
- - systemd-sysv
-
- pkgs_extra:
- - postgresql12-contrib
-
- {%- else %}
use_upstream_repo: false
pkgs_extra:
- postgresql-contrib
- {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
#ssl = on
- openssl
- ca-certificates
-# Remove the RANDFILE parameter in openssl.cnf as it makes openssl fail in Ubuntu 18.04
-# Saving and restoring the rng state is not necessary anymore in the openssl 1.1.1
-# random generator, cf
-# https://github.com/openssl/openssl/issues/7754
-#
-extra_snakeoil_certs_file_comment_etc_openssl_conf:
- file.comment:
- - name: /etc/ssl/openssl.cnf
- - regex: ^RANDFILE.*
- - onlyif: grep -q ^RANDFILE /etc/ssl/openssl.cnf
- - require_in:
- - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
-
extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run:
# Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
- # These dirs are not to CentOS-ish, but this is a helper script
+ # These dirs are not too CentOS-ish, but this is a helper script
# and they should be enough
/bin/bash -c "mkdir -p /etc/ssl/certs/ /etc/ssl/private/ && \
openssl req \
user: {{ database_user }}
extra_conn_params:
client_encoding: UTF8
- # Centos7 does not enable SSL by default, so we disable
- # it here just for testing of the formula purposes only.
- # You should not do this in production, and should
- # configure Postgres certificates correctly
- {%- if grains.os_family in ('RedHat',) %}
- sslmode: disable
- {%- endif %}
tls:
# certificate: ''
### POSTGRESQL
postgres:
- # Centos-7's postgres package is too old, so we need to force using upstream's
- # This is not required in Debian's family as they already ship with PG +11
- {%- if salt['grains.get']('os_family') == 'RedHat' %}
- use_upstream_repo: true
- version: '12'
-
- pkgs_deps:
- - libicu
- - libxslt
- - systemd-sysv
-
- pkgs_extra:
- - postgresql12-contrib
-
- {%- else %}
use_upstream_repo: false
pkgs_extra:
- postgresql-contrib
- {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
# If you want to enable communications' encryption to the DB server,
- openssl
- ca-certificates
-# Remove the RANDFILE parameter in openssl.cnf as it makes openssl fail in Ubuntu 18.04
-# Saving and restoring the rng state is not necessary anymore in the openssl 1.1.1
-# random generator, cf
-# https://github.com/openssl/openssl/issues/7754
-#
-extra_snakeoil_certs_file_comment_etc_openssl_conf:
- file.comment:
- - name: /etc/ssl/openssl.cnf
- - regex: ^RANDFILE.*
- - onlyif: grep -q ^RANDFILE /etc/ssl/openssl.cnf
- - require_in:
- - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
-
extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run:
# Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
else
# Install a few dependency packages
# First, let's figure out the OS we're working on
- OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \")
- echo "Detected distro: ${OS_ID}"
-
- case ${OS_ID} in
- "centos")
- echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
- sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
- setenforce permissive
- yum install -y curl git jq
- ;;
- "debian"|"ubuntu")
- # Wait 2 minutes for any apt locks to clear
- # This option is supported from apt 1.9.1 and ignored in older apt versions.
- # Cf. https://blog.sinjakli.co.uk/2021/10/25/waiting-for-apt-locks-without-the-hacky-bash-scripts/
- DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update
- DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
- ;;
- esac
+ OS_IDS="$(. /etc/os-release && echo "${ID:-} ${ID_LIKE:-}")"
+ echo "Detected distro families: $OS_IDS"
+
+ for OS_ID in $OS_IDS; do
+ case "$OS_ID" in
+ rhel)
+ echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
+ sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
+ setenforce permissive
+ yum install -y curl git jq
+ break
+ ;;
+ debian)
+ DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update
+ DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
+ break
+ ;;
+ esac
+ done
if which salt-call; then
echo "Salt already installed"
projects / arvados.git / commitdiff