20797: Add initial support for rocky8 package builds
authorBrett Smith <brett.smith@curii.com>
Fri, 28 Jul 2023 17:48:08 +0000 (13:48 -0400)
committerBrett Smith <brett.smith@curii.com>
Fri, 28 Jul 2023 20:37:19 +0000 (16:37 -0400)
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

17 files changed:
.licenseignore
apps/workbench/fpm-info.sh
build/package-build-dockerfiles/Makefile
build/package-build-dockerfiles/rocky8/Dockerfile [new file with mode: 0644]
build/package-test-dockerfiles/Makefile
build/package-test-dockerfiles/rocky8/Dockerfile [new file with mode: 0644]
build/package-test-dockerfiles/rocky8/localrepo.repo [new file with mode: 0644]
build/package-testing/common-test-rails-server-package.sh
build/package-testing/rpm-common-test-packages.sh
build/package-testing/test-packages-rocky8.sh [new symlink]
build/run-build-packages.sh
build/run-library.sh
cmd/arvados-client/fpm-info.sh
lib/pam/fpm-info.sh
services/api/fpm-info.sh
services/fuse/fpm-info.sh
services/keep-web/fpm-info.sh

index 361f09b4e3f6c774060a46f85dd6b1edca173dbc..36d2876c554ad30eb29a4b7b05fd4692294920eb 100644 (file)
@@ -8,6 +8,7 @@ AUTHORS
 */bootstrap.js
 *bootstrap-theme.css
 build/package-test-dockerfiles/centos7/localrepo.repo
+build/package-test-dockerfiles/rocky8/localrepo.repo
 build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados
 *by-sa-3.0.html
 *by-sa-3.0.txt
index a09638a35e1add08cc66a4bf140403b988043622..a933fb702b2930b440a974a625c9733add68b259 100644 (file)
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 case "$TARGET" in
-    centos*)
+    centos*|rocky*)
         fpm_depends+=(git bison make automake gcc gcc-c++ graphviz shared-mime-info)
         ;;
     ubuntu1804)
index b8f6315e79410f471230f88336cb297e98206c95..416e746d20e284402dbf82fc1d32df053bc0a895 100644 (file)
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 SHELL := '/bin/bash'
-all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated
+all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
@@ -17,6 +17,10 @@ debian11/generated: common-generated-all
        test -d debian11/generated || mkdir debian11/generated
        cp -f -rlt debian11/generated common-generated/*
 
+rocky8/generated: common-generated-all
+       test -d rocky8/generated || mkdir rocky8/generated
+       cp -f -rlt rocky8/generated common-generated/*
+
 ubuntu1804/generated: common-generated-all
        test -d ubuntu1804/generated || mkdir ubuntu1804/generated
        cp -f -rlt ubuntu1804/generated common-generated/*
diff --git a/build/package-build-dockerfiles/rocky8/Dockerfile b/build/package-build-dockerfiles/rocky8/Dockerfile
new file mode 100644 (file)
index 0000000..0fd4002
--- /dev/null
@@ -0,0 +1,97 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ARG HOSTTYPE
+ARG BRANCH
+ARG GOVERSION
+
+FROM rockylinux:8.6-minimal as build_x86_64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+
+FROM rockylinux:8.6-minimal as build_aarch64
+# Install go
+ONBUILD ARG GOVERSION
+ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+# Install dependencies.
+RUN microdnf --enablerepo=devel install \
+    automake \
+    bison \
+    bzip2 \
+    fuse-devel \
+    gcc \
+    gcc-c++ \
+    git \
+    libattr-devel \
+    libcurl-devel \
+    libffi-devel \
+    libtool \
+    libyaml-devel \
+    make \
+    nss-devel \
+    openssl-devel \
+    pam-devel \
+    patch \
+    postgresql-devel \
+    procps-ng \
+    python3 \
+    python3-devel \
+    python3-pip \
+    python3-virtualenv \
+    readline-devel \
+    rpm-build \
+    ruby \
+    sqlite-devel \
+    tar \
+    unzip \
+    wget \
+    which \
+    xz-libs \
+    zlib-devel
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN gpg --import --no-tty /tmp/mpapis.asc && \
+    gpg --import --no-tty /tmp/pkuczynski.asc && \
+    curl -L https://get.rvm.io | bash -s stable && \
+    /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+    /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
+    echo "gem: --no-document" >> ~/.gemrc && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
+    cd /tmp/arvados/services/api && \
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados/apps/workbench && \
+    /usr/local/rvm/bin/rvm-exec default bundle install && \
+    cd /tmp/arvados && \
+    go mod download
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "rocky8"]
index 849decb9a5016bb2676ea4a9e0e92ba639edd6d9..96a80055ac9a84e5256e5bba65a93dac3c4a48f0 100644 (file)
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated
+all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
@@ -16,6 +16,10 @@ debian11/generated: common-generated-all
        test -d debian11/generated || mkdir debian11/generated
        cp -f -rlt debian11/generated common-generated/*
 
+rocky8/generated: common-generated-all
+       test -d rocky8/generated || mkdir rocky8/generated
+       cp -f -rlt rocky8/generated common-generated/*
+
 ubuntu1804/generated: common-generated-all
        test -d ubuntu1804/generated || mkdir ubuntu1804/generated
        cp -f -rlt ubuntu1804/generated common-generated/*
diff --git a/build/package-test-dockerfiles/rocky8/Dockerfile b/build/package-test-dockerfiles/rocky8/Dockerfile
new file mode 100644 (file)
index 0000000..e1c6f27
--- /dev/null
@@ -0,0 +1,48 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM rockylinux:8.6-minimal
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+
+# Install dependencies.
+RUN microdnf --enablerepo=devel install \
+    autoconf \
+    automake \
+    bison \
+    bzip2 \
+    cpio \
+    diffutils \
+    findutils \
+    gcc-c++ \
+    glibc-devel \
+    glibc-headers \
+    gzip \
+    libffi-devel \
+    libtool \
+    make \
+    openssl-devel \
+    patch \
+    procps-ng \
+    python3 \
+    readline-devel \
+    ruby \
+    shadow-utils \
+    sqlite-devel \
+    tar \
+    wget \
+    which \
+    zlib-devel
+
+# Install RVM
+ADD generated/mpapis.asc /tmp/
+ADD generated/pkuczynski.asc /tmp/
+RUN touch /var/lib/rpm/* && \
+    gpg --import --no-tty /tmp/mpapis.asc && \
+    gpg --import --no-tty /tmp/pkuczynski.asc && \
+    curl -L https://get.rvm.io | bash -s stable && \
+    /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+    /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
diff --git a/build/package-test-dockerfiles/rocky8/localrepo.repo b/build/package-test-dockerfiles/rocky8/localrepo.repo
new file mode 100644 (file)
index 0000000..a4f6ab3
--- /dev/null
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/rocky8
+gpgcheck=0
+enabled=1
index e04556bb6b50b2312129726cbddcd354b4fb7685..a60fcc465259ae89d0dc99fd114f179d8fffd6a7 100755 (executable)
@@ -47,6 +47,10 @@ case "$TARGET" in
         yum install --assumeyes httpd
         yum reinstall --assumeyes "$PACKAGE_NAME"
         ;;
+    rocky*)
+        microdnf install httpd
+        microdnf reinstall "$PACKAGE_NAME"
+        ;;
     *)
         echo -e "$0: Unknown target '$TARGET'.\n" >&2
         exit 1
index 12450dd4f954acf65a58fe637880697be5918861..7cd2be7cd3fb11c72f11fca0ef8610f994e0d22a 100755 (executable)
@@ -14,16 +14,23 @@ if [[ "$DEBUG" != "0" ]]; then
   STDERR_IF_DEBUG=/dev/stderr
 fi
 
-target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b')
+target="$(basename "$0" .sh)"
+target="${target##*-}"
 
-yum -q clean all
+case "$target" in
+    centos*) yum -q clean all ;;
+    rocky*) microdnf clean all ;;
+esac
 touch /var/lib/rpm/*
 
 export ARV_PACKAGES_DIR="/arvados/packages/$target"
 
 rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
 
-yum install --assumeyes -e 0 $1
+case "$target" in
+    centos*) yum install --assumeyes -e 0 $1 ;;
+    rocky*) microdnf install $1 ;;
+esac
 
 rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
 
diff --git a/build/package-testing/test-packages-rocky8.sh b/build/package-testing/test-packages-rocky8.sh
new file mode 120000 (symlink)
index 0000000..64ef604
--- /dev/null
@@ -0,0 +1 @@
+rpm-common-test-packages.sh
\ No newline at end of file
index aded25b592a3a941e794d0223cd0f0c8ea5f412a..960932dbe3a9037e8041059f9a629b982308b033 100755 (executable)
@@ -141,6 +141,13 @@ case "$TARGET" in
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
         export PYCURL_SSL_LIBRARY=nss
         ;;
+    rocky*)
+        FORMAT=rpm
+        PYTHON3_PACKAGE=$(rpm -qf "$(which python"$PYTHON3_VERSION")" --queryformat '%{NAME}\n')
+        PYTHON3_PKG_PREFIX=python3
+        PYTHON3_PREFIX=/usr
+        PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
+        ;;
     *)
         echo -e "$0: Unknown target '$TARGET'.\n" >&2
         exit 1
index c2466faac0f38f66ade7a3e2d3c509a1e9acbbe8..49cab6a0f1acef2fa9ccf670bcaf51ef4be46b65 100755 (executable)
@@ -919,11 +919,6 @@ fpm_build_virtualenv_worker () {
   LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
   COMMAND_ARR+=('--license' "$LICENSE_STRING")
 
-  if [[ "$package_format" == "rpm" ]]; then
-    # Make sure to conflict with the old rh-python36 packages we used to publish
-    COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
-  fi
-
   if [[ "$DEBUG" != "0" ]]; then
     COMMAND_ARR+=('--verbose' '--log' 'info')
   fi
@@ -940,9 +935,21 @@ fpm_build_virtualenv_worker () {
   fi
 
   COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
-
-  # avoid warning
-  COMMAND_ARR+=('--deb-no-default-config-files')
+  case "$package_format" in
+      deb)
+          COMMAND_ARR+=(
+              # Avoid warning
+              --deb-no-default-config-files
+          ) ;;
+      rpm)
+          COMMAND_ARR+=(
+              # Conflict with older packages we used to publish
+              --conflicts "rh-python36-python-$PKG"
+              # Do not generate /usr/lib/.build-id links on RH8+
+              # (otherwise our packages conflict with platform-python)
+              --rpm-rpmbuild-define "_build_id_links none"
+          ) ;;
+  esac
 
   # Append --depends X and other arguments specified by fpm-info.sh in
   # the package source dir. These are added last so they can override
index 0e4415a83ff3c581920f61614811ab1d58c2cea2..c717632452d44e27d4bb07e9f2dbee967f596e5d 100644 (file)
@@ -5,7 +5,7 @@
 fpm_depends+=(fuse)
 
 case "$TARGET" in
-    centos*)
+    centos*|rocky*)
         fpm_depends+=(fuse-libs)
         ;;
 esac
index 43c04a67e2c647cf0ce6942bb6633d309dbea3bc..952fb557c746ce53e6589ff6a6c1aac1bb977bed 100644 (file)
@@ -3,5 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 fpm_depends+=(ca-certificates)
+case "$TARGET" in
+    rocky*) fpm_depends+=(pam) ;;
+esac
 
 fpm_args+=(--conflicts=libpam-arvados)
index 570f4601c5d9bdeaf7220c6b4676be25a316875d..69c9474703c95e270221bc1f598f769efa589cca 100644 (file)
@@ -5,7 +5,7 @@
 fpm_depends+=('git >= 1.7.10')
 
 case "$TARGET" in
-    centos*)
+    centos*|rocky*)
         fpm_depends+=(libcurl-devel postgresql-devel bison make automake gcc gcc-c++ postgresql shared-mime-info)
         ;;
     ubuntu1804)
index f789abe69270c024e73a5294666bc06169b45026..4d98172f8db647ddbd5f40a49315d001a1c96065 100644 (file)
@@ -5,7 +5,7 @@
 fpm_depends+=(fuse)
 
 case "$TARGET" in
-    centos*)
+    centos*|rocky*)
         fpm_depends+=(fuse-libs)
         ;;
     debian* | ubuntu*)
index 6bcbf67fe0eb42cde628665455bc3c874a012951..41d020efe5208b34f3f842df7eb30e2f495836f6 100644 (file)
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 case "$TARGET" in
-    centos*)
+    centos*|rocky*)
         fpm_depends+=(mailcap)
         ;;
     debian* | ubuntu*)