From 421d6f9ea1b935527961f182b7e5cb00f9ef1b62 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Fri, 28 Jul 2023 13:48:08 -0400 Subject: [PATCH] 20797: Add initial support for rocky8 package builds Arvados-DCO-1.1-Signed-off-by: Brett Smith --- .licenseignore | 1 + apps/workbench/fpm-info.sh | 2 +- build/package-build-dockerfiles/Makefile | 6 +- .../rocky8/Dockerfile | 97 +++++++++++++++++++ build/package-test-dockerfiles/Makefile | 6 +- .../rocky8/Dockerfile | 48 +++++++++ .../rocky8/localrepo.repo | 5 + .../common-test-rails-server-package.sh | 4 + .../rpm-common-test-packages.sh | 13 ++- build/package-testing/test-packages-rocky8.sh | 1 + build/run-build-packages.sh | 7 ++ build/run-library.sh | 23 +++-- cmd/arvados-client/fpm-info.sh | 2 +- lib/pam/fpm-info.sh | 3 + services/api/fpm-info.sh | 2 +- services/fuse/fpm-info.sh | 2 +- services/keep-web/fpm-info.sh | 2 +- 17 files changed, 206 insertions(+), 18 deletions(-) create mode 100644 build/package-build-dockerfiles/rocky8/Dockerfile create mode 100644 build/package-test-dockerfiles/rocky8/Dockerfile create mode 100644 build/package-test-dockerfiles/rocky8/localrepo.repo create mode 120000 build/package-testing/test-packages-rocky8.sh diff --git a/.licenseignore b/.licenseignore index 361f09b4e3..36d2876c55 100644 --- a/.licenseignore +++ b/.licenseignore @@ -8,6 +8,7 @@ AUTHORS */bootstrap.js *bootstrap-theme.css build/package-test-dockerfiles/centos7/localrepo.repo +build/package-test-dockerfiles/rocky8/localrepo.repo build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados *by-sa-3.0.html *by-sa-3.0.txt diff --git a/apps/workbench/fpm-info.sh b/apps/workbench/fpm-info.sh index a09638a35e..a933fb702b 100644 --- a/apps/workbench/fpm-info.sh +++ b/apps/workbench/fpm-info.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 case "$TARGET" in - centos*) + centos*|rocky*) fpm_depends+=(git bison make automake gcc gcc-c++ graphviz shared-mime-info) ;; ubuntu1804) diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile index b8f6315e79..416e746d20 100644 --- a/build/package-build-dockerfiles/Makefile +++ b/build/package-build-dockerfiles/Makefile @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 SHELL := '/bin/bash' -all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated +all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated @@ -17,6 +17,10 @@ debian11/generated: common-generated-all test -d debian11/generated || mkdir debian11/generated cp -f -rlt debian11/generated common-generated/* +rocky8/generated: common-generated-all + test -d rocky8/generated || mkdir rocky8/generated + cp -f -rlt rocky8/generated common-generated/* + ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated cp -f -rlt ubuntu1804/generated common-generated/* diff --git a/build/package-build-dockerfiles/rocky8/Dockerfile b/build/package-build-dockerfiles/rocky8/Dockerfile new file mode 100644 index 0000000000..0fd4002fbd --- /dev/null +++ b/build/package-build-dockerfiles/rocky8/Dockerfile @@ -0,0 +1,97 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +ARG HOSTTYPE +ARG BRANCH +ARG GOVERSION + +FROM rockylinux:8.6-minimal as build_x86_64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/ + +FROM rockylinux:8.6-minimal as build_aarch64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/ + +FROM build_${HOSTTYPE} + +MAINTAINER Arvados Package Maintainers + +# Install dependencies. +RUN microdnf --enablerepo=devel install \ + automake \ + bison \ + bzip2 \ + fuse-devel \ + gcc \ + gcc-c++ \ + git \ + libattr-devel \ + libcurl-devel \ + libffi-devel \ + libtool \ + libyaml-devel \ + make \ + nss-devel \ + openssl-devel \ + pam-devel \ + patch \ + postgresql-devel \ + procps-ng \ + python3 \ + python3-devel \ + python3-pip \ + python3-virtualenv \ + readline-devel \ + rpm-build \ + ruby \ + sqlite-devel \ + tar \ + unzip \ + wget \ + which \ + xz-libs \ + zlib-devel + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ + echo "gem: --no-document" >> ~/.gemrc && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + +RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) +# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ +ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" + +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. +RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ + cd /tmp/arvados/services/api && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados/apps/workbench && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados && \ + go mod download + +ENV WORKSPACE /arvados +CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "rocky8"] diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile index 849decb9a5..96a80055ac 100644 --- a/build/package-test-dockerfiles/Makefile +++ b/build/package-test-dockerfiles/Makefile @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0 -all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated +all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated @@ -16,6 +16,10 @@ debian11/generated: common-generated-all test -d debian11/generated || mkdir debian11/generated cp -f -rlt debian11/generated common-generated/* +rocky8/generated: common-generated-all + test -d rocky8/generated || mkdir rocky8/generated + cp -f -rlt rocky8/generated common-generated/* + ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated cp -f -rlt ubuntu1804/generated common-generated/* diff --git a/build/package-test-dockerfiles/rocky8/Dockerfile b/build/package-test-dockerfiles/rocky8/Dockerfile new file mode 100644 index 0000000000..e1c6f27ec8 --- /dev/null +++ b/build/package-test-dockerfiles/rocky8/Dockerfile @@ -0,0 +1,48 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +FROM rockylinux:8.6-minimal +MAINTAINER Arvados Package Maintainers + +# Install dependencies. +RUN microdnf --enablerepo=devel install \ + autoconf \ + automake \ + bison \ + bzip2 \ + cpio \ + diffutils \ + findutils \ + gcc-c++ \ + glibc-devel \ + glibc-headers \ + gzip \ + libffi-devel \ + libtool \ + make \ + openssl-devel \ + patch \ + procps-ng \ + python3 \ + readline-devel \ + ruby \ + shadow-utils \ + sqlite-devel \ + tar \ + wget \ + which \ + zlib-devel + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN touch /var/lib/rpm/* && \ + gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9 + +COPY localrepo.repo /etc/yum.repos.d/localrepo.repo diff --git a/build/package-test-dockerfiles/rocky8/localrepo.repo b/build/package-test-dockerfiles/rocky8/localrepo.repo new file mode 100644 index 0000000000..a4f6ab3b8a --- /dev/null +++ b/build/package-test-dockerfiles/rocky8/localrepo.repo @@ -0,0 +1,5 @@ +[localrepo] +name=Arvados Test +baseurl=file:///arvados/packages/rocky8 +gpgcheck=0 +enabled=1 diff --git a/build/package-testing/common-test-rails-server-package.sh b/build/package-testing/common-test-rails-server-package.sh index e04556bb6b..a60fcc4652 100755 --- a/build/package-testing/common-test-rails-server-package.sh +++ b/build/package-testing/common-test-rails-server-package.sh @@ -47,6 +47,10 @@ case "$TARGET" in yum install --assumeyes httpd yum reinstall --assumeyes "$PACKAGE_NAME" ;; + rocky*) + microdnf install httpd + microdnf reinstall "$PACKAGE_NAME" + ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 exit 1 diff --git a/build/package-testing/rpm-common-test-packages.sh b/build/package-testing/rpm-common-test-packages.sh index 12450dd4f9..7cd2be7cd3 100755 --- a/build/package-testing/rpm-common-test-packages.sh +++ b/build/package-testing/rpm-common-test-packages.sh @@ -14,16 +14,23 @@ if [[ "$DEBUG" != "0" ]]; then STDERR_IF_DEBUG=/dev/stderr fi -target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b') +target="$(basename "$0" .sh)" +target="${target##*-}" -yum -q clean all +case "$target" in + centos*) yum -q clean all ;; + rocky*) microdnf clean all ;; +esac touch /var/lib/rpm/* export ARV_PACKAGES_DIR="/arvados/packages/$target" rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before" -yum install --assumeyes -e 0 $1 +case "$target" in + centos*) yum install --assumeyes -e 0 $1 ;; + rocky*) microdnf install $1 ;; +esac rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after" diff --git a/build/package-testing/test-packages-rocky8.sh b/build/package-testing/test-packages-rocky8.sh new file mode 120000 index 0000000000..64ef6040c1 --- /dev/null +++ b/build/package-testing/test-packages-rocky8.sh @@ -0,0 +1 @@ +rpm-common-test-packages.sh \ No newline at end of file diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index aded25b592..960932dbe3 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -141,6 +141,13 @@ case "$TARGET" in PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages export PYCURL_SSL_LIBRARY=nss ;; + rocky*) + FORMAT=rpm + PYTHON3_PACKAGE=$(rpm -qf "$(which python"$PYTHON3_VERSION")" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=python3 + PYTHON3_PREFIX=/usr + PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages + ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 exit 1 diff --git a/build/run-library.sh b/build/run-library.sh index c2466faac0..49cab6a0f1 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -919,11 +919,6 @@ fpm_build_virtualenv_worker () { LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"` COMMAND_ARR+=('--license' "$LICENSE_STRING") - if [[ "$package_format" == "rpm" ]]; then - # Make sure to conflict with the old rh-python36 packages we used to publish - COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG") - fi - if [[ "$DEBUG" != "0" ]]; then COMMAND_ARR+=('--verbose' '--log' 'info') fi @@ -940,9 +935,21 @@ fpm_build_virtualenv_worker () { fi COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE") - - # avoid warning - COMMAND_ARR+=('--deb-no-default-config-files') + case "$package_format" in + deb) + COMMAND_ARR+=( + # Avoid warning + --deb-no-default-config-files + ) ;; + rpm) + COMMAND_ARR+=( + # Conflict with older packages we used to publish + --conflicts "rh-python36-python-$PKG" + # Do not generate /usr/lib/.build-id links on RH8+ + # (otherwise our packages conflict with platform-python) + --rpm-rpmbuild-define "_build_id_links none" + ) ;; + esac # Append --depends X and other arguments specified by fpm-info.sh in # the package source dir. These are added last so they can override diff --git a/cmd/arvados-client/fpm-info.sh b/cmd/arvados-client/fpm-info.sh index 0e4415a83f..c717632452 100644 --- a/cmd/arvados-client/fpm-info.sh +++ b/cmd/arvados-client/fpm-info.sh @@ -5,7 +5,7 @@ fpm_depends+=(fuse) case "$TARGET" in - centos*) + centos*|rocky*) fpm_depends+=(fuse-libs) ;; esac diff --git a/lib/pam/fpm-info.sh b/lib/pam/fpm-info.sh index 43c04a67e2..952fb557c7 100644 --- a/lib/pam/fpm-info.sh +++ b/lib/pam/fpm-info.sh @@ -3,5 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 fpm_depends+=(ca-certificates) +case "$TARGET" in + rocky*) fpm_depends+=(pam) ;; +esac fpm_args+=(--conflicts=libpam-arvados) diff --git a/services/api/fpm-info.sh b/services/api/fpm-info.sh index 570f4601c5..69c9474703 100644 --- a/services/api/fpm-info.sh +++ b/services/api/fpm-info.sh @@ -5,7 +5,7 @@ fpm_depends+=('git >= 1.7.10') case "$TARGET" in - centos*) + centos*|rocky*) fpm_depends+=(libcurl-devel postgresql-devel bison make automake gcc gcc-c++ postgresql shared-mime-info) ;; ubuntu1804) diff --git a/services/fuse/fpm-info.sh b/services/fuse/fpm-info.sh index f789abe692..4d98172f8d 100644 --- a/services/fuse/fpm-info.sh +++ b/services/fuse/fpm-info.sh @@ -5,7 +5,7 @@ fpm_depends+=(fuse) case "$TARGET" in - centos*) + centos*|rocky*) fpm_depends+=(fuse-libs) ;; debian* | ubuntu*) diff --git a/services/keep-web/fpm-info.sh b/services/keep-web/fpm-info.sh index 6bcbf67fe0..41d020efe5 100644 --- a/services/keep-web/fpm-info.sh +++ b/services/keep-web/fpm-info.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 case "$TARGET" in - centos*) + centos*|rocky*) fpm_depends+=(mailcap) ;; debian* | ubuntu*) -- 2.30.2