#
# SPDX-License-Identifier: AGPL-3.0
-FROM centos:7
+ARG HOSTTYPE
+
+FROM centos:7 as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+
+FROM centos:7 as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
+ENV DEBIAN_FRONTEND noninteractive
+
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel fuse-devel xz-libs git wget pam-devel
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
&& make install \
&& ln -sf /usr/local/src/bash-4.4.12/bash /bin/bash
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
-
# Need to "touch" RPM database to workaround bug in interaction between
# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
RUN touch /var/lib/rpm/* && yum -q -y install python3 python3-pip python3-devel
# Install virtualenv
RUN /usr/bin/pip3 install 'virtualenv<20'
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
# The version of setuptools that comes with CentOS is way too old
RUN pip3 install 'setuptools<45'
#
# SPDX-License-Identifier: AGPL-3.0
+ARG HOSTTYPE
+
## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
-FROM debian:buster
+FROM debian:buster as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM debian:buster as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian10"]
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN echo $MAKE && git clone -b 17417-switch-to-mini_racer --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
#
# SPDX-License-Identifier: AGPL-3.0
-FROM ubuntu:bionic
+ARG HOSTTYPE
+
+FROM ubuntu:bionic as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM ubuntu:bionic as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev equivs
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
#
# SPDX-License-Identifier: AGPL-3.0
-FROM ubuntu:focal
+ARG HOSTTYPE
+
+FROM ubuntu:focal as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM ubuntu:focal as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2004"]
Build only a specific package
--only-test <package>
Test only a specific package
---only-arch <arch>
- Build only a specific architecture (amd64 or arm64)
+--arch <arch>
+ Build a specific architecture (amd64 or arm64, defaults to native architecture)
--force-build
Build even if the package exists upstream or if it has already been
built locally
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,only-arch:,build-version: \
+ help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
fi
TARGET=debian10
-ONLY_ARCH=
+ARCH=
FORCE_BUILD=0
COMMAND=
DEBUG=
--only-build)
ONLY_BUILD="$2"; shift
;;
- --only-arch)
- ONLY_ARCH="$2"; shift
+ --arch)
+ ARCH="$2"; shift
;;
--debug)
DEBUG=" --debug"
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
--env "FORCE_BUILD=$FORCE_BUILD" \
- --env "ONLY_ARCH=$ONLY_ARCH" \
+ --env "ARCH=$ARCH" \
"$IMAGE" $COMMAND
then
echo
Distribution to build packages for (default: debian10)
--only-build <package>
Build only a specific package (or $ONLY_BUILD from environment)
---only-arch <architecture>
- Build only a specific architecture (or $ONLY_ARCH from environment)
+--arch <arch>
+ Build a specific architecture (or $ARCH from environment, defaults to native architecture)
--force-build
Build even if the package exists upstream or if it has already been
built locally
COMMAND=
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,build-bundle-packages,debug,target:,only-build:,only-arch:,force-build \
+ help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--force-build)
FORCE_BUILD=1
;;
- --only-arch)
- ONLY_ARCH="$2"; shift
+ --arch)
+ ARCH="$2"; shift
;;
--debug)
DEBUG=1
native_arch="arm64"
fi
- if [[ -n "$ONLY_ARCH" ]]; then
- if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ONLY_ARCH" ]]; then
- package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ONLY_ARCH" "$license_file"
+ if [[ -n "$ARCH" ]]; then
+ if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ARCH" ]]; then
+ package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ARCH" "$license_file"
else
- echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ONLY_ARCH"
+ echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ARCH"
fi
else
archs=($native_arch)
fi
}
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [apache-2.0.txt]
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
package_go_binary_worker() {
local src_path="$1"; shift
local prog="$1"; shift
native_arch="arm64"
fi
- if [[ -n "$ONLY_ARCH" ]] && [[ "$ONLY_ARCH" == "$native_arch" ]]; then
- fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ONLY_ARCH"
- elif [[ -z "$ONLY_ARCH" ]]; then
+ if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
+ fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
+ elif [[ -z "$ARCH" ]]; then
for arch in $native_arch; do
fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
done
else
- echo "Error: no cross compilation support for Python yet, can not build $PKG for $ONLY_ARCH"
+ echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
fi
}