the ruby gems in our docker images used for package building: make sure
to preseed with the currently checked out branch of the source tree,
not 'main'. This avoids potential compatibility issues between the
version of Ruby and certain gems.
The preseeding is worthwhile to speed up repeated package builds. When
testing on an 8 core i7-10510U:
* save ~60s on rails package build if the gems are fresh
* save ~30s on go package build if the go mods are fresh
* total gain for worst case scenario (workbench1 package) is almost 50%
(85 seconds): ~3m down to ~1m35s
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>
target=${target#./}
echo $target
cd $target
target=${target#./}
echo $target
cd $target
- docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
+ docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
FROM centos:7 as build_x86_64
# Install go
FROM centos:7 as build_x86_64
# Install go
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
FROM debian:buster as build_x86_64
## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
FROM debian:buster as build_x86_64
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
FROM debian:bullseye as build_x86_64
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
FROM debian:bullseye as build_x86_64
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
FROM ubuntu:bionic as build_x86_64
# Install go
FROM ubuntu:bionic as build_x86_64
# Install go
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
# SPDX-License-Identifier: AGPL-3.0
ARG HOSTTYPE
FROM ubuntu:focal as build_x86_64
# Install go
FROM ubuntu:focal as build_x86_64
# Install go
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados && \
+ if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
cd /tmp/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle install && \
cd /tmp/arvados/apps/workbench && \
-time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
popd
if test -z "$packages" ; then
popd
if test -z "$packages" ; then
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
+ # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
+ # docker run command below.
+ # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
+ cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+ printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
# Build packages.
if docker run \
--rm \
"${docker_volume_args[@]}" \
# Build packages.
if docker run \
--rm \
"${docker_volume_args[@]}" \
+ -v $tmpfile:/root/.bundle/config \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
--env "FORCE_BUILD=$FORCE_BUILD" \
--env "ARCH=$ARCH" \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
--env "FORCE_BUILD=$FORCE_BUILD" \
--env "ARCH=$ARCH" \
+ --env "MAKE=make --jobs $cores" \
"$IMAGE" $COMMAND
then
echo
"$IMAGE" $COMMAND
then
echo
FINAL_EXITCODE=$?
echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
fi
FINAL_EXITCODE=$?
echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
fi
+ # Clean up the bundle config file
+ rm -f $tmpfile
fi
if test -n "$package_fails" ; then
fi
if test -n "$package_fails" ; then