From: Ward Vandewege Date: Wed, 16 Mar 2022 12:43:42 +0000 (-0400) Subject: 18766: be a bit smarter about the preseeding of the go module cache and X-Git-Tag: 2.4.0~46^2~1 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/afec9ddba0b865918815ca3345278decc1fc6697 18766: be a bit smarter about the preseeding of the go module cache and the ruby gems in our docker images used for package building: make sure to preseed with the currently checked out branch of the source tree, not 'main'. This avoids potential compatibility issues between the version of Ruby and certain gems. The preseeding is worthwhile to speed up repeated package builds. When testing on an 8 core i7-10510U: * save ~60s on rails package build if the gems are fresh * save ~30s on go package build if the go mods are fresh * total gain for worst case scenario (workbench1 package) is almost 50% (85 seconds): ~3m down to ~1m35s Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh index 5ed33dc9f3..5f8817f20a 100755 --- a/build/package-build-dockerfiles/build-all-build-containers.sh +++ b/build/package-build-dockerfiles/build-all-build-containers.sh @@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do target=${target#./} echo $target cd $target - docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE . + docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) . cd .. done diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile index 01662d6ac5..e44d231edf 100644 --- a/build/package-build-dockerfiles/centos7/Dockerfile +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM centos:7 as build_x86_64 # Install go @@ -64,7 +65,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile index edd47cf69c..ed0a0cdc1f 100644 --- a/build/package-build-dockerfiles/debian10/Dockerfile +++ b/build/package-build-dockerfiles/debian10/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH ## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries FROM debian:buster as build_x86_64 @@ -51,7 +52,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile index cb4c695c4c..cfeaf2463a 100644 --- a/build/package-build-dockerfiles/debian11/Dockerfile +++ b/build/package-build-dockerfiles/debian11/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH ## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries FROM debian:bullseye as build_x86_64 @@ -56,7 +57,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile index b026fa2a88..9b20b41a4e 100644 --- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM ubuntu:bionic as build_x86_64 # Install go @@ -50,7 +51,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile index 1457670ce6..f28e6fef1d 100644 --- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM ubuntu:focal as build_x86_64 # Install go @@ -61,7 +62,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index e06a732979..c1cc2e5877 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -195,7 +195,7 @@ fi echo $TARGET cd $TARGET -time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE . +time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) . popd if test -z "$packages" ; then @@ -307,16 +307,24 @@ else set +e mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null set -e + # give bundle (almost) all the cores. See also the MAKE env var that is passed into the + # docker run command below. + # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ + tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX) + cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) + printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile # Build packages. if docker run \ --rm \ "${docker_volume_args[@]}" \ + -v $tmpfile:/root/.bundle/config \ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \ --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "ONLY_BUILD=$ONLY_BUILD" \ --env "FORCE_BUILD=$FORCE_BUILD" \ --env "ARCH=$ARCH" \ + --env "MAKE=make --jobs $cores" \ "$IMAGE" $COMMAND then echo @@ -325,6 +333,8 @@ else FINAL_EXITCODE=$? echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2 fi + # Clean up the bundle config file + rm -f $tmpfile fi if test -n "$package_fails" ; then