18766: be a bit smarter about the preseeding of the go module cache and

the ruby gems in our docker images used for package building: make sure
to preseed with the currently checked out branch of the source tree,
not 'main'. This avoids potential compatibility issues between the
version of Ruby and certain gems.

The preseeding is worthwhile to speed up repeated package builds. When
testing on an 8 core i7-10510U:

* save ~60s on rails package build if the gems are fresh
* save ~30s on go package build if the go mods are fresh
* total gain for worst case scenario (workbench1 package) is almost 50%
 (85 seconds): ~3m down to ~1m35s

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh
index 5ed33dc9f3ac5f22b4899577f1b0244f5bf18e6f..5f8817f20a8521c3bf914a061611b6f89afdc2b8 100755 (executable)
--- a/build/package-build-dockerfiles/build-all-build-containers.sh
+++ b/build/package-build-dockerfiles/build-all-build-containers.sh
@@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do
   target=${target#./}
   echo $target
   cd $target
-  docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
+  docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
   cd ..
 done
 

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 01662d6ac5cc75e3835cc3823049dfd907b18bf7..e44d231edf4a74171f56975fe5b029fca0315ff3 100644 (file)
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM centos:7 as build_x86_64
 # Install go
@@ -64,7 +65,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \

diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile
index edd47cf69c422c54db39da524434d6208feda625..ed0a0cdc1f7af4fdc850673a848f89cfb3a0a089 100644 (file)
--- a/build/package-build-dockerfiles/debian10/Dockerfile
+++ b/build/package-build-dockerfiles/debian10/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 ## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries
 FROM debian:buster as build_x86_64
@@ -51,7 +52,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \

diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile
index cb4c695c4c330fae56118f79bd1d327541961068..cfeaf2463a7c7a26155fbdad46f75b3e2da19092 100644 (file)
--- a/build/package-build-dockerfiles/debian11/Dockerfile
+++ b/build/package-build-dockerfiles/debian11/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 ## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
 FROM debian:bullseye as build_x86_64
@@ -56,7 +57,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \

diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index b026fa2a88a694e7d4c1a8dfecb9d455da46389d..9b20b41a4e9a553c532f683c54e8970a36661c5e 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM ubuntu:bionic as build_x86_64
 # Install go
@@ -50,7 +51,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \

diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
index 1457670ce63f77d3dc0cb2e44d960041e833156c..f28e6fef1d8baf8fb6acc517abed728fb1bbb7f8 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 ARG HOSTTYPE
+ARG BRANCH
 
 FROM ubuntu:focal as build_x86_64
 # Install go
@@ -61,7 +62,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr
 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
 
+# Preseed the go module cache and the ruby gems, using the currently checked
+# out branch of the source tree. This avoids potential compatibility issues
+# between the version of Ruby and certain gems.
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+    cd /tmp/arvados && \
+    if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \
     cd /tmp/arvados/services/api && \
     /usr/local/rvm/bin/rvm-exec default bundle install && \
     cd /tmp/arvados/apps/workbench && \

diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index e06a7329790e3dfa7b1430e33c52a2b86c383cc2..c1cc2e5877b6a695fb25605585e1c0e9a79ac041 100755 (executable)
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -195,7 +195,7 @@ fi
 
 echo $TARGET
 cd $TARGET
-time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) .
 popd
 
 if test -z "$packages" ; then
@@ -307,16 +307,24 @@ else
     set +e
     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
     set -e
+    # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
+    # docker run command below.
+    # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+    tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
+    cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+    printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
     # Build packages.
     if docker run \
         --rm \
         "${docker_volume_args[@]}" \
+        -v $tmpfile:/root/.bundle/config \
         --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
         --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
         --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
         --env "ONLY_BUILD=$ONLY_BUILD" \
         --env "FORCE_BUILD=$FORCE_BUILD" \
         --env "ARCH=$ARCH" \
+        --env "MAKE=make --jobs $cores" \
         "$IMAGE" $COMMAND
     then
         echo
@@ -325,6 +333,8 @@ else
         FINAL_EXITCODE=$?
         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
     fi
+    # Clean up the bundle config file
+    rm -f $tmpfile
 fi
 
 if test -n "$package_fails" ; then