21700: Work around incomplete `bundle cache` bug

It seems like the problem I was working around for zlib 3.1.0 on Ubuntu
20.04 earlier can affect basically any gem at any step, if you happen to
already have the exact same version installed on the system. This commit
adds workarounds for all gems at all steps of the bundle setup.

Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

diff --git a/build/rails-package-scripts/postinst.sh b/build/rails-package-scripts/postinst.sh
index b72347aca3acda52308eaf78eee2de12b889627d..d3755a61d822c7bca0844ccdd29430511ab3a890 100644 (file)
--- a/build/rails-package-scripts/postinst.sh
+++ b/build/rails-package-scripts/postinst.sh
@@ -197,20 +197,15 @@ configure_version() {
   run_and_report "Running bundle config set --local path $SHARED_PATH/vendor_bundle" \
                  bundle config set --local path "$SHARED_PATH/vendor_bundle"
 
-  run_and_report "Running bundle install" \
-                 bundle install --prefer-local --quiet
-
-  # As of April 2024/Bundler 2.4, for some reason `bundle install` skips
-  # zlib if it's already installed as a system-wide gem, which it often will
-  # be because arvados gems pull it in. If this happened, install it in the
-  # bundle manually as a workaround.
-  if ! bin/bundle info zlib >/dev/null 2>&1; then
-      local RUBY_VERSION="$(ruby -e 'puts RUBY_VERSION')"
-      run_and_report "Adding zlib to bundle" \
-                     gem install \
-                     --install-dir="$SHARED_PATH/vendor_bundle/ruby/$RUBY_VERSION" \
-                     vendor/cache/zlib-*.gem
-  fi
+  # As of April 2024/Bundler 2.4, `bundle install` tends not to install gems
+  # which are already installed system-wide, which causes bundle activation to
+  # fail later. Work around this by installing all gems manually.
+  local gem_dir="$SHARED_PATH/vendor_bundle/ruby/$(ruby -e 'puts RUBY_VERSION')"
+  find vendor/cache -maxdepth 1 -name '*.gem' -print0 \
+      | run_and_report "Installing bundle gems" \
+                       xargs -0r gem install --quiet --install-dir="$gem_dir"
+  run_and_report "Running bundle install" bundle install --prefer-local --quiet
+  run_and_report "Verifying bundle is complete" bundle exec true
 
   echo -n "Ensuring directory and file permissions ..."
   # Ensure correct ownership of a few files

diff --git a/build/run-library.sh b/build/run-library.sh
index 314d231c8028e91f54259b4e26a23a779c74ff9b..2082dc94a0272d42d673de795ca244588f85a3a8 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -516,8 +516,34 @@ handle_rails_package() {
         cd "$srcdir"
         mkdir -p tmp
         git rev-parse HEAD >git-commit.version
+        # Please make sure you read `bundle help config` carefully before you
+        # modify any of these settings. Some of their names are not intuitive.
+        #
+        # `bundle cache` caches from Git and paths, not just rubygems.org.
         bundle config set cache_all true
-        bundle package
+        # Disallow changes to Gemfile.
+        bundle config set deployment true
+        # Avoid loading system-wide gems (although this seems to not work 100%).
+        bundle config set disable_shared_gems true
+        # `bundle cache` only downloads gems, doesn't install them.
+        # Our Rails postinst script does the install step.
+        bundle config set no_install true
+        # As of April 2024/Bundler 2.4, `bundle cache` seems to skip downloading
+        # gems that are already available system-wide... and then it complains
+        # that your bundle is incomplete. Work around this by fetching gems
+        # manually.
+        mkdir -p vendor/cache
+        awk -- '
+BEGIN { OFS=":"; ORS="\0"; }
+(/^[[:space:]]*$/) { level=0; }
+($0 == "GEM" || $0 == "  specs:") { level+=1; }
+(level == 2 && NF == 2 && $1 ~ /^[[:alpha:]][-_[:alnum:]]*$/ && $2 ~ /^\([[:digit:]]+[-_+.[:alnum:]]*\)$/) {
+    print $1, substr($2, 2, length($2) - 2);
+}
+' Gemfile.lock | env -C vendor/cache xargs -0r gem fetch
+        # Despite the bug, we still run `bundle cache` to make sure Bundler is
+        # happy for later steps.
+        bundle cache
     )
     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
         echo "ERROR: $pkgname package prep failed" >&2