X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e792b65d396b15b2c6c90ab2ab70d19c81f98460..6fe37f77904b854774699acbe6858047221cd377:/build/run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 6380e1cb19..bebd5b8cbf 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -111,7 +111,7 @@ handle_ruby_gem() {
         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
 
         # -q appears to be broken in gem version 2.2.2
-        $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
+        gem build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
     fi
 }
 
@@ -121,6 +121,9 @@ package_workbench2() {
     local src=services/workbench2
     local dst=/var/www/arvados-workbench2/workbench2
     local description="Arvados Workbench 2"
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+        return 0
+    fi
     cd "$WORKSPACE/$src"
     local version="$(version_from_git)"
     rm -rf ./build
@@ -262,6 +265,13 @@ package_go_binary_worker() {
       binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
     fi
 
+    case "$package_format" in
+        # As of April 2024 we package identical Go binaries under different
+        # packages and names. This upsets the build id database, so don't
+        # register ourselves there.
+        rpm) switches+=(--rpm-rpmbuild-define="_build_id_links none") ;;
+    esac
+
     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
     if [[ -e "${systemd_unit}" ]]; then
         switches+=(
@@ -436,34 +446,29 @@ test_package_presence() {
     # sure it gets picked up by the test and/or upload steps.
     # Get the list of packages from the repos
 
+    local pkg_url
     if [[ "$FORCE_BUILD" == "1" ]]; then
       echo "Package $full_pkgname build forced with --force-build, building"
+      return 0
     elif [[ "$FORMAT" == "deb" ]]; then
-      declare -A dd
-      dd[debian11]=bullseye
-      dd[debian12]=bookworm
-      dd[ubuntu2004]=focal
-      dd[ubuntu2204]=jammy
-      D=${dd[$TARGET]}
+      local codename
+      case "$TARGET" in
+          debian11) codename=bullseye ;;
+          debian12) codename=bookworm ;;
+          ubuntu2004) codename=focal ;;
+          ubuntu2204) codename=jammy ;;
+          *)
+              echo "FIXME: Don't know deb URL path for $TARGET, building"
+              return 0
+              ;;
+      esac
+      local repo_subdir
       if [ ${pkgname:0:3} = "lib" ]; then
         repo_subdir=${pkgname:0:4}
       else
         repo_subdir=${pkgname:0:1}
       fi
-
-      repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
-      echo "${repo_pkg_list}" |grep -q ${full_pkgname}
-      if [ $? -eq 0 ] ; then
-        echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
-        curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
-        return 1
-      elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
-        echo "Package $full_pkgname exists, not rebuilding!"
-        return 1
-      else
-        echo "Package $full_pkgname not found, building"
-        return 0
-      fi
+      pkg_url="http://apt.arvados.org/$codename/pool/main/$repo_subdir/$pkgname/$full_pkgname"
     else
       local rpm_root
       case "$TARGET" in
@@ -473,18 +478,18 @@ test_package_presence() {
           return 0
           ;;
       esac
-      local rpm_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
+      pkg_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
+    fi
 
-      if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$rpm_url"; then
-        echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
-        return 1
-      elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
-        echo "Package $full_pkgname exists, not rebuilding!"
-        return 1
-      else
-        echo "Package $full_pkgname not found, building"
-        return 0
-      fi
+    if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$pkg_url"; then
+      echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+      return 1
+    elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
+      echo "Package $full_pkgname exists, not rebuilding!"
+      return 1
+    else
+      echo "Package $full_pkgname not found, building"
+      return 0
     fi
 }
 
@@ -506,8 +511,38 @@ handle_rails_package() {
         cd "$srcdir"
         mkdir -p tmp
         git rev-parse HEAD >git-commit.version
+        # Please make sure you read `bundle help config` carefully before you
+        # modify any of these settings. Some of their names are not intuitive.
+        #
+        # `bundle cache` caches from Git and paths, not just rubygems.org.
         bundle config set cache_all true
-        bundle package
+        # Disallow changes to Gemfile.
+        bundle config set deployment true
+        # Avoid loading system-wide gems (although this seems to not work 100%).
+        bundle config set disable_shared_gems true
+        # `bundle cache` only downloads gems, doesn't install them.
+        # Our Rails postinst script does the install step.
+        bundle config set no_install true
+        # As of April 2024/Bundler 2.4, `bundle cache` seems to skip downloading
+        # gems that are already available system-wide... and then it complains
+        # that your bundle is incomplete. Work around this by fetching gems
+        # manually.
+        # TODO: Once all our supported distros have Ruby 3+, we can modify
+        # the awk script to print "NAME:VERSION" output, and pipe that directly
+        # to `xargs -0r gem fetch` for reduced overhead.
+        mkdir -p vendor/cache
+        awk -- '
+BEGIN { OFS="\0"; ORS="\0"; }
+(/^[A-Z ]*$/) { level1=$0; }
+(/^  [[:alpha:]]+:$/) { level2=substr($0, 3, length($0) - 3); next; }
+(/^ {0,3}[[:alpha:]]/) { level2=""; next; }
+(level1 == "GEM" && level2 == "specs" && NF == 2 && $1 ~ /^[[:alpha:]][-_[:alnum:]]*$/ && $2 ~ /\([[:digit:]]+[-_+.[:alnum:]]*\)$/) {
+    print "--version", substr($2, 2, length($2) - 2), $1;
+}
+' Gemfile.lock | env -C vendor/cache xargs -0r --max-args=3 gem fetch
+        # Despite the bug, we still run `bundle cache` to make sure Bundler is
+        # happy for later steps.
+        bundle cache
     )
     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
         echo "ERROR: $pkgname package prep failed" >&2
@@ -675,15 +710,15 @@ fpm_build_virtualenv_worker () {
       fi
   fi
   if [[ $? -ne 0 ]]; then
-    printf "Error, unable to download/build wheel for %s @ %s" "$PKG" "$PKG_DIR"
-    exit 1
-  elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
-    printf "Error, unable to update local wheel repository"
+    printf "Error, unable to download/build wheel for %s @ %s\n" "$PKG" "$PKG_DIR"
     exit 1
   fi
 
   if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0
+  elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
+    printf "Error, unable to update local wheel repository\n"
+    exit 1
   fi
 
   local venv_dir="$PYTHON_BUILDROOT/$PYTHON_PKG"
@@ -701,7 +736,7 @@ fpm_build_virtualenv_worker () {
 
   local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
   if [[ -z "$pip_wheel" ]]; then
-    printf "Error, unable to find built wheel for $PKG"
+    printf "Error, unable to find built wheel for $PKG\n"
     exit 1
   elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
     printf "Error, unable to run