Merge branch '21894-diagnostics-dev-version-mismatch'

[arvados.git] / build / run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 03d99b13274d233e8d3548a8631ded2566c3be6b..6bbfa36cdeaf5bd7a128423f881e09eb95186291 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -111,7 +111,7 @@ handle_ruby_gem() {
         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
 
         # -q appears to be broken in gem version 2.2.2
         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
 
         # -q appears to be broken in gem version 2.2.2

-        $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
+        gem build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"

     fi
 }
 
     fi
 }
 


@@ -121,6 +121,9 @@ package_workbench2() {
     local src=services/workbench2
     local dst=/var/www/arvados-workbench2/workbench2
     local description="Arvados Workbench 2"
     local src=services/workbench2
     local dst=/var/www/arvados-workbench2/workbench2
     local description="Arvados Workbench 2"

+    if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+        return 0
+    fi

     cd "$WORKSPACE/$src"
     local version="$(version_from_git)"
     rm -rf ./build
     cd "$WORKSPACE/$src"
     local version="$(version_from_git)"
     rm -rf ./build


@@ -443,34 +446,30 @@ test_package_presence() {
     # sure it gets picked up by the test and/or upload steps.
     # Get the list of packages from the repos
 
     # sure it gets picked up by the test and/or upload steps.
     # Get the list of packages from the repos
 

+    local pkg_url

     if [[ "$FORCE_BUILD" == "1" ]]; then
       echo "Package $full_pkgname build forced with --force-build, building"
     if [[ "$FORCE_BUILD" == "1" ]]; then
       echo "Package $full_pkgname build forced with --force-build, building"

+      return 0

     elif [[ "$FORMAT" == "deb" ]]; then
     elif [[ "$FORMAT" == "deb" ]]; then

-      declare -A dd
-      dd[debian11]=bullseye
-      dd[debian12]=bookworm
-      dd[ubuntu2004]=focal
-      dd[ubuntu2204]=jammy
-      D=${dd[$TARGET]}
+      local codename
+      case "$TARGET" in
+          debian11) codename=bullseye ;;
+          debian12) codename=bookworm ;;
+          ubuntu2004) codename=focal ;;
+          ubuntu2204) codename=jammy ;;
+          ubuntu2404) codename=noble ;;
+          *)
+              echo "FIXME: Don't know deb URL path for $TARGET, building"
+              return 0
+              ;;
+      esac
+      local repo_subdir

       if [ ${pkgname:0:3} = "lib" ]; then
         repo_subdir=${pkgname:0:4}
       else
         repo_subdir=${pkgname:0:1}
       fi
       if [ ${pkgname:0:3} = "lib" ]; then
         repo_subdir=${pkgname:0:4}
       else
         repo_subdir=${pkgname:0:1}
       fi

-
-      repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
-      echo "${repo_pkg_list}" |grep -q ${full_pkgname}
-      if [ $? -eq 0 ] ; then
-        echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
-        curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
-        return 1
-      elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
-        echo "Package $full_pkgname exists, not rebuilding!"
-        return 1
-      else
-        echo "Package $full_pkgname not found, building"
-        return 0
-      fi
+      pkg_url="http://apt.arvados.org/$codename/pool/main/$repo_subdir/$pkgname/$full_pkgname"

     else
       local rpm_root
       case "$TARGET" in
     else
       local rpm_root
       case "$TARGET" in


@@ -480,18 +479,18 @@ test_package_presence() {
           return 0
           ;;
       esac
           return 0
           ;;
       esac

-      local rpm_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
+      pkg_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
+    fi

 
 

-      if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$rpm_url"; then
-        echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
-        return 1
-      elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
-        echo "Package $full_pkgname exists, not rebuilding!"
-        return 1
-      else
-        echo "Package $full_pkgname not found, building"
-        return 0
-      fi
+    if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$pkg_url"; then
+      echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+      return 1
+    elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
+      echo "Package $full_pkgname exists, not rebuilding!"
+      return 1
+    else
+      echo "Package $full_pkgname not found, building"
+      return 0

     fi
 }
 
     fi
 }
 


@@ -513,8 +512,38 @@ handle_rails_package() {
         cd "$srcdir"
         mkdir -p tmp
         git rev-parse HEAD >git-commit.version
         cd "$srcdir"
         mkdir -p tmp
         git rev-parse HEAD >git-commit.version

+        # Please make sure you read `bundle help config` carefully before you
+        # modify any of these settings. Some of their names are not intuitive.
+        #
+        # `bundle cache` caches from Git and paths, not just rubygems.org.

         bundle config set cache_all true
         bundle config set cache_all true

-        bundle package
+        # Disallow changes to Gemfile.
+        bundle config set deployment true
+        # Avoid loading system-wide gems (although this seems to not work 100%).
+        bundle config set disable_shared_gems true
+        # `bundle cache` only downloads gems, doesn't install them.
+        # Our Rails postinst script does the install step.
+        bundle config set no_install true
+        # As of April 2024/Bundler 2.4, `bundle cache` seems to skip downloading
+        # gems that are already available system-wide... and then it complains
+        # that your bundle is incomplete. Work around this by fetching gems
+        # manually.
+        # TODO: Once all our supported distros have Ruby 3+, we can modify
+        # the awk script to print "NAME:VERSION" output, and pipe that directly
+        # to `xargs -0r gem fetch` for reduced overhead.
+        mkdir -p vendor/cache
+        awk -- '
+BEGIN { OFS="\0"; ORS="\0"; }
+(/^[A-Z ]*$/) { level1=$0; }
+(/^  [[:alpha:]]+:$/) { level2=substr($0, 3, length($0) - 3); next; }
+(/^ {0,3}[[:alpha:]]/) { level2=""; next; }
+(level1 == "GEM" && level2 == "specs" && NF == 2 && $1 ~ /^[[:alpha:]][-_[:alnum:]]*$/ && $2 ~ /\([[:digit:]]+[-_+.[:alnum:]]*\)$/) {
+    print "--version", substr($2, 2, length($2) - 2), $1;
+}
+' Gemfile.lock | env -C vendor/cache xargs -0r --max-args=3 gem fetch
+        # Despite the bug, we still run `bundle cache` to make sure Bundler is
+        # happy for later steps.
+        bundle cache

     )
     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
         echo "ERROR: $pkgname package prep failed" >&2
     )
     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
         echo "ERROR: $pkgname package prep failed" >&2


@@ -682,15 +711,15 @@ fpm_build_virtualenv_worker () {
       fi
   fi
   if [[ $? -ne 0 ]]; then
       fi
   fi
   if [[ $? -ne 0 ]]; then

-    printf "Error, unable to download/build wheel for %s @ %s" "$PKG" "$PKG_DIR"
-    exit 1
-  elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
-    printf "Error, unable to update local wheel repository"
+    printf "Error, unable to download/build wheel for %s @ %s\n" "$PKG" "$PKG_DIR"

     exit 1
   fi
 
   if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0
     exit 1
   fi
 
   if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0

+  elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
+    printf "Error, unable to update local wheel repository\n"
+    exit 1

   fi
 
   local venv_dir="$PYTHON_BUILDROOT/$PYTHON_PKG"
   fi
 
   local venv_dir="$PYTHON_BUILDROOT/$PYTHON_PKG"


@@ -708,7 +737,7 @@ fpm_build_virtualenv_worker () {
 
   local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
   if [[ -z "$pip_wheel" ]]; then
 
   local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
   if [[ -z "$pip_wheel" ]]; then

-    printf "Error, unable to find built wheel for $PKG"
+    printf "Error, unable to find built wheel for $PKG\n"

     exit 1
   elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
     printf "Error, unable to run
     exit 1
   elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
     printf "Error, unable to run