Merge branch '21521-aptly-flock'

Refs #21521.

Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

diff --git a/jenkins/packer-images/jenkins-image-common.sh b/jenkins/packer-images/jenkins-image-common.sh
index e68b092c0779fd178f3c757787f52ab09cc8f28e..9f3c4616fd61029704c12d5c188e639325b3aed8 100755 (executable)
--- a/jenkins/packer-images/jenkins-image-common.sh
+++ b/jenkins/packer-images/jenkins-image-common.sh
@@ -11,29 +11,31 @@ sudo su -c "echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCfRJenfxGPFuJ/W2KUs6Wf0wa
 sudo su -c "echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDH8swFWEfEfHhA+C5ezV8SXO/PkzGD1SH5VAQP/XDIrtnUocBZ3CE30lSyqYJI/EVKVqVa/ICQ0YpUwiMK6+3Jr9QQJwVyTmPji2nY3InL+1XAucN6HFJGKY9bYSsNOuKooj22GwBWw3gfJNLg/8qtpVykEq1yRpyh6pGsXT+J5nUZ723vZZTh//sxdN4CM8D8zoDgHc4RbL+zvESnCDrDbtMhg2u1h14RWiFOBAnzYuWcgtVDy2HA9iS0hJFB2UOV50byXLrEetxJ84PTwRsV2irq1y63g58VxwYOUrVZ08MY5qFvHExBjPqeqhRMzE7GufWM5F1CcUuGviOGFWfqMnfG4VOirPkFtRoK2oKRxH+NVPoUXWWxItJQ1dZ9hLDDWgAbxAvLS4Nnl2hvOVAbC7RVpXfoAhIPpL48oS1UprbsZIMxk2ZmRSJB1ykD3aLUvoO4zoD6xADt8uLiPvVYgFWUy1doLxHZqdY1Omc91owgQVPKvQ4vhqsJehQl4ZDS+O+8S7aC5m8sQ/V+NqiiXLH22vN58K7qNrkHWdb1n+rhilMbA5zp3cSKBgwmmNdupyPkJOKvf3IS7i4El+c8RFmRQv4FzGrdjGXAP8LPtt1dWPgHTFYjmrkOHLmfWM/y8cuyPWW/HEp3Y/msPQRlS3Gymce//vAWgN4T9yN46w== lucas@notebook" >> /home/jenkins/.ssh/authorized_keys
 
 # Install a few dependency packages
-# First, let's figure out the OS we're working on
-OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \" | cut -f 2 -d = )
-echo "Detected distro: ${OS_ID}"
-
-case ${OS_ID} in
-  centos)
-    PREINSTALL_CMD="/bin/true"
-    INSTALL_CMD="yum install -y"
-    POSTINSTALL_CMD="/bin/true"
-    PKGS="git nmap-ncat java-11-openjdk"
-    ;;
-  debian|ubuntu)
-    if [ ${OS_ID} = "debian" ]; then
-      echo "deb http://deb.debian.org/debian buster-backports main" | sudo tee /etc/apt/sources.list.d/buster-backports.list
-    fi
-    PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get update"
-    INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get install -y"
-    POSTINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get purge --autoremove -y"
-    # SUFFIX packages with - to remove them
-    # Remove unattended-upgrades so that it doesn't interfere with our nodes at startup
-    PKGS="git netcat-traditional default-jdk unattended-upgrades-"
-    ;;
-esac
+. /etc/os-release
+PREINSTALL_CMD='echo "error: unknown distro" >&2; false'
+for OS_ID in ${ID:-} ${ID_LIKE:-}; do
+  case ${OS_ID} in
+    rhel)
+      PREINSTALL_CMD="/bin/true"
+      INSTALL_CMD="yum install -y"
+      POSTINSTALL_CMD="/bin/true"
+      PKGS="git nmap-ncat java-11-openjdk"
+      break
+      ;;
+    debian)
+      if [[ "$VERSION_CODENAME" == buster ]]; then
+        echo "deb http://deb.debian.org/debian buster-backports main" | sudo tee /etc/apt/sources.list.d/buster-backports.list
+      fi
+      PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get update"
+      INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get install -y"
+      POSTINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt-get purge --autoremove -y"
+      # SUFFIX packages with - to remove them
+      # Remove unattended-upgrades so that it doesn't interfere with our nodes at startup
+      PKGS="git netcat-traditional default-jdk unattended-upgrades-"
+      break
+      ;;
+  esac
+done
 
 sudo su -c "${PREINSTALL_CMD}"
 sudo su -c "${INSTALL_CMD} ${PKGS}"

diff --git a/jenkins/run_upload_packages.py b/jenkins/run_upload_packages.py
index c6949e82844db7b98c6a36c28bb6807731a8de07..216b56b14958d10773ec74ad0b0f566b8beceb86 100755 (executable)
--- a/jenkins/run_upload_packages.py
+++ b/jenkins/run_upload_packages.py
@@ -205,14 +205,17 @@ class DistroPackageSuite(PackageSuite):
 
 class DebianPackageSuite(DistroPackageSuite):
     APT_SCRIPT = """
+set -e
 cd "$1"; shift
 DISTNAME=$1; shift
+# aptly implements its own locking, but its wait strategy as of April 2024 is
+# not patient enough to accommodate multiple simultaneous uploads.
+APTLY_LOCK="${XDG_RUNTIME_DIR:-/tmp}/aptly-upload.lock"
+aptly() {
+  flock --wait=300 "$APTLY_LOCK" aptly "$@"
+}
 for package in "$@"; do
-  set +e
-  aptly repo search "$DISTNAME" "${package%.deb}" >/dev/null 2>&1
-  RET=$?
-  set -e
-  if [[ $RET -eq 0 ]]; then
+  if aptly repo search "$DISTNAME" "${package%.deb}" >/dev/null 2>&1; then
     echo "Not adding $package, it is already present in repo $DISTNAME"
     rm "$package"
   else