From 1c44466e5983383785eaf25f3b5da01936cd2cad Mon Sep 17 00:00:00 2001
From: Ward Vandewege <ward@curoverse.com>
Date: Wed, 22 Oct 2014 09:35:49 -0400
Subject: [PATCH] Make run-deploy.sh a bit smarter about which compute nodes
 are actually up.

Note: there are still race conditions to be addressed here. Or maybe
better, we probably just shouldn't care about compute node puppet runs
not completing succesfully. Either the node will get shut down by node
manager, or a regular hourly puppet run will DTRT.

No issue #
---
 jenkins/run-deploy.sh | 43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/jenkins/run-deploy.sh b/jenkins/run-deploy.sh
index a7ffac7..a55678b 100755
--- a/jenkins/run-deploy.sh
+++ b/jenkins/run-deploy.sh
@@ -136,24 +136,35 @@ fi
 
 title "Deploying workbench complete"
 
-# Update compute0
-title "Update compute0"
-
-ssh -p2222 root@compute0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
-
-if [[ "$ECODE" == "2" ]]; then
-  # Puppet exits '2' if there are changes. For real!
-  ECODE=0
-fi
-
-if [[ "$ECODE" != "0" ]]; then
-  title "!!!!!! Update compute0 FAILED !!!!!!"
-  EXITCODE=$(($EXITCODE + $ECODE))
+# Update compute node(s)
+title "Update compute node(s)"
+
+# Get list of nodes that are up
+COMPRESSED_NODE_LIST=`ssh -p2222 root@$IDENTIFIER -C "sinfo --long -p crypto -r -o "%N" -h"`
+
+if [[ "$COMPRESSED_NODE_LIST" != '' ]]; then
+  COMPUTE_NODES=`ssh -p2222 root@$IDENTIFIER -C "scontrol show hostname $COMPRESSED_NODE_LIST"`
+
+  SUM_ECODE=0
+  for node in $COMPUTE_NODES; do
+    echo "Updating $node.$IDENTIFIER"
+    ssh -p2222  -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t"
+    ECODE=$?
+    if [[ "$ECODE" != "255" && "$ECODE" != "2" && "$ECODE" != "0"  ]]; then
+      # 255 -> connection timed out. Just ignore that, it's possible the compute node was being shut down.
+      # Puppet exits '2' if there are changes. For real!
+      SUM_ECODE=$(($SUM_ECODE + $ECODE))
+      echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
+    fi
+  done
+
+  if [[ "$SUM_ECODE" != "0" ]]; then
+    title "!!!!!! Update compute node(s) FAILED !!!!!!"
+    EXITCODE=$(($EXITCODE + $SUM_ECODE))
+  fi
 fi
 
-title "Update compute0 complete"
+title "Update compute node(s) complete"
 
 title "Update shell"
 
-- 
2.30.2