X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/ad3df9216ff20898eb96353e5d6c52b271a87375..7490e64759f56a8da52c8259da8b96baa769a6fe:/jenkins/run-deploy.sh

diff --git a/jenkins/run-deploy.sh b/jenkins/run-deploy.sh
index a7ffac7..d4e31f1 100755
--- a/jenkins/run-deploy.sh
+++ b/jenkins/run-deploy.sh
@@ -1,15 +1,37 @@
 #!/bin/bash
 
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Deploy Arvados to a cluster
+
+Syntax:
+        WORKSPACE=/path/to/arvados $(basename $0) <identifier> <deploy_repo_name>
+
+Options:
+
+identifier             Arvados cluster name
+deploy_repo_name       Name for the repository with the (capistrano) deploy scripts
+
+WORKSPACE=path         Path to the Arvados source tree to deploy from
+
+EOF
+
+
 IDENTIFIER=$1
 DEPLOY_REPO=$2
 
-if [[ "$IDENTIFIER" == '' ]]; then
-  echo "Syntax: $0 <identifier> <deploy_repo_name>"
+if [[ "$IDENTIFIER" == '' || "$DEPLOY_REPO" == '' ]]; then
+  echo >&2 "$helpmessage"
+  echo >&2
   exit 1
 fi
 
-if [[ "$DEPLOY_REPO" == '' ]]; then
-  echo "Syntax: $0 <identifier> <deploy_repo_name>"
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+  echo >&2 "$helpmessage"
+  echo >&2
+  echo >&2 "Error: WORKSPACE environment variable not set"
+  echo >&2
   exit 1
 fi
 
@@ -33,6 +55,29 @@ rvm use default
 # Just say what version of ruby we're running
 ruby --version
 
+function run_puppet() {
+  node=$1
+  return_var=$2
+
+  TMP_FILE=`mktemp`
+  ssh -t -p2222 -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t" | tee $TMP_FILE
+
+  ECODE=$?
+  RESULT=$(cat $TMP_FILE)
+
+  if [[ "$ECODE" != "255" && ! ("$RESULT" =~ 'already in progress') && "$ECODE" != "2" && "$ECODE" != "0"  ]]; then
+    # Puppet exists 255 if the connection timed out. Just ignore that, it's possible that this node is
+    #   a compute node that was being shut down.
+    # Puppet exits 2 if there are changes. For real!
+    # Puppet prints 'Notice: Run of Puppet configuration client already in progress' if another puppet process
+    #   was already running
+    echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
+  fi
+  rm -f $TMP_FILE
+  echo
+  eval "$return_var=$ECODE"
+}
+
 function ensure_symlink() {
   if [[ ! -L $WORKSPACE/$1 ]]; then
     ln -s $WORKSPACE/$DEPLOY_REPO/$1 $WORKSPACE/$1
@@ -136,30 +181,33 @@ fi
 
 title "Deploying workbench complete"
 
-# Update compute0
-title "Update compute0"
+# Update compute node(s)
+title "Update compute node(s)"
 
-ssh -p2222 root@compute0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
+# Get list of nodes that are up
+COMPRESSED_NODE_LIST=`ssh -p2222 root@$IDENTIFIER -C "sinfo --long -p crypto -r -o "%N" -h"`
 
-ECODE=$?
+if [[ "$COMPRESSED_NODE_LIST" != '' ]]; then
+  COMPUTE_NODES=`ssh -p2222 root@$IDENTIFIER -C "scontrol show hostname $COMPRESSED_NODE_LIST"`
 
-if [[ "$ECODE" == "2" ]]; then
-  # Puppet exits '2' if there are changes. For real!
-  ECODE=0
-fi
+  SUM_ECODE=0
+  for node in $COMPUTE_NODES; do
+    echo "Updating $node.$IDENTIFIER"
+    run_puppet $node ECODE
+    SUM_ECODE=$(($SUM_ECODE + $ECODE))
+  done
 
-if [[ "$ECODE" != "0" ]]; then
-  title "!!!!!! Update compute0 FAILED !!!!!!"
-  EXITCODE=$(($EXITCODE + $ECODE))
+  if [[ "$SUM_ECODE" != "0" ]]; then
+    title "!!!!!! Update compute node(s) FAILED !!!!!!"
+    EXITCODE=$(($EXITCODE + $SUM_ECODE))
+  fi
 fi
 
-title "Update compute0 complete"
+title "Update compute node(s) complete"
 
 title "Update shell"
 
-ssh -p2222 root@shell.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
+run_puppet shell ECODE
 
 if [[ "$ECODE" == "2" ]]; then
   # Puppet exits '2' if there are changes. For real!
@@ -175,9 +223,7 @@ title "Update shell complete"
 
 title "Update keep0"
 
-ssh -p2222 root@keep0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
+run_puppet keep0 ECODE
 
 if [[ "$ECODE" == "2" ]]; then
   # Puppet exits '2' if there are changes. For real!