-title "Deploying workbench complete"
-
-# Update compute node(s)
-title "Update compute node(s)"
-
-# Get list of nodes that are up
-COMPRESSED_NODE_LIST=`ssh -p2222 root@$IDENTIFIER -C "sinfo --long -p crypto -r -o "%N" -h"`
-
-if [[ "$COMPRESSED_NODE_LIST" != '' ]]; then
- COMPUTE_NODES=`ssh -p2222 root@$IDENTIFIER -C "scontrol show hostname $COMPRESSED_NODE_LIST"`
-
- SUM_ECODE=0
- for node in $COMPUTE_NODES; do
- echo "Updating $node.$IDENTIFIER"
- RESULT=`ssh -p2222 -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node.$IDENTIFIER -C "/usr/bin/puppet agent -t"`
- ECODE=$?
- if [[ "$ECODE" != "255" && ! ("$RESULT" =~ 'already in progress') && "$ECODE" != "2" && "$ECODE" != "0" ]]; then
- # 255 -> connection timed out. Just ignore that, it's possible the compute node was being shut down.
- # Puppet exits '2' if there are changes. For real!
- # Puppet prints 'Notice: Run of Puppet configuration client already in progress' if another puppet process was already running
- SUM_ECODE=$(($SUM_ECODE + $ECODE))
- echo "ERROR updating $node.$IDENTIFIER: exit code $ECODE"
- fi
- done
-
- if [[ "$SUM_ECODE" != "0" ]]; then
- title "!!!!!! Update compute node(s) FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $SUM_ECODE))
- fi
-fi
-
-title "Update compute node(s) complete"
-
-title "Update shell"
-
-ssh -p2222 root@shell.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
-
-if [[ "$ECODE" == "2" ]]; then
- # Puppet exits '2' if there are changes. For real!
- ECODE=0
-fi
-
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! Update shell FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
-fi
-
-title "Update shell complete"
-
-title "Update keep0"
-
-ssh -p2222 root@keep0.$IDENTIFIER -C "/usr/bin/puppet agent -t"
-
-ECODE=$?
-
-if [[ "$ECODE" == "2" ]]; then
- # Puppet exits '2' if there are changes. For real!