- title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' on $node"
- TMP_FILE=`mktemp`
- if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
- ECODE=$?
+ if [[ "$LOCAL" == "0" ]]; then
+ title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
+ TMP_FILE=`mktemp`
+ if [[ "$DEBUG" != "0" ]]; then
+ echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
+ ECODE=$?
+ else
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
+ ECODE=$?
+ fi
+
+ if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
+ # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
+ # a shell node that is down.
+ title "ERROR running command on $node: exit code $ECODE"
+ if [[ "$DEBUG" == "0" ]]; then
+ title "Command output follows:"
+ cat $TMP_FILE
+ fi
+ fi
+ if [[ "$ECODE" == "255" ]]; then
+ title "Connection denied or timed out"
+ fi