#
# SPDX-License-Identifier: AGPL-3.0
+set -e
+
DEBUG=0
UNMANAGED=0
SSH_PORT=22
;;
-d | --debug)
DEBUG=1
+ set -x
shift
;;
--)
COLUMNS=80
PUPPET_AGENT='
-__rvm_unload
+if [[ -e "/usr/local/rvm/scripts/rvm" ]]; then
+ source /usr/local/rvm/scripts/rvm
+ __rvm_unload
+fi
now() { date +%s; }
let endtime="$(now) + 600"
while [ "$endtime" -gt "$(now)" ]; do
let endtime="$(now) + 600"
while [ "$endtime" -gt "$(now)" ]; do
apt-get update
- apt-get -y upgrade
+ DEBIAN_FRONTEND=noninteractive apt-get -y upgrade
apt_exitcode=$?
if [ 0 = "$apt_exitcode" ]; then
break
sleep $[ $RANDOM / 6000 ].$[ $RANDOM / 1000 ]
TMP_FILE=`mktemp`
if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$APT_AGENT'" | tee $TMP_FILE
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$APT_AGENT'" 2>&1 | sed 's/^/['"${node}"'] /' | tee $TMP_FILE
else
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$APT_AGENT'" > $TMP_FILE 2>&1
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$APT_AGENT'" 2>&1 | sed 's/^/['"${node}"'] /' > $TMP_FILE 2>&1
fi
ECODE=${PIPESTATUS[0]}
if [[ "$ECODE" == "0" ]]; then
rm -f $TMP_FILE
- echo $node successfully updated
+ title "$node successfully updated"
else
- echo $node exit code: $ECODE see $TMP_FILE for details
+ title "$node exit code: $ECODE see $TMP_FILE for details"
fi
}
sleep $[ $RANDOM / 6000 ].$[ $RANDOM / 1000 ]
TMP_FILE=`mktemp`
if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$PUPPET_AGENT'" | tee $TMP_FILE
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$PUPPET_AGENT'" 2>&1 | sed 's/^/['"${node}"'] /' | tee $TMP_FILE
else
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$PUPPET_AGENT'" > $TMP_FILE 2>&1
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 5" root@$node -C bash -c "'$PUPPET_AGENT'" 2>&1 | sed 's/^/['"${node}"'] /' > $TMP_FILE 2>&1
fi
ECODE=${PIPESTATUS[0]}
fi
}
-function run_command() {
- node=$1;shift
- return_var=$1;shift
- command=$@
-
- title "Running '$command' on $node"
- TMP_FILE=`mktemp`
- if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" root@$node -C "$command" | tee $TMP_FILE
- else
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" root@$node -C "$command" > $TMP_FILE 2>&1
- fi
-
- ECODE=$?
- RESULT=$(cat $TMP_FILE)
-
- if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
- # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
- # a shell node that is down.
- title "ERROR running command on $node: exit code $ECODE"
- if [[ "$DEBUG" == "0" ]]; then
- title "Command output follows:"
- echo $RESULT
- fi
- fi
- if [[ "$ECODE" == "255" ]]; then
- title "Connection timed out"
- ECODE=0
- fi
- rm -f $TMP_FILE
- eval "$return_var=$ECODE"
-}
-
if [[ "$NODE" == "" ]] || [[ "$NODE" == "$IDENTIFIER.arvadosapi.com" ]]; then
title "Updating API server"
SUM_ECODE=0
fi
title "Gathering list of nodes"
-if [[ "$IDENTIFIER" == "ce8i5" ]]; then
- start_nodes="keep keep0 shell"
- SHELL_NODE_FOR_ARV_KEEPDOCKER=""
-else
- SHELL_NODES=`ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv virtual_machine list |jq .items[].hostname -r`
- KEEP_NODES=`ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv keep_service list |jq .items[].service_host -r`
- SHELL_NODE_FOR_ARV_KEEPDOCKER="shell.$IDENTIFIER"
- start_nodes="workbench manage switchyard $SHELL_NODES $KEEP_NODES"
+start_nodes="workbench"
+if [[ "$IDENTIFIER" != "ce8i5" ]] && [[ "$IDENTIFIER" != "tordo" ]]; then
+ start_nodes="$start_nodes manage switchyard"
fi
+SHELL_NODES=`ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv virtual_machine list |jq .items[].hostname -r`
+KEEP_NODES=`ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv keep_service list |jq .items[].service_host -r`
+SHELL_NODE_FOR_ARV_KEEPDOCKER="shell.$IDENTIFIER"
+start_nodes="$start_nodes $SHELL_NODES $KEEP_NODES"
nodes=""
for n in $start_nodes; do
## at this point nodes should be an array containing
## manage.qr1hi, keep.qr1hi, etc
## that should be defined in the .ssh/config file
- title "Updating in parallel: $nodes"
+ title "Updating in parallel:$nodes"
export -f update_node
export -f run_puppet
export -f run_apt
title "Found Arvados Standard Docker Images project with uuid $DOCKER_IMAGES_PROJECT"
- VERSION=`ssh -o "StrictHostKeyChecking no" $IDENTIFIER apt-cache policy python-arvados-cwl-runner|grep Candidate`
- VERSION=`echo $VERSION|cut -f2 -d' '|cut -f1 -d-`
+ if [[ "$SHELL_NODE_FOR_ARV_KEEPDOCKER" == "" ]]; then
+ VERSION=`ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" -o "LogLevel QUIET" $IDENTIFIER apt-cache policy python3-arvados-cwl-runner|grep Candidate`
+ VERSION=`echo $VERSION|cut -f2 -d' '|cut -f1 -d-`
- if [[ "$?" != "0" ]] || [[ "$VERSION" == "" ]]; then
- title "ERROR: unable to get arvados/jobs Docker image version"
- exit 1
- else
- title "Found version for arvados/jobs Docker image: $VERSION"
- fi
+ if [[ "$?" != "0" ]] || [[ "$VERSION" == "" ]]; then
+ title "ERROR: unable to get python3-arvados-cwl-runner version"
+ exit 1
+ else
+ title "Found version for python3-arvados-cwl-runner: $VERSION"
+ fi
- if [[ "$SHELL_NODE_FOR_ARV_KEEPDOCKER" == "" ]]; then
- ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker |grep -q $VERSION
- if [[ "$?" == "0" ]]; then
- title "Found latest arvados/jobs Docker image, nothing to upload"
- # Just in case it isn't yet, tag the image as latest
- title "Tag arvados/jobs Docker image $VERSION as latest"
- ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs latest
- if [[ $? -ne 0 ]]; then
- title "'arv-keepdocker' failed..."
- exit 1
- fi
+ set +e
+ CLEAN_VERSION=`echo $VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g`
+ ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker |grep -qP "arvados/jobs +$CLEAN_VERSION "
+ if [[ $? -eq 0 ]]; then
+ set -e
+ title "Found arvados/jobs Docker image version $CLEAN_VERSION, nothing to upload"
else
- title "Installing latest arvados/jobs Docker image"
- ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --pull --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs $VERSION
- if [[ $? -ne 0 ]]; then
- title "'arv-keepdocker' failed..."
- exit 1
- fi
- ## adding latest tag too refs 9254
- docker tag arvados/jobs:$VERSION arvados/jobs:latest
- ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs latest
+ set -e
+ title "Installing arvados/jobs Docker image version $CLEAN_VERSION"
+ ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --pull --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs $CLEAN_VERSION
if [[ $? -ne 0 ]]; then
title "'arv-keepdocker' failed..."
exit 1
fi
fi
else
- run_command $SHELL_NODE_FOR_ARV_KEEPDOCKER ECODE "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker" |grep -q $VERSION
+ VERSION=`ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" -o "LogLevel QUIET" $SHELL_NODE_FOR_ARV_KEEPDOCKER apt-cache policy python3-arvados-cwl-runner|grep Candidate`
+ VERSION=`echo $VERSION|cut -f2 -d' '|cut -f1 -d-`
- if [[ "$?" == "0" ]]; then
- title "Found latest arvados/jobs Docker image, nothing to upload"
- # Just in case it isn't yet, tag the image as latest
- title "Tag arvados/jobs Docker image $VERSION as latest"
- ssh -o "StrictHostKeyChecking no" $SHELL_NODE_FOR_ARV_KEEPDOCKER "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs latest"
- if [[ $? -ne 0 ]]; then
- title "'arv-keepdocker' failed..."
- exit 1
- fi
+ if [[ "$?" != "0" ]] || [[ "$VERSION" == "" ]]; then
+ title "ERROR: unable to get python3-arvados-cwl-runner version"
+ exit 1
else
- title "Installing latest arvados/jobs Docker image"
- ssh -o "StrictHostKeyChecking no" $SHELL_NODE_FOR_ARV_KEEPDOCKER "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --pull --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs $VERSION"
- if [[ $? -ne 0 ]]; then
- title "'arv-keepdocker' failed..."
- exit 1
- fi
- ## adding latest tag too refs 9254
- ssh -o "StrictHostKeyChecking no" $SHELL_NODE_FOR_ARV_KEEPDOCKER docker tag arvados/jobs:$VERSION arvados/jobs:latest
- ssh -o "StrictHostKeyChecking no" $SHELL_NODE_FOR_ARV_KEEPDOCKER "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs latest"
+ title "Found version for python3-arvados-cwl-runner: $VERSION"
+ fi
+
+ set +e
+ CLEAN_VERSION=`echo $VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g`
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" -o "LogLevel QUIET" $SHELL_NODE_FOR_ARV_KEEPDOCKER "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker" |grep -qP "arvados/jobs +$CLEAN_VERSION "
+ if [[ $? -eq 0 ]]; then
+ set -e
+ title "Found arvados/jobs Docker image version $CLEAN_VERSION, nothing to upload"
+ else
+ set -e
+ title "Installing arvados/jobs Docker image version $CLEAN_VERSION"
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" -o "LogLevel QUIET" $SHELL_NODE_FOR_ARV_KEEPDOCKER "ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN arv-keepdocker --pull --project-uuid=$DOCKER_IMAGES_PROJECT arvados/jobs $CLEAN_VERSION"
if [[ $? -ne 0 ]]; then
title "'arv-keepdocker' failed..."
exit 1