DEBUG=0
SSH_PORT=22
-ACCT=ci
+JOBS=1
+ACCT=
+LOCAL=0
function usage {
echo >&2
echo >&2 " -d, --debug Enable debug output"
echo >&2 " -h, --help Display this help and exit"
echo >&2 " -s, --scopes Print required scopes to run tests"
+ echo >&2 " -j, --jobs <jobs> Allow N jobs at once; 1 job with no arg."
+ echo >&2 " -l, --local Run arvados-cwl-runner locally, not on shell.<identifier>"
echo >&2
}
}
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hdp:s \
- --long help,scopes,debug,port:,acct: \
+TEMP=`getopt -o hdlp:sj: \
+ --long help,scopes,debug,local,port:,acct:,jobs: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
print_scopes
exit 0
;;
+ -j | --jobs)
+ JOBS="$2"; shift 2
+ ;;
+ -l | --local)
+ LOCAL=1
+ shift
+ ;;
--)
shift
break
return_var=$2
command=$3
- title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' on $node"
- TMP_FILE=`mktemp`
- if [[ "$DEBUG" != "0" ]]; then
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
- ECODE=$?
+ if [[ "$LOCAL" == "0" ]]; then
+ title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
+ TMP_FILE=`mktemp`
+ if [[ "$DEBUG" != "0" ]]; then
+ echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
+ ECODE=$?
+ else
+ ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
+ ECODE=$?
+ fi
+
+ if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
+ # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
+ # a shell node that is down.
+ title "ERROR running command on $node: exit code $ECODE"
+ if [[ "$DEBUG" == "0" ]]; then
+ title "Command output follows:"
+ cat $TMP_FILE
+ fi
+ fi
+ if [[ "$ECODE" == "255" ]]; then
+ title "Connection denied or timed out"
+ fi
else
- ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
- ECODE=$?
- fi
+ title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' locally"
+ TMP_FILE=`mktemp`
+ if [[ "$DEBUG" != "0" ]]; then
+ bash -c "$command" | tee $TMP_FILE
+ ECODE=$?
+ else
+ bash -c "$command" > $TMP_FILE 2>&1
+ ECODE=$?
+ fi
- if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
- # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
- # a shell node that is down.
- title "ERROR running command on $node: exit code $ECODE"
- if [[ "$DEBUG" == "0" ]]; then
- title "Command output follows:"
- cat $TMP_FILE
+ if [[ "$ECODE" != "0" ]]; then
+ title "ERROR running command locally: exit code $ECODE"
+ if [[ "$DEBUG" == "0" ]]; then
+ title "Command output follows:"
+ cat $TMP_FILE
+ fi
fi
fi
- if [[ "$ECODE" == "255" ]]; then
- title "Connection denied or timed out"
- fi
rm -f $TMP_FILE
eval "$return_var=$ECODE"
}
+if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
+ ACCT=$USER
+fi
+
+if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
+ ACCT=ci
+fi
+
title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
. $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados-cwl-runner-with-checksum.sh ]]; then printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum \"\$@\"' > arvados-cwl-runner-with-checksum.sh; chmod 755 arvados-cwl-runner-with-checksum.sh; fi"
+run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
if [[ "$ECODE" != "0" ]]; then
echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./run_test.sh RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh "
+run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
+
+if [[ "$ECODE" != "0" ]]; then
+ echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
+ exit $ECODE
+fi
-run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN git clone --depth 1 https://git.$IDENTIFIER.arvadosapi.com/arvados.git; fi"
+run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi"
if [[ "$ECODE" != "0" ]]; then
- echo "Failed to git clone --depth 1 git@git.$IDENTIFIER.arvadosapi.com:arvados.git"
+ echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git"
exit $ECODE
fi
-run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./arvados-tests.sh"
+run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
exit $ECODE