3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: AGPL-3.0
8 # extglob needs to be enabled for the suppression of tokens in the output
19 echo >&2 "usage: $0 [options] <identifier>"
21 echo >&2 " <identifier> Arvados cluster name"
23 echo >&2 "$0 options:"
24 echo >&2 " -p, --port <ssh port> SSH port to use (default 22)"
25 echo >&2 " --acct <username> Account to log in with"
26 echo >&2 " -d, --debug Enable debug output"
27 echo >&2 " -h, --help Display this help and exit"
28 echo >&2 " -s, --scopes Print required scopes to run tests"
29 echo >&2 " -j, --jobs <jobs> Allow N jobs at once; 1 job with no arg."
30 echo >&2 " -l, --local Run arvados-cwl-runner locally, not on shell.<identifier>"
34 function print_scopes {
35 echo >&2 " Required scope for the token used to run the tests:"
37 echo >&2 " arv api_client_authorization create_system_auth --scopes "
38 echo >&2 "[\"GET /arvados/v1/virtual_machines\","
39 echo >&2 "\"GET /arvados/v1/keep_services\","
40 echo >&2 "\"GET /arvados/v1/keep_services/\","
41 echo >&2 "\"GET /arvados/v1/groups\","
42 echo >&2 "\"GET /arvados/v1/groups/\","
43 echo >&2 "\"GET /arvados/v1/links\","
44 echo >&2 "\"GET /arvados/v1/collections\","
45 echo >&2 "\"POST /arvados/v1/collections\","
46 echo >&2 "\"POST /arvados/v1/links\","
47 echo >&2 "\"GET /arvados/v1/users/current\","
48 echo >&2 "\"POST /arvados/v1/users/current\","
49 echo >&2 "\"GET /arvados/v1/jobs\","
50 echo >&2 "\"POST /arvados/v1/jobs\","
51 echo >&2 "\"GET /arvados/v1/pipeline_instances\","
52 echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
53 echo >&2 "\"POST /arvados/v1/pipeline_instances\","
54 echo >&2 "\"GET /arvados/v1/collections/\","
55 echo >&2 "\"POST /arvados/v1/collections/\","
56 echo >&2 "\"GET /arvados/v1/container_requests\","
57 echo >&2 "\"GET /arvados/v1/container_requests/\","
58 echo >&2 "\"POST /arvados/v1/container_requests\","
59 echo >&2 "\"POST /arvados/v1/container_requests/\","
60 echo >&2 "\"GET /arvados/v1/containers\","
61 echo >&2 "\"GET /arvados/v1/containers/\","
62 echo >&2 "\"GET /arvados/v1/repositories\","
63 echo >&2 "\"GET /arvados/v1/repositories/\","
64 echo >&2 "\"GET /arvados/v1/logs\" ]"
68 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
69 TEMP=`getopt -o hdlp:sj: \
70 --long help,scopes,debug,local,port:,acct:,jobs: \
73 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
74 # Note the quotes around `$TEMP': they are essential!
81 SSH_PORT="$2"; shift 2
114 if [[ "$IDENTIFIER" == '' ]]; then
125 let endtime="$(now) + 600"
126 while [ "$endtime" -gt "$(now)" ]; do
127 puppet agent --test --detailed-exitcodes
129 if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
135 exit ${agent_exitcode:-99}
139 date=`date +'%Y-%m-%d %H:%M:%S'`
140 printf "%s\n" "$date $1"
143 function run_command() {
148 if [[ "$LOCAL" == "0" ]]; then
149 title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
151 if [[ "$DEBUG" != "0" ]]; then
152 echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
153 ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
156 ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
160 if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then
161 # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
162 # a shell node that is down.
163 title "ERROR running command on $node: exit code $ECODE"
164 if [[ "$DEBUG" == "0" ]]; then
165 title "Command output follows:"
169 if [[ "$ECODE" == "255" ]]; then
170 title "Connection denied or timed out"
173 title "Running '${command/ARVADOS_API_TOKEN=*([-a-z0-9\/])/ARVADOS_API_TOKEN=suppressed}' locally"
175 if [[ "$DEBUG" != "0" ]]; then
176 bash -c "$command" | tee $TMP_FILE
179 bash -c "$command" > $TMP_FILE 2>&1
183 if [[ "$ECODE" != "0" ]]; then
184 title "ERROR running command locally: exit code $ECODE"
185 if [[ "$DEBUG" == "0" ]]; then
186 title "Command output follows:"
192 eval "$return_var=$ECODE"
195 if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
199 if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
203 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
204 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
205 . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
207 title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
209 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
210 title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
214 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
216 if [[ "$ECODE" != "0" ]]; then
217 echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
221 run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
223 if [[ "$ECODE" != "0" ]]; then
224 echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
228 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint"
230 if [[ "$ECODE" != "0" ]]; then
231 echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
235 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi"
237 if [[ "$ECODE" != "0" ]]; then
238 echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git"
242 run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"