From e4376aca8fd1e81a03b8534cab6cbd07220c45b9 Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Tue, 12 Oct 2021 14:03:09 -0400 Subject: [PATCH] 18264: clean up the way we run the CWL tests. Only run the upstream suite from a new run-cwl-test-suite.sh script (the arvados CWL tests are run separately now, directly from CI). Remove old run-cwl-test.sh script. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- jenkins/run-cwl-test-suite.sh | 116 ++++++++++++++++ jenkins/run-cwl-test.sh | 244 ---------------------------------- 2 files changed, 116 insertions(+), 244 deletions(-) create mode 100755 jenkins/run-cwl-test-suite.sh delete mode 100755 jenkins/run-cwl-test.sh diff --git a/jenkins/run-cwl-test-suite.sh b/jenkins/run-cwl-test-suite.sh new file mode 100755 index 0000000..cad0c34 --- /dev/null +++ b/jenkins/run-cwl-test-suite.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +set -o pipefail +set -e + +DEBUG=0 +JOBS=1 + +function usage { + echo >&2 + echo >&2 "usage: $0 [options] " + echo >&2 + echo >&2 " Arvados cluster name" + echo >&2 + echo >&2 "$0 options:" + echo >&2 " -d, --debug Enable debug output" + echo >&2 " -h, --help Display this help and exit" + echo >&2 " -s, --scopes Print required scopes to run tests" + echo >&2 " -j, --jobs Allow N jobs at once; 1 job with no arg." + echo >&2 +} + +function print_scopes { + echo >&2 " Required scope for the token used to run the tests:" + echo >&2 + echo >&2 " arv api_client_authorization create_system_auth --scopes " + echo >&2 "[\"GET /arvados/v1/virtual_machines\"," + echo >&2 "\"GET /arvados/v1/keep_services\"," + echo >&2 "\"GET /arvados/v1/keep_services/\"," + echo >&2 "\"GET /arvados/v1/groups\"," + echo >&2 "\"GET /arvados/v1/groups/\"," + echo >&2 "\"GET /arvados/v1/links\"," + echo >&2 "\"GET /arvados/v1/collections\"," + echo >&2 "\"POST /arvados/v1/collections\"," + echo >&2 "\"POST /arvados/v1/links\"," + echo >&2 "\"GET /arvados/v1/users/current\"," + echo >&2 "\"POST /arvados/v1/users/current\"," + echo >&2 "\"GET /arvados/v1/jobs\"," + echo >&2 "\"POST /arvados/v1/jobs\"," + echo >&2 "\"GET /arvados/v1/pipeline_instances\"," + echo >&2 "\"GET /arvados/v1/pipeline_instances/\"," + echo >&2 "\"POST /arvados/v1/pipeline_instances\"," + echo >&2 "\"GET /arvados/v1/collections/\"," + echo >&2 "\"POST /arvados/v1/collections/\"," + echo >&2 "\"GET /arvados/v1/container_requests\"," + echo >&2 "\"GET /arvados/v1/container_requests/\"," + echo >&2 "\"POST /arvados/v1/container_requests\"," + echo >&2 "\"POST /arvados/v1/container_requests/\"," + echo >&2 "\"GET /arvados/v1/containers\"," + echo >&2 "\"GET /arvados/v1/containers/\"," + echo >&2 "\"GET /arvados/v1/repositories\"," + echo >&2 "\"GET /arvados/v1/repositories/\"," + echo >&2 "\"GET /arvados/v1/logs\" ]" + echo >&2 +} + +# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). +TEMP=`getopt -o hdlp:sj: \ + --long help,scopes,debug,jobs: \ + -n "$0" -- "$@"` + +if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi +# Note the quotes around `$TEMP': they are essential! +eval set -- "$TEMP" + +while [ $# -ge 1 ] +do + case $1 in + -d | --debug) + DEBUG=1 + shift + ;; + -s | --scopes) + print_scopes + exit 0 + ;; + -j | --jobs) + JOBS="$2"; shift 2 + ;; + --) + shift + break + ;; + *) + usage + exit 1 + ;; + esac +done + + +title () { + date=`date +'%Y-%m-%d %H:%M:%S'` + printf "%s\n" "$date $1" +} + +title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN" +if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then + title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set." + exit 1 +fi + +if [[ ! -e common-workflow-language ]]; then + git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git +fi + +printf "%s\n%s\n" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 "$@"' > arvados-cwl-runner-with-checksum.sh +chmod 755 arvados-cwl-runner-with-checksum.sh + +cd common-workflow-language +git pull +./run_test.sh -j$JOBS --timeout=900 RUNNER=../arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint diff --git a/jenkins/run-cwl-test.sh b/jenkins/run-cwl-test.sh deleted file mode 100755 index 742ddcb..0000000 --- a/jenkins/run-cwl-test.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/bin/bash - -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -set -o pipefail -# extglob needs to be enabled for the suppression of tokens in the output -shopt -s extglob - -DEBUG=0 -SSH_PORT=22 -JOBS=1 -ACCT= -LOCAL=0 - -function usage { - echo >&2 - echo >&2 "usage: $0 [options] " - echo >&2 - echo >&2 " Arvados cluster name" - echo >&2 - echo >&2 "$0 options:" - echo >&2 " -p, --port SSH port to use (default 22)" - echo >&2 " --acct Account to log in with" - echo >&2 " -d, --debug Enable debug output" - echo >&2 " -h, --help Display this help and exit" - echo >&2 " -s, --scopes Print required scopes to run tests" - echo >&2 " -j, --jobs Allow N jobs at once; 1 job with no arg." - echo >&2 " -l, --local Run arvados-cwl-runner locally, not on shell." - echo >&2 -} - -function print_scopes { - echo >&2 " Required scope for the token used to run the tests:" - echo >&2 - echo >&2 " arv api_client_authorization create_system_auth --scopes " - echo >&2 "[\"GET /arvados/v1/virtual_machines\"," - echo >&2 "\"GET /arvados/v1/keep_services\"," - echo >&2 "\"GET /arvados/v1/keep_services/\"," - echo >&2 "\"GET /arvados/v1/groups\"," - echo >&2 "\"GET /arvados/v1/groups/\"," - echo >&2 "\"GET /arvados/v1/links\"," - echo >&2 "\"GET /arvados/v1/collections\"," - echo >&2 "\"POST /arvados/v1/collections\"," - echo >&2 "\"POST /arvados/v1/links\"," - echo >&2 "\"GET /arvados/v1/users/current\"," - echo >&2 "\"POST /arvados/v1/users/current\"," - echo >&2 "\"GET /arvados/v1/jobs\"," - echo >&2 "\"POST /arvados/v1/jobs\"," - echo >&2 "\"GET /arvados/v1/pipeline_instances\"," - echo >&2 "\"GET /arvados/v1/pipeline_instances/\"," - echo >&2 "\"POST /arvados/v1/pipeline_instances\"," - echo >&2 "\"GET /arvados/v1/collections/\"," - echo >&2 "\"POST /arvados/v1/collections/\"," - echo >&2 "\"GET /arvados/v1/container_requests\"," - echo >&2 "\"GET /arvados/v1/container_requests/\"," - echo >&2 "\"POST /arvados/v1/container_requests\"," - echo >&2 "\"POST /arvados/v1/container_requests/\"," - echo >&2 "\"GET /arvados/v1/containers\"," - echo >&2 "\"GET /arvados/v1/containers/\"," - echo >&2 "\"GET /arvados/v1/repositories\"," - echo >&2 "\"GET /arvados/v1/repositories/\"," - echo >&2 "\"GET /arvados/v1/logs\" ]" - echo >&2 -} - -# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). -TEMP=`getopt -o hdlp:sj: \ - --long help,scopes,debug,local,port:,acct:,jobs: \ - -n "$0" -- "$@"` - -if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi -# Note the quotes around `$TEMP': they are essential! -eval set -- "$TEMP" - -while [ $# -ge 1 ] -do - case $1 in - -p | --port) - SSH_PORT="$2"; shift 2 - ;; - --acct) - ACCT="$2"; shift 2 - ;; - -d | --debug) - DEBUG=1 - shift - ;; - -s | --scopes) - print_scopes - exit 0 - ;; - -j | --jobs) - JOBS="$2"; shift 2 - ;; - -l | --local) - LOCAL=1 - shift - ;; - --) - shift - break - ;; - *) - usage - exit 1 - ;; - esac -done - -IDENTIFIER=$1 - -if [[ "$IDENTIFIER" == '' ]]; then - usage - exit 1 -fi - -EXITCODE=0 - -COLUMNS=80 - -PUPPET_AGENT=' -now() { date +%s; } -let endtime="$(now) + 600" -while [ "$endtime" -gt "$(now)" ]; do - puppet agent --test --detailed-exitcodes - agent_exitcode=$? - if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then - break - else - sleep 10s - fi -done -exit ${agent_exitcode:-99} -' - -title () { - date=`date +'%Y-%m-%d %H:%M:%S'` - printf "%s\n" "$date $1" -} - -function run_command() { - node=$1 - return_var=$2 - command=$3 - - if [[ "$LOCAL" == "0" ]]; then - title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node" - TMP_FILE=`mktemp` - if [[ "$DEBUG" != "0" ]]; then - echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE - ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE - ECODE=$? - else - ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1 - ECODE=$? - fi - - if [[ "$ECODE" != "255" && "$ECODE" != "0" ]]; then - # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is - # a shell node that is down. - title "ERROR running command on $node: exit code $ECODE" - if [[ "$DEBUG" == "0" ]]; then - title "Command output follows:" - cat $TMP_FILE - fi - fi - if [[ "$ECODE" == "255" ]]; then - title "Connection denied or timed out" - fi - else - title "Running '${command/ARVADOS_API_TOKEN=*([-a-z0-9\/])/ARVADOS_API_TOKEN=suppressed}' locally" - TMP_FILE=`mktemp` - if [[ "$DEBUG" != "0" ]]; then - bash -c "$command" | tee $TMP_FILE - ECODE=$? - else - bash -c "$command" > $TMP_FILE 2>&1 - ECODE=$? - fi - - if [[ "$ECODE" != "0" ]]; then - title "ERROR running command locally: exit code $ECODE" - if [[ "$DEBUG" == "0" ]]; then - title "Command output follows:" - cat $TMP_FILE - fi - fi - fi - rm -f $TMP_FILE - eval "$return_var=$ECODE" -} - -if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then - ACCT=$USER -fi - -if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then - ACCT=ci -fi - -title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN" -if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then - . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf -else - title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found." -fi -if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then - title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set." - exit 1 -fi - -run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi" - -if [[ "$ECODE" != "0" ]]; then - echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git" - exit $ECODE -fi - -run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh" - -if [[ "$ECODE" != "0" ]]; then - echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh" - exit $ECODE -fi - -run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint" - -if [[ "$ECODE" != "0" ]]; then - echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh" - exit $ECODE -fi - -run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi" - -if [[ "$ECODE" != "0" ]]; then - echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git" - exit $ECODE -fi - -run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS" - -exit $ECODE -- 2.30.2