jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 set -o pipefail
   8 # extglob needs to be enabled for the suppression of tokens in the output
   9 shopt -s extglob
  10
  11 DEBUG=0
  12 SSH_PORT=22
  13 JOBS=1
  14 ACCT=
  15 LOCAL=0
  16
  17 function usage {
  18     echo >&2
  19     echo >&2 "usage: $0 [options] <identifier>"
  20     echo >&2
  21     echo >&2 "   <identifier>                 Arvados cluster name"
  22     echo >&2
  23     echo >&2 "$0 options:"
  24     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  25     echo >&2 "      --acct <username>         Account to log in with"
  26     echo >&2 "  -d, --debug                   Enable debug output"
  27     echo >&2 "  -h, --help                    Display this help and exit"
  28     echo >&2 "  -s, --scopes                  Print required scopes to run tests"
  29     echo >&2 "  -j, --jobs <jobs>             Allow N jobs at once; 1 job with no arg."
  30     echo >&2 "  -l, --local                   Run arvados-cwl-runner locally, not on shell.<identifier>"
  31     echo >&2
  32 }
  33
  34 function print_scopes {
  35     echo >&2 " Required scope for the token used to run the tests:"
  36     echo >&2
  37     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  38     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  39     echo >&2 "\"GET /arvados/v1/keep_services\","
  40     echo >&2 "\"GET /arvados/v1/keep_services/\","
  41     echo >&2 "\"GET /arvados/v1/groups\","
  42     echo >&2 "\"GET /arvados/v1/groups/\","
  43     echo >&2 "\"GET /arvados/v1/links\","
  44     echo >&2 "\"GET /arvados/v1/collections\","
  45     echo >&2 "\"POST /arvados/v1/collections\","
  46     echo >&2 "\"POST /arvados/v1/links\","
  47     echo >&2 "\"GET /arvados/v1/users/current\","
  48     echo >&2 "\"POST /arvados/v1/users/current\","
  49     echo >&2 "\"GET /arvados/v1/jobs\","
  50     echo >&2 "\"POST /arvados/v1/jobs\","
  51     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  52     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  53     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  54     echo >&2 "\"GET /arvados/v1/collections/\","
  55     echo >&2 "\"POST /arvados/v1/collections/\","
  56     echo >&2 "\"GET /arvados/v1/container_requests\","
  57     echo >&2 "\"GET /arvados/v1/container_requests/\","
  58     echo >&2 "\"POST /arvados/v1/container_requests\","
  59     echo >&2 "\"POST /arvados/v1/container_requests/\","
  60     echo >&2 "\"GET /arvados/v1/containers\","
  61     echo >&2 "\"GET /arvados/v1/containers/\","
  62     echo >&2 "\"GET /arvados/v1/repositories\","
  63     echo >&2 "\"GET /arvados/v1/repositories/\","
  64     echo >&2 "\"GET /arvados/v1/logs\" ]"
  65     echo >&2
  66 }
  67
  68 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  69 TEMP=`getopt -o hdlp:sj: \
  70     --long help,scopes,debug,local,port:,acct:,jobs: \
  71     -n "$0" -- "$@"`
  72
  73 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  74 # Note the quotes around `$TEMP': they are essential!
  75 eval set -- "$TEMP"
  76
  77 while [ $# -ge 1 ]
  78 do
  79     case $1 in
  80         -p | --port)
  81             SSH_PORT="$2"; shift 2
  82             ;;
  83         --acct)
  84             ACCT="$2"; shift 2
  85             ;;
  86         -d | --debug)
  87             DEBUG=1
  88             shift
  89             ;;
  90         -s | --scopes)
  91             print_scopes
  92             exit 0
  93             ;;
  94         -j | --jobs)
  95             JOBS="$2"; shift 2
  96             ;;
  97         -l | --local)
  98             LOCAL=1
  99             shift
 100             ;;
 101         --)
 102             shift
 103             break
 104             ;;
 105         *)
 106             usage
 107             exit 1
 108             ;;
 109     esac
 110 done
 111
 112 IDENTIFIER=$1
 113
 114 if [[ "$IDENTIFIER" == '' ]]; then
 115   usage
 116   exit 1
 117 fi
 118
 119 EXITCODE=0
 120
 121 COLUMNS=80
 122
 123 PUPPET_AGENT='
 124 now() { date +%s; }
 125 let endtime="$(now) + 600"
 126 while [ "$endtime" -gt "$(now)" ]; do
 127     puppet agent --test --detailed-exitcodes
 128     agent_exitcode=$?
 129     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
 130         break
 131     else
 132         sleep 10s
 133     fi
 134 done
 135 exit ${agent_exitcode:-99}
 136 '
 137
 138 title () {
 139   date=`date +'%Y-%m-%d %H:%M:%S'`
 140   printf "%s\n" "$date $1"
 141 }
 142
 143 function run_command() {
 144   node=$1
 145   return_var=$2
 146   command=$3
 147
 148   if [[ "$LOCAL" == "0" ]]; then
 149     title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
 150     TMP_FILE=`mktemp`
 151     if [[ "$DEBUG" != "0" ]]; then
 152       echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 153       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 154       ECODE=$?
 155     else
 156       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
 157       ECODE=$?
 158     fi
 159
 160     if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 161       # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
 162       #   a shell node that is down.
 163       title "ERROR running command on $node: exit code $ECODE"
 164       if [[ "$DEBUG" == "0" ]]; then
 165         title "Command output follows:"
 166         cat $TMP_FILE
 167       fi
 168     fi
 169     if [[ "$ECODE" == "255" ]]; then
 170       title "Connection denied or timed out"
 171     fi
 172   else
 173     title "Running '${command/ARVADOS_API_TOKEN=*([-a-z0-9\/])/ARVADOS_API_TOKEN=suppressed}' locally"
 174     TMP_FILE=`mktemp`
 175     if [[ "$DEBUG" != "0" ]]; then
 176       bash -c "$command" | tee $TMP_FILE
 177       ECODE=$?
 178     else
 179       bash -c "$command" > $TMP_FILE 2>&1
 180       ECODE=$?
 181     fi
 182
 183     if [[ "$ECODE" != "0"  ]]; then
 184       title "ERROR running command locally: exit code $ECODE"
 185       if [[ "$DEBUG" == "0" ]]; then
 186         title "Command output follows:"
 187         cat $TMP_FILE
 188       fi
 189     fi
 190   fi
 191   rm -f $TMP_FILE
 192   eval "$return_var=$ECODE"
 193 }
 194
 195 if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
 196   ACCT=$USER
 197 fi
 198
 199 if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
 200   ACCT=ci
 201 fi
 202
 203 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 204 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 205   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 206 else
 207   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 208 fi
 209 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 210   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 211   exit 1
 212 fi
 213
 214 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
 215
 216 if [[ "$ECODE" != "0" ]]; then
 217   echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
 218   exit $ECODE
 219 fi
 220
 221 run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 222
 223 if [[ "$ECODE" != "0" ]]; then
 224   echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 225   exit $ECODE
 226 fi
 227
 228 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh -Sdocker_entrypoint"
 229
 230 if [[ "$ECODE" != "0" ]]; then
 231   echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 232   exit $ECODE
 233 fi
 234
 235 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi"
 236
 237 if [[ "$ECODE" != "0" ]]; then
 238   echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git"
 239   exit $ECODE
 240 fi
 241
 242 run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
 243
 244 exit $ECODE