jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 set -o pipefail
   8
   9 DEBUG=0
  10 SSH_PORT=22
  11 JOBS=1
  12 ACCT=
  13 LOCAL=0
  14
  15 function usage {
  16     echo >&2
  17     echo >&2 "usage: $0 [options] <identifier>"
  18     echo >&2
  19     echo >&2 "   <identifier>                 Arvados cluster name"
  20     echo >&2
  21     echo >&2 "$0 options:"
  22     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  23     echo >&2 "      --acct <username>         Account to log in with"
  24     echo >&2 "  -d, --debug                   Enable debug output"
  25     echo >&2 "  -h, --help                    Display this help and exit"
  26     echo >&2 "  -s, --scopes                  Print required scopes to run tests"
  27     echo >&2 "  -j, --jobs <jobs>             Allow N jobs at once; 1 job with no arg."
  28     echo >&2 "  -l, --local                   Run arvados-cwl-runner locally, not on shell.<identifier>"
  29     echo >&2
  30 }
  31
  32 function print_scopes {
  33     echo >&2 " Required scope for the token used to run the tests:"
  34     echo >&2
  35     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  36     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  37     echo >&2 "\"GET /arvados/v1/keep_services\","
  38     echo >&2 "\"GET /arvados/v1/keep_services/\","
  39     echo >&2 "\"GET /arvados/v1/groups\","
  40     echo >&2 "\"GET /arvados/v1/groups/\","
  41     echo >&2 "\"GET /arvados/v1/links\","
  42     echo >&2 "\"GET /arvados/v1/collections\","
  43     echo >&2 "\"POST /arvados/v1/collections\","
  44     echo >&2 "\"POST /arvados/v1/links\","
  45     echo >&2 "\"GET /arvados/v1/users/current\","
  46     echo >&2 "\"POST /arvados/v1/users/current\","
  47     echo >&2 "\"GET /arvados/v1/jobs\","
  48     echo >&2 "\"POST /arvados/v1/jobs\","
  49     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  50     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  51     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  52     echo >&2 "\"GET /arvados/v1/collections/\","
  53     echo >&2 "\"POST /arvados/v1/collections/\","
  54     echo >&2 "\"GET /arvados/v1/container_requests\","
  55     echo >&2 "\"GET /arvados/v1/container_requests/\","
  56     echo >&2 "\"POST /arvados/v1/container_requests\","
  57     echo >&2 "\"POST /arvados/v1/container_requests/\","
  58     echo >&2 "\"GET /arvados/v1/containers\","
  59     echo >&2 "\"GET /arvados/v1/containers/\","
  60     echo >&2 "\"GET /arvados/v1/repositories\","
  61     echo >&2 "\"GET /arvados/v1/repositories/\","
  62     echo >&2 "\"GET /arvados/v1/logs\" ]"
  63     echo >&2
  64 }
  65
  66 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  67 TEMP=`getopt -o hdlp:sj: \
  68     --long help,scopes,debug,local,port:,acct:,jobs: \
  69     -n "$0" -- "$@"`
  70
  71 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  72 # Note the quotes around `$TEMP': they are essential!
  73 eval set -- "$TEMP"
  74
  75 while [ $# -ge 1 ]
  76 do
  77     case $1 in
  78         -p | --port)
  79             SSH_PORT="$2"; shift 2
  80             ;;
  81         --acct)
  82             ACCT="$2"; shift 2
  83             ;;
  84         -d | --debug)
  85             DEBUG=1
  86             shift
  87             ;;
  88         -s | --scopes)
  89             print_scopes
  90             exit 0
  91             ;;
  92         -j | --jobs)
  93             JOBS="$2"; shift 2
  94             ;;
  95         -l | --local)
  96             LOCAL=1
  97             shift
  98             ;;
  99         --)
 100             shift
 101             break
 102             ;;
 103         *)
 104             usage
 105             exit 1
 106             ;;
 107     esac
 108 done
 109
 110 IDENTIFIER=$1
 111
 112 if [[ "$IDENTIFIER" == '' ]]; then
 113   usage
 114   exit 1
 115 fi
 116
 117 EXITCODE=0
 118
 119 COLUMNS=80
 120
 121 PUPPET_AGENT='
 122 now() { date +%s; }
 123 let endtime="$(now) + 600"
 124 while [ "$endtime" -gt "$(now)" ]; do
 125     puppet agent --test --detailed-exitcodes
 126     agent_exitcode=$?
 127     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
 128         break
 129     else
 130         sleep 10s
 131     fi
 132 done
 133 exit ${agent_exitcode:-99}
 134 '
 135
 136 title () {
 137   date=`date +'%Y-%m-%d %H:%M:%S'`
 138   printf "%s\n" "$date $1"
 139 }
 140
 141 function run_command() {
 142   node=$1
 143   return_var=$2
 144   command=$3
 145
 146   if [[ "$LOCAL" == "0" ]]; then
 147     title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
 148     TMP_FILE=`mktemp`
 149     if [[ "$DEBUG" != "0" ]]; then
 150       echo ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 151       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 152       ECODE=$?
 153     else
 154       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
 155       ECODE=$?
 156     fi
 157
 158     if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 159       # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
 160       #   a shell node that is down.
 161       title "ERROR running command on $node: exit code $ECODE"
 162       if [[ "$DEBUG" == "0" ]]; then
 163         title "Command output follows:"
 164         cat $TMP_FILE
 165       fi
 166     fi
 167     if [[ "$ECODE" == "255" ]]; then
 168       title "Connection denied or timed out"
 169     fi
 170   else
 171     title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' locally"
 172     TMP_FILE=`mktemp`
 173     if [[ "$DEBUG" != "0" ]]; then
 174       bash -c "$command" | tee $TMP_FILE
 175       ECODE=$?
 176     else
 177       bash -c "$command" > $TMP_FILE 2>&1
 178       ECODE=$?
 179     fi
 180
 181     if [[ "$ECODE" != "0"  ]]; then
 182       title "ERROR running command locally: exit code $ECODE"
 183       if [[ "$DEBUG" == "0" ]]; then
 184         title "Command output follows:"
 185         cat $TMP_FILE
 186       fi
 187     fi
 188   fi
 189   rm -f $TMP_FILE
 190   eval "$return_var=$ECODE"
 191 }
 192
 193 if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
 194   ACCT=$USER
 195 fi
 196
 197 if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
 198   ACCT=ci
 199 fi
 200
 201 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 202 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 203   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 204 else
 205   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 206 fi
 207 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 208   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 209   exit 1
 210 fi
 211
 212 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
 213
 214 if [[ "$ECODE" != "0" ]]; then
 215   echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
 216   exit $ECODE
 217 fi
 218
 219 run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse --eval-timeout 60 \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 220
 221 if [[ "$ECODE" != "0" ]]; then
 222   echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 223   exit $ECODE
 224 fi
 225
 226 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 227
 228 if [[ "$ECODE" != "0" ]]; then
 229   echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 230   exit $ECODE
 231 fi
 232
 233 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/arvados/arvados.git; fi"
 234
 235 if [[ "$ECODE" != "0" ]]; then
 236   echo "Failed to git clone --depth 1 https://git.arvados.org/arvados.git"
 237   exit $ECODE
 238 fi
 239
 240 run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
 241
 242 exit $ECODE