jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 set -o pipefail
   8
   9 DEBUG=0
  10 SSH_PORT=22
  11 JOBS=1
  12 ACCT=
  13 LOCAL=0
  14
  15 function usage {
  16     echo >&2
  17     echo >&2 "usage: $0 [options] <identifier>"
  18     echo >&2
  19     echo >&2 "   <identifier>                 Arvados cluster name"
  20     echo >&2
  21     echo >&2 "$0 options:"
  22     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  23     echo >&2 "      --acct <username>         Account to log in with"
  24     echo >&2 "  -d, --debug                   Enable debug output"
  25     echo >&2 "  -h, --help                    Display this help and exit"
  26     echo >&2 "  -s, --scopes                  Print required scopes to run tests"
  27     echo >&2 "  -j, --jobs <jobs>             Allow N jobs at once; 1 job with no arg."
  28     echo >&2 "  -l, --local                   Run arvados-cwl-runner locally, not on shell.<identifier>"
  29     echo >&2
  30 }
  31
  32 function print_scopes {
  33     echo >&2 " Required scope for the token used to run the tests:"
  34     echo >&2
  35     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  36     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  37     echo >&2 "\"GET /arvados/v1/keep_services\","
  38     echo >&2 "\"GET /arvados/v1/keep_services/\","
  39     echo >&2 "\"GET /arvados/v1/groups\","
  40     echo >&2 "\"GET /arvados/v1/groups/\","
  41     echo >&2 "\"GET /arvados/v1/links\","
  42     echo >&2 "\"GET /arvados/v1/collections\","
  43     echo >&2 "\"POST /arvados/v1/collections\","
  44     echo >&2 "\"POST /arvados/v1/links\","
  45     echo >&2 "\"GET /arvados/v1/users/current\","
  46     echo >&2 "\"POST /arvados/v1/users/current\","
  47     echo >&2 "\"GET /arvados/v1/jobs\","
  48     echo >&2 "\"POST /arvados/v1/jobs\","
  49     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  50     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  51     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  52     echo >&2 "\"GET /arvados/v1/collections/\","
  53     echo >&2 "\"POST /arvados/v1/collections/\","
  54     echo >&2 "\"GET /arvados/v1/container_requests\","
  55     echo >&2 "\"GET /arvados/v1/container_requests/\","
  56     echo >&2 "\"POST /arvados/v1/container_requests\","
  57     echo >&2 "\"POST /arvados/v1/container_requests/\","
  58     echo >&2 "\"GET /arvados/v1/containers\","
  59     echo >&2 "\"GET /arvados/v1/containers/\","
  60     echo >&2 "\"GET /arvados/v1/repositories\","
  61     echo >&2 "\"GET /arvados/v1/repositories/\","
  62     echo >&2 "\"GET /arvados/v1/logs\" ]"
  63     echo >&2
  64 }
  65
  66 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  67 TEMP=`getopt -o hdlp:sj: \
  68     --long help,scopes,debug,local,port:,acct:,jobs: \
  69     -n "$0" -- "$@"`
  70
  71 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  72 # Note the quotes around `$TEMP': they are essential!
  73 eval set -- "$TEMP"
  74
  75 while [ $# -ge 1 ]
  76 do
  77     case $1 in
  78         -p | --port)
  79             SSH_PORT="$2"; shift 2
  80             ;;
  81         --acct)
  82             ACCT="$2"; shift 2
  83             ;;
  84         -d | --debug)
  85             DEBUG=1
  86             shift
  87             ;;
  88         -s | --scopes)
  89             print_scopes
  90             exit 0
  91             ;;
  92         -j | --jobs)
  93             JOBS="$2"; shift 2
  94             ;;
  95         -l | --local)
  96             LOCAL=1
  97             shift
  98             ;;
  99         --)
 100             shift
 101             break
 102             ;;
 103         *)
 104             usage
 105             exit 1
 106             ;;
 107     esac
 108 done
 109
 110 IDENTIFIER=$1
 111
 112 if [[ "$IDENTIFIER" == '' ]]; then
 113   usage
 114   exit 1
 115 fi
 116
 117 EXITCODE=0
 118
 119 COLUMNS=80
 120
 121 PUPPET_AGENT='
 122 now() { date +%s; }
 123 let endtime="$(now) + 600"
 124 while [ "$endtime" -gt "$(now)" ]; do
 125     puppet agent --test --detailed-exitcodes
 126     agent_exitcode=$?
 127     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
 128         break
 129     else
 130         sleep 10s
 131     fi
 132 done
 133 exit ${agent_exitcode:-99}
 134 '
 135
 136 title () {
 137   date=`date +'%Y-%m-%d %H:%M:%S'`
 138   printf "%s\n" "$date $1"
 139 }
 140
 141 function run_command() {
 142   node=$1
 143   return_var=$2
 144   command=$3
 145
 146   if [[ "$LOCAL" == "0" ]]; then
 147     title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' on $node"
 148     TMP_FILE=`mktemp`
 149     if [[ "$DEBUG" != "0" ]]; then
 150       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 151       ECODE=$?
 152     else
 153       ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
 154       ECODE=$?
 155     fi
 156
 157     if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 158       # Ssh exits 255 if the connection timed out. Just ignore that, it's possible that this node is
 159       #   a shell node that is down.
 160       title "ERROR running command on $node: exit code $ECODE"
 161       if [[ "$DEBUG" == "0" ]]; then
 162         title "Command output follows:"
 163         cat $TMP_FILE
 164       fi
 165     fi
 166     if [[ "$ECODE" == "255" ]]; then
 167       title "Connection denied or timed out"
 168     fi
 169   else
 170     title "Running '${command/ARVADOS_API_TOKEN=* /ARVADOS_API_TOKEN=suppressed }' locally"
 171     TMP_FILE=`mktemp`
 172     if [[ "$DEBUG" != "0" ]]; then
 173       bash -c "$command" | tee $TMP_FILE
 174       ECODE=$?
 175     else
 176       bash -c "$command" > $TMP_FILE 2>&1
 177       ECODE=$?
 178     fi
 179
 180     if [[ "$ECODE" != "0"  ]]; then
 181       title "ERROR running command locally: exit code $ECODE"
 182       if [[ "$DEBUG" == "0" ]]; then
 183         title "Command output follows:"
 184         cat $TMP_FILE
 185       fi
 186     fi
 187   fi
 188   rm -f $TMP_FILE
 189   eval "$return_var=$ECODE"
 190 }
 191
 192 if [[ "$LOCAL" == "1" && "$ACCT" == "" ]]; then
 193   ACCT=$USER
 194 fi
 195
 196 if [[ "$LOCAL" == "0" && "$ACCT" == "" ]]; then
 197   ACCT=ci
 198 fi
 199
 200 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 201 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 202   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 203 else
 204   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 205 fi
 206 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 207   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 208   exit 1
 209 fi
 210
 211 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
 212
 213 if [[ "$ECODE" != "0" ]]; then
 214   echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
 215   exit $ECODE
 216 fi
 217
 218 run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --api containers --compute-checksum --disable-reuse \"\$@\"' > ~$ACCT/arvados-cwl-runner-with-checksum.sh; chmod 755 ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 219
 220 if [[ "$ECODE" != "0" ]]; then
 221   echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 222   exit $ECODE
 223 fi
 224
 225 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE ./run_test.sh -j$JOBS --timeout=900 RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 226
 227 if [[ "$ECODE" != "0" ]]; then
 228   echo "Failed ./run_test.sh -j$JOBS RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 229   exit $ECODE
 230 fi
 231
 232 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git clone --depth 1 https://github.com/curoverse/arvados.git; fi"
 233
 234 if [[ "$ECODE" != "0" ]]; then
 235   echo "Failed to git clone --depth 1 https://github.com/curoverse/arvados.git"
 236   exit $ECODE
 237 fi
 238
 239 run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE && git pull && ./arvados-tests.sh -j$JOBS"
 240
 241 exit $ECODE