jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: AGPL-3.0
   6
   7 set -o pipefail
   8
   9 DEBUG=0
  10 SSH_PORT=22
  11 ACCT=ci
  12
  13 function usage {
  14     echo >&2
  15     echo >&2 "usage: $0 [options] <identifier>"
  16     echo >&2
  17     echo >&2 "   <identifier>                 Arvados cluster name"
  18     echo >&2
  19     echo >&2 "$0 options:"
  20     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  21     echo >&2 "      --acct <username>         Account to log in with"
  22     echo >&2 "  -d, --debug                   Enable debug output"
  23     echo >&2 "  -h, --help                    Display this help and exit"
  24     echo >&2 "  -s, --scopes                  Print required scopes to run tests"
  25     echo >&2
  26 }
  27
  28 function print_scopes {
  29     echo >&2 " Required scope for the token used to run the tests:"
  30     echo >&2
  31     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  32     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  33     echo >&2 "\"GET /arvados/v1/keep_services\","
  34     echo >&2 "\"GET /arvados/v1/keep_services/\","
  35     echo >&2 "\"GET /arvados/v1/groups\","
  36     echo >&2 "\"GET /arvados/v1/groups/\","
  37     echo >&2 "\"GET /arvados/v1/links\","
  38     echo >&2 "\"GET /arvados/v1/collections\","
  39     echo >&2 "\"POST /arvados/v1/collections\","
  40     echo >&2 "\"POST /arvados/v1/links\","
  41     echo >&2 "\"GET /arvados/v1/users/current\","
  42     echo >&2 "\"POST /arvados/v1/users/current\","
  43     echo >&2 "\"GET /arvados/v1/jobs\","
  44     echo >&2 "\"POST /arvados/v1/jobs\","
  45     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  46     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  47     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  48     echo >&2 "\"GET /arvados/v1/collections/\","
  49     echo >&2 "\"POST /arvados/v1/collections/\","
  50     echo >&2 "\"GET /arvados/v1/container_requests\","
  51     echo >&2 "\"GET /arvados/v1/container_requests/\","
  52     echo >&2 "\"POST /arvados/v1/container_requests\","
  53     echo >&2 "\"POST /arvados/v1/container_requests/\","
  54     echo >&2 "\"GET /arvados/v1/containers\","
  55     echo >&2 "\"GET /arvados/v1/containers/\","
  56     echo >&2 "\"GET /arvados/v1/repositories\","
  57     echo >&2 "\"GET /arvados/v1/repositories/\","
  58     echo >&2 "\"GET /arvados/v1/logs\" ]"
  59     echo >&2
  60 }
  61
  62 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  63 TEMP=`getopt -o hdp:s \
  64     --long help,scopes,debug,port:,acct: \
  65     -n "$0" -- "$@"`
  66
  67 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  68 # Note the quotes around `$TEMP': they are essential!
  69 eval set -- "$TEMP"
  70
  71 while [ $# -ge 1 ]
  72 do
  73     case $1 in
  74         -p | --port)
  75             SSH_PORT="$2"; shift 2
  76             ;;
  77         --acct)
  78             ACCT="$2"; shift 2
  79             ;;
  80         -d | --debug)
  81             DEBUG=1
  82             shift
  83             ;;
  84         -s | --scopes)
  85             print_scopes
  86             exit 0
  87             ;;
  88         --)
  89             shift
  90             break
  91             ;;
  92         *)
  93             usage
  94             exit 1
  95             ;;
  96     esac
  97 done
  98
  99 IDENTIFIER=$1
 100
 101 if [[ "$IDENTIFIER" == '' ]]; then
 102   usage
 103   exit 1
 104 fi
 105
 106 EXITCODE=0
 107
 108 COLUMNS=80
 109
 110 PUPPET_AGENT='
 111 now() { date +%s; }
 112 let endtime="$(now) + 600"
 113 while [ "$endtime" -gt "$(now)" ]; do
 114     puppet agent --test --detailed-exitcodes
 115     agent_exitcode=$?
 116     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
 117         break
 118     else
 119         sleep 10s
 120     fi
 121 done
 122 exit ${agent_exitcode:-99}
 123 '
 124
 125 title () {
 126   date=`date +'%Y-%m-%d %H:%M:%S'`
 127   printf "%s\n" "$date $1"
 128 }
 129
 130 function run_command() {
 131   node=$1
 132   return_var=$2
 133   command=$3
 134
 135   title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' on $node"
 136   TMP_FILE=`mktemp`
 137   if [[ "$DEBUG" != "0" ]]; then
 138     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 139     ECODE=$?
 140   else
 141     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
 142     ECODE=$?
 143   fi
 144
 145   if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 146     # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
 147     #   a shell node that is down.
 148     title "ERROR running command on $node: exit code $ECODE"
 149     if [[ "$DEBUG" == "0" ]]; then
 150       title "Command output follows:"
 151       cat $TMP_FILE
 152     fi
 153   fi
 154   if [[ "$ECODE" == "255" ]]; then
 155     title "Connection denied or timed out"
 156   fi
 157   rm -f $TMP_FILE
 158   eval "$return_var=$ECODE"
 159 }
 160
 161 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 162 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 163   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 164 else
 165   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 166 fi
 167 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 168   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 169   exit 1
 170 fi
 171
 172 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git; fi"
 173
 174 if [[ "$ECODE" != "0" ]]; then
 175   echo "Failed to git clone --depth 1 https://github.com/common-workflow-language/common-workflow-language.git"
 176   exit $ECODE
 177 fi
 178
 179 run_command shell.$IDENTIFIER ECODE "printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum --disable-reuse \"\$@\"' > arvados-cwl-runner-with-checksum.sh; chmod 755 arvados-cwl-runner-with-checksum.sh"
 180
 181 if [[ "$ECODE" != "0" ]]; then
 182   echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 183   exit $ECODE
 184 fi
 185
 186 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./run_test.sh RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh "
 187
 188 if [[ "$ECODE" != "0" ]]; then
 189   echo "Failed ./run_test.sh RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh"
 190   exit $ECODE
 191 fi
 192
 193 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados ]]; then ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN git clone --depth 1 https://git.$IDENTIFIER.arvadosapi.com/arvados.git; fi"
 194
 195 if [[ "$ECODE" != "0" ]]; then
 196   echo "Failed to git clone --depth 1 git@git.$IDENTIFIER.arvadosapi.com:arvados.git"
 197   exit $ECODE
 198 fi
 199
 200 run_command shell.$IDENTIFIER ECODE "cd arvados/sdk/cwl/tests; export ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN && git pull && ./arvados-tests.sh"
 201
 202 exit $ECODE