jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 DEBUG=0
   4 SSH_PORT=22
   5
   6 function usage {
   7     echo >&2
   8     echo >&2 "usage: $0 [options] <identifier>"
   9     echo >&2
  10     echo >&2 "   <identifier>                 Arvados cluster name"
  11     echo >&2
  12     echo >&2 "$0 options:"
  13     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  14     echo >&2 "  -d, --debug                   Enable debug output"
  15     echo >&2 "  -h, --help                    Display this help and exit"
  16     echo >&2
  17     echo >&2 " Required scope for the token used to run the tests:"
  18     echo >&2
  19     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  20     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  21     echo >&2 "\"GET /arvados/v1/keep_services\","
  22     echo >&2 "\"GET /arvados/v1/keep_services/\","
  23     echo >&2 "\"GET /arvados/v1/groups\","
  24     echo >&2 "\"GET /arvados/v1/groups/\","
  25     echo >&2 "\"GET /arvados/v1/links\","
  26     echo >&2 "\"GET /arvados/v1/collections\","
  27     echo >&2 "\"POST /arvados/v1/collections\","
  28     echo >&2 "\"POST /arvados/v1/links\","
  29     echo >&2 "\"GET /arvados/v1/users/current\","
  30     echo >&2 "\"POST /arvados/v1/users/current\","
  31     echo >&2 "\"GET /arvados/v1/jobs\","
  32     echo >&2 "\"POST /arvados/v1/jobs\","
  33     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  34     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  35     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  36     echo >&2 "\"GET /arvados/v1/collections/\","
  37     echo >&2 "\"POST /arvados/v1/collections/\","
  38     echo >&2 "\"GET /arvados/v1/logs\" ]"
  39     echo >&2
  40 }
  41
  42 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  43 TEMP=`getopt -o hdp: \
  44     --long help,debug,port: \
  45     -n "$0" -- "$@"`
  46
  47 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  48 # Note the quotes around `$TEMP': they are essential!
  49 eval set -- "$TEMP"
  50
  51 while [ $# -ge 1 ]
  52 do
  53     case $1 in
  54         -p | --port)
  55             SSH_PORT="$2"; shift 2
  56             ;;
  57         -d | --debug)
  58             DEBUG=1
  59             shift
  60             ;;
  61         --)
  62             shift
  63             break
  64             ;;
  65         *)
  66             usage
  67             exit 1
  68             ;;
  69     esac
  70 done
  71
  72 IDENTIFIER=$1
  73
  74 if [[ "$IDENTIFIER" == '' ]]; then
  75   usage
  76   exit 1
  77 fi
  78
  79 EXITCODE=0
  80
  81 COLUMNS=80
  82
  83 PUPPET_AGENT='
  84 now() { date +%s; }
  85 let endtime="$(now) + 600"
  86 while [ "$endtime" -gt "$(now)" ]; do
  87     puppet agent --test --detailed-exitcodes
  88     agent_exitcode=$?
  89     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
  90         break
  91     else
  92         sleep 10s
  93     fi
  94 done
  95 exit ${agent_exitcode:-99}
  96 '
  97
  98 title () {
  99   date=`date +'%Y-%m-%d %H:%M:%S'`
 100   printf "%s\n" "$date $1"
 101 }
 102
 103 function run_command() {
 104   node=$1
 105   return_var=$2
 106   command=$3
 107
 108   title "Running '${command/ARVADOS_API_TOKEN=??????????????????????????????????????????????????/ARVADOS_API_TOKEN=suppressed}' on $node"
 109   TMP_FILE=`mktemp`
 110   if [[ "$DEBUG" != "0" ]]; then
 111     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" ci@$node -C "$command" | tee $TMP_FILE
 112   else
 113     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" ci@$node -C "$command" > $TMP_FILE 2>&1
 114   fi
 115
 116   ECODE=$?
 117
 118   if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 119     # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
 120     #   a shell node that is down.
 121     title "ERROR running command on $node: exit code $ECODE"
 122     if [[ "$DEBUG" == "0" ]]; then
 123       title "Command output follows:"
 124       cat $TMP_FILE
 125     fi
 126   fi
 127   if [[ "$ECODE" == "255" ]]; then
 128     title "Connection timed out"
 129     ECODE=0
 130   fi
 131   rm -f $TMP_FILE
 132   eval "$return_var=$ECODE"
 133 }
 134
 135 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 136 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 137   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 138 else
 139   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 140 fi
 141 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 142   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 143   exit 1
 144 fi
 145
 146 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone https://github.com/common-workflow-language/common-workflow-language.git; fi"
 147
 148 if [[ "$ECODE" != "0" ]]; then
 149   echo "Failed to git clone https://github.com/common-workflow-language/common-workflow-language.git"
 150   exit $ECODE
 151 fi
 152
 153 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados-cwl-runner-with-checksum.sh ]]; then printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum \"\$@\"' > arvados-cwl-runner-with-checksum.sh; chmod 755 arvados-cwl-runner-with-checksum.sh; fi"
 154
 155 if [[ "$ECODE" != "0" ]]; then
 156   echo "Failed to create ~ci/arvados-cwl-runner-with-checksum.sh"
 157   exit $ECODE
 158 fi
 159
 160 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./run_test.sh RUNNER=/home/ci/arvados-cwl-runner-with-checksum.sh "
 161
 162 exit $ECODE
 163