jenkins/run-cwl-test.sh

   1 #!/bin/bash
   2
   3 set -o pipefail
   4
   5 DEBUG=0
   6 SSH_PORT=22
   7 ACCT=ci
   8
   9 function usage {
  10     echo >&2
  11     echo >&2 "usage: $0 [options] <identifier>"
  12     echo >&2
  13     echo >&2 "   <identifier>                 Arvados cluster name"
  14     echo >&2
  15     echo >&2 "$0 options:"
  16     echo >&2 "  -p, --port <ssh port>         SSH port to use (default 22)"
  17     echo >&2 "      --acct <username>         Account to log in with"
  18     echo >&2 "  -d, --debug                   Enable debug output"
  19     echo >&2 "  -h, --help                    Display this help and exit"
  20     echo >&2
  21     echo >&2 " Required scope for the token used to run the tests:"
  22     echo >&2
  23     echo >&2 " arv api_client_authorization create_system_auth     --scopes "
  24     echo >&2 "[\"GET /arvados/v1/virtual_machines\","
  25     echo >&2 "\"GET /arvados/v1/keep_services\","
  26     echo >&2 "\"GET /arvados/v1/keep_services/\","
  27     echo >&2 "\"GET /arvados/v1/groups\","
  28     echo >&2 "\"GET /arvados/v1/groups/\","
  29     echo >&2 "\"GET /arvados/v1/links\","
  30     echo >&2 "\"GET /arvados/v1/collections\","
  31     echo >&2 "\"POST /arvados/v1/collections\","
  32     echo >&2 "\"POST /arvados/v1/links\","
  33     echo >&2 "\"GET /arvados/v1/users/current\","
  34     echo >&2 "\"POST /arvados/v1/users/current\","
  35     echo >&2 "\"GET /arvados/v1/jobs\","
  36     echo >&2 "\"POST /arvados/v1/jobs\","
  37     echo >&2 "\"GET /arvados/v1/pipeline_instances\","
  38     echo >&2 "\"GET /arvados/v1/pipeline_instances/\","
  39     echo >&2 "\"POST /arvados/v1/pipeline_instances\","
  40     echo >&2 "\"GET /arvados/v1/collections/\","
  41     echo >&2 "\"POST /arvados/v1/collections/\","
  42     echo >&2 "\"GET /arvados/v1/logs\" ]"
  43     echo >&2
  44 }
  45
  46 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  47 TEMP=`getopt -o hdp: \
  48     --long help,debug,port:,acct: \
  49     -n "$0" -- "$@"`
  50
  51 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
  52 # Note the quotes around `$TEMP': they are essential!
  53 eval set -- "$TEMP"
  54
  55 while [ $# -ge 1 ]
  56 do
  57     case $1 in
  58         -p | --port)
  59             SSH_PORT="$2"; shift 2
  60             ;;
  61         --acct)
  62             ACCT="$2"; shift 2
  63             ;;
  64         -d | --debug)
  65             DEBUG=1
  66             shift
  67             ;;
  68         --)
  69             shift
  70             break
  71             ;;
  72         *)
  73             usage
  74             exit 1
  75             ;;
  76     esac
  77 done
  78
  79 IDENTIFIER=$1
  80
  81 if [[ "$IDENTIFIER" == '' ]]; then
  82   usage
  83   exit 1
  84 fi
  85
  86 EXITCODE=0
  87
  88 COLUMNS=80
  89
  90 PUPPET_AGENT='
  91 now() { date +%s; }
  92 let endtime="$(now) + 600"
  93 while [ "$endtime" -gt "$(now)" ]; do
  94     puppet agent --test --detailed-exitcodes
  95     agent_exitcode=$?
  96     if [ 0 = "$agent_exitcode" ] || [ 2 = "$agent_exitcode" ]; then
  97         break
  98     else
  99         sleep 10s
 100     fi
 101 done
 102 exit ${agent_exitcode:-99}
 103 '
 104
 105 title () {
 106   date=`date +'%Y-%m-%d %H:%M:%S'`
 107   printf "%s\n" "$date $1"
 108 }
 109
 110 function run_command() {
 111   node=$1
 112   return_var=$2
 113   command=$3
 114
 115   title "Running '${command/ARVADOS_API_TOKEN=*/ARVADOS_API_TOKEN=suppressed}' on $node"
 116   TMP_FILE=`mktemp`
 117   if [[ "$DEBUG" != "0" ]]; then
 118     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" | tee $TMP_FILE
 119     ECODE=$?
 120   else
 121     ssh -t -p$SSH_PORT -o "StrictHostKeyChecking no" -o "ConnectTimeout 125" $ACCT@$node -C "$command" > $TMP_FILE 2>&1
 122     ECODE=$?
 123   fi
 124
 125   if [[ "$ECODE" != "255" && "$ECODE" != "0"  ]]; then
 126     # Ssh exists 255 if the connection timed out. Just ignore that, it's possible that this node is
 127     #   a shell node that is down.
 128     title "ERROR running command on $node: exit code $ECODE"
 129     if [[ "$DEBUG" == "0" ]]; then
 130       title "Command output follows:"
 131       cat $TMP_FILE
 132     fi
 133   fi
 134   if [[ "$ECODE" == "255" ]]; then
 135     title "Connection denied or timed out"
 136   fi
 137   rm -f $TMP_FILE
 138   eval "$return_var=$ECODE"
 139 }
 140
 141 title "Loading ARVADOS_API_HOST and ARVADOS_API_TOKEN"
 142 if [[ -f "$HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf" ]]; then
 143   . $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf
 144 else
 145   title "WARNING: $HOME/.config/arvados/$IDENTIFIER.arvadosapi.com.conf not found."
 146 fi
 147 if [[ "$ARVADOS_API_HOST" == "" ]] || [[ "$ARVADOS_API_TOKEN" == "" ]]; then
 148   title "ERROR: ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set."
 149   exit 1
 150 fi
 151
 152 run_command shell.$IDENTIFIER ECODE "if [[ ! -e common-workflow-language ]]; then git clone https://github.com/common-workflow-language/common-workflow-language.git; fi"
 153
 154 if [[ "$ECODE" != "0" ]]; then
 155   echo "Failed to git clone https://github.com/common-workflow-language/common-workflow-language.git"
 156   exit $ECODE
 157 fi
 158
 159 run_command shell.$IDENTIFIER ECODE "if [[ ! -e arvados-cwl-runner-with-checksum.sh ]]; then printf \"%s\n%s\n\" '#!/bin/sh' 'exec arvados-cwl-runner --compute-checksum \"\$@\"' > arvados-cwl-runner-with-checksum.sh; chmod 755 arvados-cwl-runner-with-checksum.sh; fi"
 160
 161 if [[ "$ECODE" != "0" ]]; then
 162   echo "Failed to create ~$ACCT/arvados-cwl-runner-with-checksum.sh"
 163   exit $ECODE
 164 fi
 165
 166 run_command shell.$IDENTIFIER ECODE "cd common-workflow-language; git pull; ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ./run_test.sh RUNNER=/home/$ACCT/arvados-cwl-runner-with-checksum.sh "
 167
 168 exit $ECODE