build/run-build-packages-python-and-ruby.sh

   1 #!/bin/bash
   2
   3 COLUMNS=80
   4
   5 . `dirname "$(readlink -f "$0")"`/run-library.sh
   6 #. `dirname "$(readlink -f "$0")"`/libcloud-pin
   7
   8 read -rd "\000" helpmessage <<EOF
   9 $(basename $0): Build Arvados Python packages and Ruby gems
  10
  11 Syntax:
  12         WORKSPACE=/path/to/arvados $(basename $0) [options]
  13
  14 Options:
  15
  16 --debug
  17     Output debug information (default: false)
  18 --upload
  19     If the build and test steps are successful, upload the python
  20     packages to pypi and the gems to rubygems (default: false)
  21
  22 WORKSPACE=path         Path to the Arvados source tree to build packages from
  23
  24 EOF
  25
  26 exit_cleanly() {
  27     trap - INT
  28     report_outcomes
  29     exit ${#failures[@]}
  30 }
  31
  32 gem_wrapper() {
  33   local gem_name="$1"; shift
  34   local gem_directory="$1"; shift
  35
  36   title "Start $gem_name gem build"
  37   timer_reset
  38
  39   cd "$gem_directory"
  40   handle_ruby_gem $gem_name
  41
  42   checkexit $? "$gem_name gem build"
  43   title "End of $gem_name gem build (`timer`)"
  44 }
  45
  46 python_wrapper() {
  47   local package_name="$1"; shift
  48   local package_directory="$1"; shift
  49
  50   title "Start $package_name python package build"
  51   timer_reset
  52
  53   cd "$package_directory"
  54   if [[ $DEBUG > 0 ]]; then
  55     echo `pwd`
  56   fi
  57   handle_python_package
  58
  59   checkexit $? "$package_name python package build"
  60   title "End of $package_name python package build (`timer`)"
  61 }
  62
  63 TARGET=
  64 UPLOAD=0
  65 DEBUG=${ARVADOS_DEBUG:-0}
  66
  67 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  68     help,debug,upload,target: \
  69     -- "" "$@")
  70 if [ $? -ne 0 ]; then
  71     exit 1
  72 fi
  73
  74 eval set -- "$PARSEDOPTS"
  75 while [ $# -gt 0 ]; do
  76     case "$1" in
  77         --help)
  78             echo >&2 "$helpmessage"
  79             echo >&2
  80             exit 1
  81             ;;
  82         --target)
  83             TARGET="$2"; shift
  84             ;;
  85         --upload)
  86             UPLOAD=1
  87             ;;
  88         --debug)
  89             DEBUG=1
  90             ;;
  91         --)
  92             if [ $# -gt 1 ]; then
  93                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  94                 exit 1
  95             fi
  96             ;;
  97     esac
  98     shift
  99 done
 100
 101 if ! [[ -n "$WORKSPACE" ]]; then
 102   echo >&2 "$helpmessage"
 103   echo >&2
 104   echo >&2 "Error: WORKSPACE environment variable not set"
 105   echo >&2
 106   exit 1
 107 fi
 108
 109 STDOUT_IF_DEBUG=/dev/null
 110 STDERR_IF_DEBUG=/dev/null
 111 DASHQ_UNLESS_DEBUG=-q
 112 if [[ "$DEBUG" != 0 ]]; then
 113     STDOUT_IF_DEBUG=/dev/stdout
 114     STDERR_IF_DEBUG=/dev/stderr
 115     DASHQ_UNLESS_DEBUG=
 116 fi
 117
 118 EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
 119 EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
 120
 121 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 122 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 123 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 124   # error; for some reason, the path is not accessible
 125   # to the script (e.g. permissions re-evaled after suid)
 126   exit 1  # fail
 127 fi
 128
 129 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 130 debug_echo "Workspace is $WORKSPACE"
 131
 132 if [[ -f /etc/profile.d/rvm.sh ]]; then
 133     source /etc/profile.d/rvm.sh
 134     GEM="rvm-exec default gem"
 135 else
 136     GEM=gem
 137 fi
 138
 139 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 140 # out our git tree here
 141 chmod o+r "$WORKSPACE" -R
 142
 143 # More cleanup - make sure all executables that we'll package are 755
 144 cd "$WORKSPACE"
 145 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 146
 147 # Now fix our umask to something better suited to building and publishing
 148 # gems and packages
 149 umask 0022
 150
 151 debug_echo "umask is" `umask`
 152
 153 gem_wrapper arvados "$WORKSPACE/sdk/ruby"
 154 gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
 155 gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
 156
 157 GEM_BUILD_FAILURES=0
 158 if [ ${#failures[@]} -ne 0 ]; then
 159   GEM_BUILD_FAILURES=${#failures[@]}
 160 fi
 161
 162 python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 163 python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
 164 python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
 165 python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
 166 python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
 167
 168 PYTHON_BUILD_FAILURES=0
 169 if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
 170   PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
 171 fi
 172
 173 if [[ "$UPLOAD" != 0 ]]; then
 174
 175   if [[ $DEBUG > 0 ]]; then
 176     EXTRA_UPLOAD_FLAGS=" --verbose"
 177   else
 178     EXTRA_UPLOAD_FLAGS=""
 179   fi
 180
 181   if [[ ! -e "$WORKSPACE/packages" ]]; then
 182     mkdir -p "$WORKSPACE/packages"
 183   fi
 184
 185   title "Start upload python packages"
 186   timer_reset
 187
 188   if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
 189     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
 190   else
 191     echo "Skipping python packages upload, there were errors building the packages"
 192   fi
 193   checkexit $? "upload python packages"
 194   title "End of upload python packages (`timer`)"
 195
 196   title "Start upload ruby gems"
 197   timer_reset
 198
 199   if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
 200     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
 201   else
 202     echo "Skipping ruby gem upload, there were errors building the packages"
 203   fi
 204   checkexit $? "upload ruby gems"
 205   title "End of upload ruby gems (`timer`)"
 206
 207 fi
 208
 209 exit_cleanly