build/run-build-packages-python-and-ruby.sh

   1 #!/bin/bash
   2
   3 COLUMNS=80
   4
   5 . `dirname "$(readlink -f "$0")"`/run-library.sh
   6 #. `dirname "$(readlink -f "$0")"`/libcloud-pin
   7
   8 read -rd "\000" helpmessage <<EOF
   9 $(basename $0): Build Arvados Python packages and Ruby gems
  10
  11 Syntax:
  12         WORKSPACE=/path/to/arvados $(basename $0) [options]
  13
  14 Options:
  15
  16 --debug
  17     Output debug information (default: false)
  18 --upload
  19     If the build and test steps are successful, upload the python
  20     packages to pypi and the gems to rubygems (default: false)
  21
  22 WORKSPACE=path         Path to the Arvados source tree to build packages from
  23
  24 EOF
  25
  26 EXITCODE=0
  27
  28 exit_cleanly() {
  29     trap - INT
  30     report_outcomes
  31     exit $EXITCODE
  32 }
  33
  34 gem_wrapper() {
  35   local gem_name="$1"; shift
  36   local gem_directory="$1"; shift
  37
  38   title "Start $gem_name gem build"
  39   timer_reset
  40
  41   cd "$gem_directory"
  42   handle_ruby_gem $gem_name
  43
  44   checkexit $? "$gem_name gem build"
  45   title "End of $gem_name gem build (`timer`)"
  46 }
  47
  48 python_wrapper() {
  49   local package_name="$1"; shift
  50   local package_directory="$1"; shift
  51
  52   title "Start $package_name python package build"
  53   timer_reset
  54
  55   cd "$package_directory"
  56   handle_python_package
  57
  58   checkexit $? "$package_name python package build"
  59   title "End of $package_name python package build (`timer`)"
  60 }
  61
  62 TARGET=
  63 UPLOAD=0
  64 DEBUG=${ARVADOS_DEBUG:-0}
  65
  66 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  67     help,debug,upload,target: \
  68     -- "" "$@")
  69 if [ $? -ne 0 ]; then
  70     exit 1
  71 fi
  72
  73 eval set -- "$PARSEDOPTS"
  74 while [ $# -gt 0 ]; do
  75     case "$1" in
  76         --help)
  77             echo >&2 "$helpmessage"
  78             echo >&2
  79             exit 1
  80             ;;
  81         --target)
  82             TARGET="$2"; shift
  83             ;;
  84         --upload)
  85             UPLOAD=1
  86             ;;
  87         --debug)
  88             DEBUG=1
  89             ;;
  90         --)
  91             if [ $# -gt 1 ]; then
  92                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  93                 exit 1
  94             fi
  95             ;;
  96     esac
  97     shift
  98 done
  99
 100 if ! [[ -n "$WORKSPACE" ]]; then
 101   echo >&2 "$helpmessage"
 102   echo >&2
 103   echo >&2 "Error: WORKSPACE environment variable not set"
 104   echo >&2
 105   exit 1
 106 fi
 107
 108 STDOUT_IF_DEBUG=/dev/null
 109 STDERR_IF_DEBUG=/dev/null
 110 DASHQ_UNLESS_DEBUG=-q
 111 if [[ "$DEBUG" != 0 ]]; then
 112     STDOUT_IF_DEBUG=/dev/stdout
 113     STDERR_IF_DEBUG=/dev/stderr
 114     DASHQ_UNLESS_DEBUG=
 115 fi
 116
 117 EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
 118 EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
 119
 120 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 121 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 122 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 123   # error; for some reason, the path is not accessible
 124   # to the script (e.g. permissions re-evaled after suid)
 125   exit 1  # fail
 126 fi
 127
 128 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 129 debug_echo "Workspace is $WORKSPACE"
 130
 131 if [[ -f /etc/profile.d/rvm.sh ]]; then
 132     source /etc/profile.d/rvm.sh
 133     GEM="rvm-exec default gem"
 134 else
 135     GEM=gem
 136 fi
 137
 138 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 139 # out our git tree here
 140 chmod o+r "$WORKSPACE" -R
 141
 142 # More cleanup - make sure all executables that we'll package are 755
 143 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 144
 145 # Now fix our umask to something better suited to building and publishing
 146 # gems and packages
 147 umask 0022
 148
 149 debug_echo "umask is" `umask`
 150
 151 FPM_GEM_PREFIX=$($GEM environment gemdir)
 152
 153 gem_wrapper arvados "$WORKSPACE/sdk/ruby"
 154 gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
 155 gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
 156
 157 GEM_BUILD_FAILURES=0
 158 if [ ${#failures[@]} -ne 0 ]; then
 159   GEM_BUILD_FAILURES=${#failures[@]}
 160 fi
 161
 162 python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 163 python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
 164 python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
 165 python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
 166 python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
 167
 168 PYTHON_BUILD_FAILURES=0
 169 if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
 170   PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
 171 fi
 172
 173 if [[ "$UPLOAD" != 0 ]]; then
 174   title "Start upload python packages"
 175   timer_reset
 176
 177   if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
 178     /usr/local/arvados-dev/jenkins/run_upload_packages.py --workspace $WORKSPACE python
 179   else
 180     echo "Skipping python packages upload, there were errors building the packages"
 181   fi
 182   checkexit $? "upload python packages"
 183   title "End of upload python packages (`timer`)"
 184
 185   title "Start upload ruby gems"
 186   timer_reset
 187
 188   if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
 189     /usr/local/arvados-dev/jenkins/run_upload_packages.py --workspace $WORKSPACE gems
 190   else
 191     echo "Skipping ruby gem upload, there were errors building the packages"
 192   fi
 193   checkexit $? "upload ruby gems"
 194   title "End of upload ruby gems (`timer`)"
 195
 196 fi
 197
 198 exit_cleanly