build/run-build-packages-python-and-ruby.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 COLUMNS=80
   7
   8 . `dirname "$(readlink -f "$0")"`/run-library.sh
   9 #. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
  10
  11 read -rd "\000" helpmessage <<EOF
  12 $(basename $0): Build Arvados Python packages and Ruby gems
  13
  14 Syntax:
  15         WORKSPACE=/path/to/arvados $(basename $0) [options]
  16
  17 Options:
  18
  19 --debug
  20     Output debug information (default: false)
  21 --upload
  22     If the build and test steps are successful, upload the python
  23     packages to pypi and the gems to rubygems (default: false)
  24
  25 WORKSPACE=path         Path to the Arvados source tree to build packages from
  26
  27 EOF
  28
  29 exit_cleanly() {
  30     trap - INT
  31     report_outcomes
  32     exit ${#failures[@]}
  33 }
  34
  35 gem_wrapper() {
  36   local gem_name="$1"; shift
  37   local gem_directory="$1"; shift
  38
  39   title "Start $gem_name gem build"
  40   timer_reset
  41
  42   cd "$gem_directory"
  43   handle_ruby_gem $gem_name
  44
  45   checkexit $? "$gem_name gem build"
  46   title "End of $gem_name gem build (`timer`)"
  47 }
  48
  49 python_wrapper() {
  50   local package_name="$1"; shift
  51   local package_directory="$1"; shift
  52
  53   title "Start $package_name python package build"
  54   timer_reset
  55
  56   cd "$package_directory"
  57   if [[ $DEBUG > 0 ]]; then
  58     echo `pwd`
  59   fi
  60   handle_python_package
  61
  62   checkexit $? "$package_name python package build"
  63   title "End of $package_name python package build (`timer`)"
  64 }
  65
  66 TARGET=
  67 UPLOAD=0
  68 DEBUG=${ARVADOS_DEBUG:-0}
  69
  70 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  71     help,debug,upload,target: \
  72     -- "" "$@")
  73 if [ $? -ne 0 ]; then
  74     exit 1
  75 fi
  76
  77 eval set -- "$PARSEDOPTS"
  78 while [ $# -gt 0 ]; do
  79     case "$1" in
  80         --help)
  81             echo >&2 "$helpmessage"
  82             echo >&2
  83             exit 1
  84             ;;
  85         --target)
  86             TARGET="$2"; shift
  87             ;;
  88         --upload)
  89             UPLOAD=1
  90             ;;
  91         --debug)
  92             DEBUG=1
  93             ;;
  94         --)
  95             if [ $# -gt 1 ]; then
  96                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  97                 exit 1
  98             fi
  99             ;;
 100     esac
 101     shift
 102 done
 103
 104 if ! [[ -n "$WORKSPACE" ]]; then
 105   echo >&2 "$helpmessage"
 106   echo >&2
 107   echo >&2 "Error: WORKSPACE environment variable not set"
 108   echo >&2
 109   exit 1
 110 fi
 111
 112 STDOUT_IF_DEBUG=/dev/null
 113 STDERR_IF_DEBUG=/dev/null
 114 DASHQ_UNLESS_DEBUG=-q
 115 if [[ "$DEBUG" != 0 ]]; then
 116     STDOUT_IF_DEBUG=/dev/stdout
 117     STDERR_IF_DEBUG=/dev/stderr
 118     DASHQ_UNLESS_DEBUG=
 119 fi
 120
 121 EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
 122 EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
 123
 124 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 125 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 126 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 127   # error; for some reason, the path is not accessible
 128   # to the script (e.g. permissions re-evaled after suid)
 129   exit 1  # fail
 130 fi
 131
 132 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 133 debug_echo "Workspace is $WORKSPACE"
 134
 135 if [[ -f /etc/profile.d/rvm.sh ]]; then
 136     source /etc/profile.d/rvm.sh
 137     GEM="rvm-exec default gem"
 138 else
 139     GEM=gem
 140 fi
 141
 142 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 143 # out our git tree here
 144 chmod o+r "$WORKSPACE" -R
 145
 146 # More cleanup - make sure all executables that we'll package are 755
 147 cd "$WORKSPACE"
 148 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 149
 150 # Now fix our umask to something better suited to building and publishing
 151 # gems and packages
 152 umask 0022
 153
 154 debug_echo "umask is" `umask`
 155
 156 gem_wrapper arvados "$WORKSPACE/sdk/ruby"
 157 gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
 158 gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
 159
 160 GEM_BUILD_FAILURES=0
 161 if [ ${#failures[@]} -ne 0 ]; then
 162   GEM_BUILD_FAILURES=${#failures[@]}
 163 fi
 164
 165 python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 166 python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
 167 python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
 168 python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
 169 python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
 170
 171 PYTHON_BUILD_FAILURES=0
 172 if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
 173   PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
 174 fi
 175
 176 if [[ "$UPLOAD" != 0 ]]; then
 177
 178   if [[ $DEBUG > 0 ]]; then
 179     EXTRA_UPLOAD_FLAGS=" --verbose"
 180   else
 181     EXTRA_UPLOAD_FLAGS=""
 182   fi
 183
 184   if [[ ! -e "$WORKSPACE/packages" ]]; then
 185     mkdir -p "$WORKSPACE/packages"
 186   fi
 187
 188   title "Start upload python packages"
 189   timer_reset
 190
 191   if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
 192     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
 193   else
 194     echo "Skipping python packages upload, there were errors building the packages"
 195   fi
 196   checkexit $? "upload python packages"
 197   title "End of upload python packages (`timer`)"
 198
 199   title "Start upload ruby gems"
 200   timer_reset
 201
 202   if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
 203     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
 204   else
 205     echo "Skipping ruby gem upload, there were errors building the packages"
 206   fi
 207   checkexit $? "upload ruby gems"
 208   title "End of upload ruby gems (`timer`)"
 209
 210 fi
 211
 212 exit_cleanly