build/run-build-packages-python-and-ruby.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 COLUMNS=80
   7
   8 . `dirname "$(readlink -f "$0")"`/run-library.sh
   9 #. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
  10
  11 read -rd "\000" helpmessage <<EOF
  12 $(basename $0): Build Arvados Python packages and Ruby gems
  13
  14 Syntax:
  15         WORKSPACE=/path/to/arvados $(basename $0) [options]
  16
  17 Options:
  18
  19 --debug
  20     Output debug information (default: false)
  21 --upload
  22     If the build and test steps are successful, upload the python
  23     packages to pypi and the gems to rubygems (default: false)
  24
  25 WORKSPACE=path         Path to the Arvados source tree to build packages from
  26
  27 EOF
  28
  29 exit_cleanly() {
  30     trap - INT
  31     report_outcomes
  32     exit ${#failures[@]}
  33 }
  34
  35 gem_wrapper() {
  36   local gem_name="$1"; shift
  37   local gem_directory="$1"; shift
  38
  39   title "Start $gem_name gem build"
  40   timer_reset
  41
  42   cd "$gem_directory"
  43   handle_ruby_gem $gem_name
  44
  45   checkexit $? "$gem_name gem build"
  46   title "End of $gem_name gem build (`timer`)"
  47 }
  48
  49 python_wrapper() {
  50   local package_name="$1"; shift
  51   local package_directory="$1"; shift
  52
  53   title "Start $package_name python package build"
  54   timer_reset
  55
  56   cd "$package_directory"
  57   if [[ $DEBUG > 0 ]]; then
  58     echo `pwd`
  59   fi
  60   handle_python_package
  61
  62   checkexit $? "$package_name python package build"
  63   title "End of $package_name python package build (`timer`)"
  64 }
  65
  66 TARGET=
  67 UPLOAD=0
  68 DEBUG=${ARVADOS_DEBUG:-0}
  69
  70 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  71     help,debug,upload,target: \
  72     -- "" "$@")
  73 if [ $? -ne 0 ]; then
  74     exit 1
  75 fi
  76
  77 eval set -- "$PARSEDOPTS"
  78 while [ $# -gt 0 ]; do
  79     case "$1" in
  80         --help)
  81             echo >&2 "$helpmessage"
  82             echo >&2
  83             exit 1
  84             ;;
  85         --target)
  86             TARGET="$2"; shift
  87             ;;
  88         --upload)
  89             UPLOAD=1
  90             ;;
  91         --debug)
  92             DEBUG=1
  93             ;;
  94         --)
  95             if [ $# -gt 1 ]; then
  96                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  97                 exit 1
  98             fi
  99             ;;
 100     esac
 101     shift
 102 done
 103
 104 if ! [[ -n "$WORKSPACE" ]]; then
 105   echo >&2 "$helpmessage"
 106   echo >&2
 107   echo >&2 "Error: WORKSPACE environment variable not set"
 108   echo >&2
 109   exit 1
 110 fi
 111
 112 STDOUT_IF_DEBUG=/dev/null
 113 STDERR_IF_DEBUG=/dev/null
 114 DASHQ_UNLESS_DEBUG=-q
 115 if [[ "$DEBUG" != 0 ]]; then
 116     STDOUT_IF_DEBUG=/dev/stdout
 117     STDERR_IF_DEBUG=/dev/stderr
 118     DASHQ_UNLESS_DEBUG=
 119 fi
 120
 121 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 122 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 123 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 124   # error; for some reason, the path is not accessible
 125   # to the script (e.g. permissions re-evaled after suid)
 126   exit 1  # fail
 127 fi
 128
 129 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 130 debug_echo "Workspace is $WORKSPACE"
 131
 132 if [[ -f /etc/profile.d/rvm.sh ]]; then
 133     source /etc/profile.d/rvm.sh
 134     GEM="rvm-exec default gem"
 135 else
 136     GEM=gem
 137 fi
 138
 139 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 140 # out our git tree here
 141 chmod o+r "$WORKSPACE" -R
 142
 143 # More cleanup - make sure all executables that we'll package are 755
 144 cd "$WORKSPACE"
 145 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 146
 147 # Now fix our umask to something better suited to building and publishing
 148 # gems and packages
 149 umask 0022
 150
 151 debug_echo "umask is" `umask`
 152
 153 gem_wrapper arvados "$WORKSPACE/sdk/ruby"
 154 gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
 155 gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
 156
 157 GEM_BUILD_FAILURES=0
 158 if [ ${#failures[@]} -ne 0 ]; then
 159   GEM_BUILD_FAILURES=${#failures[@]}
 160 fi
 161
 162 python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
 163 python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
 164 python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
 165 python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
 166 python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
 167
 168 PYTHON_BUILD_FAILURES=0
 169 if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
 170   PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
 171 fi
 172
 173 if [[ "$UPLOAD" != 0 ]]; then
 174
 175   if [[ $DEBUG > 0 ]]; then
 176     EXTRA_UPLOAD_FLAGS=" --verbose"
 177   else
 178     EXTRA_UPLOAD_FLAGS=""
 179   fi
 180
 181   if [[ ! -e "$WORKSPACE/packages" ]]; then
 182     mkdir -p "$WORKSPACE/packages"
 183   fi
 184
 185   title "Start upload python packages"
 186   timer_reset
 187
 188   if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
 189     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
 190   else
 191     echo "Skipping python packages upload, there were errors building the packages"
 192   fi
 193   checkexit $? "upload python packages"
 194   title "End of upload python packages (`timer`)"
 195
 196   title "Start upload ruby gems"
 197   timer_reset
 198
 199   if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
 200     /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
 201   else
 202     echo "Skipping ruby gem upload, there were errors building the packages"
 203   fi
 204   checkexit $? "upload ruby gems"
 205   title "End of upload ruby gems (`timer`)"
 206
 207 fi
 208
 209 exit_cleanly