build/run-library.sh

   1 #!/bin/bash -xe
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 # A library of functions shared by the various scripts in this directory.
   7
   8 # This is the timestamp about when we merged changed to include licenses
   9 # with Arvados packages.  We use it as a heuristic to add revisions for
  10 # older packages.
  11 LICENSE_PACKAGE_TS=20151208015500
  12
  13 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
  14     RAILS_PACKAGE_ITERATION=1
  15 else
  16     RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
  17 fi
  18
  19 debug_echo () {
  20     echo "$@" >"$STDOUT_IF_DEBUG"
  21 }
  22
  23 find_python_program() {
  24     prog="$1"
  25     shift
  26     for prog in "$@"; do
  27         if "$prog" --version >/dev/null 2>&1; then
  28             echo "$prog"
  29             return 0
  30         fi
  31     done
  32     cat >&2 <<EOF
  33 $helpmessage
  34
  35 Error: $prog (from Python setuptools module) not found
  36
  37 EOF
  38     exit 1
  39 }
  40
  41 format_last_commit_here() {
  42     local format="$1"; shift
  43     local dir="${1:-.}"; shift
  44     TZ=UTC git log -n1 --first-parent "--format=format:$format" "$dir"
  45 }
  46
  47 version_from_git() {
  48     # Output the version being built, or if we're building a
  49     # dev/prerelease, output a version number based on the git log for
  50     # the given $subdir.
  51     local subdir="$1"; shift
  52     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
  53         echo "$ARVADOS_BUILDING_VERSION"
  54         return
  55     fi
  56
  57     local git_ts git_hash
  58     declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
  59     ARVADOS_BUILDING_VERSION="$($WORKSPACE/build/version-at-commit.sh $git_hash)"
  60     echo "$ARVADOS_BUILDING_VERSION"
  61 }
  62
  63 nohash_version_from_git() {
  64     local subdir="$1"; shift
  65     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
  66         echo "$ARVADOS_BUILDING_VERSION"
  67         return
  68     fi
  69     version_from_git $subdir | cut -d. -f1-4
  70 }
  71
  72 timestamp_from_git() {
  73     local subdir="$1"; shift
  74     format_last_commit_here "%ct" "$subdir"
  75 }
  76
  77 calculate_python_sdk_cwl_package_versions() {
  78   python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
  79   cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
  80 }
  81
  82 handle_ruby_gem() {
  83     local gem_name="$1"; shift
  84     local gem_version="$(nohash_version_from_git)"
  85     local gem_src_dir="$(pwd)"
  86
  87     if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
  88         return 0
  89     fi
  90
  91     if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
  92         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
  93
  94         # -q appears to be broken in gem version 2.2.2
  95         $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
  96     fi
  97 }
  98
  99 calculate_go_package_version() {
 100   # $__returnvar has the nameref attribute set, which means it is a reference
 101   # to another variable that is passed in as the first argument to this function.
 102   # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
 103   local -n __returnvar="$1"; shift
 104   local oldpwd="$PWD"
 105
 106   cd "$WORKSPACE"
 107   go mod download
 108
 109   # Update the version number and build a new package if the vendor
 110   # bundle has changed, or the command imports anything from the
 111   # Arvados SDK and the SDK has changed.
 112   declare -a checkdirs=(go.mod go.sum)
 113   while [ -n "$1" ]; do
 114       checkdirs+=("$1")
 115       shift
 116   done
 117   # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
 118   # Everything depends on the build directory.
 119   checkdirs+=(sdk/go lib build)
 120   local timestamp=0
 121   for dir in ${checkdirs[@]}; do
 122       cd "$WORKSPACE"
 123       ts="$(timestamp_from_git "$dir")"
 124       if [[ "$ts" -gt "$timestamp" ]]; then
 125           version=$(version_from_git "$dir")
 126           timestamp="$ts"
 127       fi
 128   done
 129   cd "$oldpwd"
 130   __returnvar="$version"
 131 }
 132
 133 # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
 134 package_go_binary() {
 135     local src_path="$1"; shift
 136     local prog="$1"; shift
 137     local package_format="$1"; shift
 138     local target_arch="$1"; shift
 139     local description="$1"; shift
 140     local license_file="${1:-agpl-3.0.txt}"; shift
 141
 142     if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
 143       # arvados-workbench depends on arvados-server at build time, so even when
 144       # only arvados-workbench is being built, we need to build arvados-server too
 145       if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
 146         return 0
 147       fi
 148     fi
 149
 150     # Only amd64 and aarch64 are supported
 151     native_arch="amd64"
 152     if [[ "$HOSTTYPE" == "aarch64" ]]; then
 153         native_arch="arm64"
 154     fi
 155
 156     if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
 157       echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
 158       return 1
 159     fi
 160
 161     if [[ -n "$target_arch" ]]; then
 162       # A target architecture has been specified
 163       package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
 164       return $?
 165     else
 166       # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
 167       # but only when building deb packages (centos does not have support for crosscompiling userspace).
 168       archs=($native_arch)
 169       if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
 170         archs=('amd64' 'arm64')
 171       fi
 172       for ta in $archs; do
 173         package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
 174         retval=$?
 175         if [[ "$retval" != 0 ]]; then
 176           return $retval
 177         fi
 178       done
 179     fi
 180 }
 181
 182 # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
 183 package_go_binary_worker() {
 184     local src_path="$1"; shift
 185     local prog="$1"; shift
 186     local package_format="$1"; shift
 187     local description="$1"; shift
 188     local native_arch="${1:-amd64}"; shift
 189     local target_arch="${1:-amd64}"; shift
 190     local license_file="${1:-agpl-3.0.txt}"; shift
 191
 192     if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then
 193       echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch"
 194       return 1
 195     fi
 196
 197     debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
 198     local basename="${src_path##*/}"
 199     calculate_go_package_version go_package_version $src_path
 200
 201     cd $WORKSPACE/packages/$TARGET
 202     test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
 203     if [[ "$?" != "0" ]]; then
 204       return 1
 205     fi
 206
 207     echo "Building $pachage_format ($target_arch) package for $prog from $src_path"
 208     if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
 209       CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
 210     else
 211       GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
 212     fi
 213
 214     local -a switches=()
 215
 216     binpath=$GOPATH/bin/${basename}
 217     if [[ "${target_arch}" != "${native_arch}" ]]; then
 218       switches+=("-a${target_arch}")
 219       binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
 220     fi
 221
 222     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
 223     if [[ -e "${systemd_unit}" ]]; then
 224         switches+=(
 225             --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
 226             --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
 227             "${systemd_unit}=/lib/systemd/system/${prog}.service")
 228     fi
 229     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
 230
 231     fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
 232 }
 233
 234 # Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
 235 package_go_so() {
 236     local src_path="$1"; shift
 237     local sofile="$1"; shift
 238     local pkg="$1"; shift
 239     local package_format="$1"; shift
 240     local target_arch="$1"; shift # supported: amd64, arm64
 241     local description="$1"; shift
 242
 243     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
 244       return 0
 245     fi
 246
 247     debug_echo "package_go_so $src_path as $pkg"
 248
 249     calculate_go_package_version go_package_version $src_path
 250     cd $WORKSPACE/packages/$TARGET
 251     test_package_presence $pkg $go_package_version go || return 1
 252     cd $WORKSPACE/$src_path
 253     go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
 254     cd $WORKSPACE/packages/$TARGET
 255     local -a fpmargs=(
 256         "--url=https://arvados.org"
 257         "--license=Apache License, Version 2.0"
 258         "--description=${description}"
 259         "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
 260     )
 261     if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
 262         fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go")
 263     fi
 264     if [[ -e "$WORKSPACE/$src_path/README" ]]; then
 265         fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
 266     fi
 267     fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
 268 }
 269
 270 default_iteration() {
 271     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
 272         echo "$ARVADOS_BUILDING_ITERATION"
 273         return
 274     fi
 275     local package_name="$1"; shift
 276     local package_version="$1"; shift
 277     local package_type="$1"; shift
 278     local iteration=1
 279     if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
 280            [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
 281         iteration=2
 282     fi
 283     echo $iteration
 284 }
 285
 286 _build_rails_package_scripts() {
 287     local pkgname="$1"; shift
 288     local destdir="$1"; shift
 289     local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
 290     for scriptname in postinst prerm postrm; do
 291         cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
 292             >"$destdir/$scriptname" || return $?
 293     done
 294 }
 295
 296 rails_package_version() {
 297     local pkgname="$1"; shift
 298     local srcdir="$1"; shift
 299     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
 300         echo "$ARVADOS_BUILDING_VERSION"
 301         return
 302     fi
 303     local version="$(version_from_git)"
 304     if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then
 305         calculate_go_package_version version cmd/arvados-server "$srcdir"
 306     fi
 307     echo $version
 308 }
 309
 310 test_rails_package_presence() {
 311   local pkgname="$1"; shift
 312   local srcdir="$1"; shift
 313
 314   if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
 315     return 1
 316   fi
 317
 318   tmppwd=`pwd`
 319
 320   cd $srcdir
 321
 322   local version="$(rails_package_version "$pkgname" "$srcdir")"
 323
 324   cd $tmppwd
 325
 326   test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
 327 }
 328
 329 get_complete_package_name() {
 330   # if the errexit flag is set, unset it until this function returns
 331   # otherwise, the shift calls below will abort the program if optional arguments are not supplied
 332   if [ -o errexit ]; then
 333     set +e
 334     trap 'set -e' RETURN
 335   fi
 336   # $__returnvar has the nameref attribute set, which means it is a reference
 337   # to another variable that is passed in as the first argument to this function.
 338   # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
 339   local -n __returnvar="$1"; shift
 340   local pkgname="$1"; shift
 341   local version="$1"; shift
 342   local pkgtype="$1"; shift
 343   local iteration="$1"; shift
 344   local arch="$1"; shift
 345   if [[ "$iteration" == "" ]]; then
 346       iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
 347   fi
 348
 349   if [[ "$arch" == "" ]]; then
 350     native_arch="amd64"
 351     rpm_native_arch="x86_64"
 352     if [[ "$HOSTTYPE" == "aarch64" ]]; then
 353       native_arch="arm64"
 354       rpm_native_arch="arm64"
 355     fi
 356     rpm_architecture="$rpm_native_arch"
 357     deb_architecture="$native_arch"
 358
 359     if [[ "$pkgtype" =~ ^(src)$ ]]; then
 360       rpm_architecture="noarch"
 361       deb_architecture="all"
 362     fi
 363
 364     # These python packages have binary components
 365     if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
 366       rpm_architecture="$rpm_native_arch"
 367       deb_architecture="$native_arch"
 368     fi
 369   else
 370     rpm_architecture=$arch
 371     deb_architecture=$arch
 372   fi
 373
 374   local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
 375   if [[ "$FORMAT" == "rpm" ]]; then
 376       # rpm packages get iteration 1 if we don't supply one
 377       iteration=${iteration:-1}
 378       complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
 379   fi
 380   __returnvar=${complete_pkgname}
 381 }
 382
 383 # Test if the package already exists, if not return 0, if it does return 1
 384 test_package_presence() {
 385     local pkgname="$1"; shift
 386     local version="$1"; shift
 387     local pkgtype="$1"; shift
 388     local iteration="$1"; shift
 389     local arch="$1"; shift
 390     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
 391       # arvados-workbench depends on arvados-server at build time, so even when
 392       # only arvados-workbench is being built, we need to build arvados-server too
 393       if [[ "$pkgname" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
 394         return 1
 395       fi
 396     fi
 397
 398     local full_pkgname
 399     get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
 400
 401     # See if we can skip building the package, only if it already exists in the
 402     # processed/ directory. If so, move it back to the packages directory to make
 403     # sure it gets picked up by the test and/or upload steps.
 404     # Get the list of packages from the repos
 405
 406     if [[ "$FORCE_BUILD" == "1" ]]; then
 407       echo "Package $full_pkgname build forced with --force-build, building"
 408     elif [[ "$FORMAT" == "deb" ]]; then
 409       declare -A dd
 410       dd[debian10]=buster
 411       dd[debian11]=bullseye
 412       dd[ubuntu1804]=bionic
 413       dd[ubuntu2004]=focal
 414       D=${dd[$TARGET]}
 415       if [ ${pkgname:0:3} = "lib" ]; then
 416         repo_subdir=${pkgname:0:4}
 417       else
 418         repo_subdir=${pkgname:0:1}
 419       fi
 420
 421       repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
 422       echo "${repo_pkg_list}" |grep -q ${full_pkgname}
 423       if [ $? -eq 0 ] ; then
 424         echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
 425         curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
 426         return 1
 427       elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
 428         echo "Package $full_pkgname exists, not rebuilding!"
 429         return 1
 430       else
 431         echo "Package $full_pkgname not found, building"
 432         return 0
 433       fi
 434     else
 435       centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
 436
 437       repo_pkg_list=$(curl -s -o - ${centos_repo})
 438       echo ${repo_pkg_list} |grep -q ${full_pkgname}
 439       if [ $? -eq 0 ]; then
 440         echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
 441         curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" ${centos_repo}${full_pkgname}
 442         return 1
 443       elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
 444         echo "Package $full_pkgname exists, not rebuilding!"
 445         return 1
 446       else
 447         echo "Package $full_pkgname not found, building"
 448         return 0
 449       fi
 450     fi
 451 }
 452
 453 handle_rails_package() {
 454     local pkgname="$1"; shift
 455
 456     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
 457         return 0
 458     fi
 459     local srcdir="$1"; shift
 460     cd "$srcdir"
 461     local license_path="$1"; shift
 462     local version="$(rails_package_version "$pkgname" "$srcdir")"
 463     echo "$version" >package-build.version
 464     local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
 465     (
 466         set -e
 467         _build_rails_package_scripts "$pkgname" "$scripts_dir"
 468         cd "$srcdir"
 469         mkdir -p tmp
 470         git rev-parse HEAD >git-commit.version
 471         bundle package --all
 472     )
 473     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
 474         echo "ERROR: $pkgname package prep failed" >&2
 475         rm -rf "$scripts_dir"
 476         EXITCODE=1
 477         return 1
 478     fi
 479     local railsdir="/var/www/${pkgname%-server}/current"
 480     local -a pos_args=("$srcdir/=$railsdir" "$pkgname" dir "$version")
 481     local license_arg="$license_path=$railsdir/$(basename "$license_path")"
 482     local -a switches=(--after-install "$scripts_dir/postinst"
 483                        --before-remove "$scripts_dir/prerm"
 484                        --after-remove "$scripts_dir/postrm")
 485     if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
 486         switches+=(--iteration $RAILS_PACKAGE_ITERATION)
 487     fi
 488     # For some reason fpm excludes need to not start with /.
 489     local exclude_root="${railsdir#/}"
 490     local -a exclude_list=(tmp log coverage Capfile\* \
 491                            config/deploy\* config/application.yml)
 492     # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
 493     if  [[ "$pkgname" != "arvados-workbench" ]]; then
 494       exclude_list+=('config/database.yml')
 495     fi
 496     for exclude in ${exclude_list[@]}; do
 497         switches+=(-x "$exclude_root/$exclude")
 498     done
 499     fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
 500               -x "$exclude_root/vendor/cache-*" \
 501               -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
 502     rm -rf "$scripts_dir"
 503 }
 504
 505 # Build python packages with a virtualenv built-in
 506 fpm_build_virtualenv () {
 507   PKG=$1
 508   shift
 509   PKG_DIR=$1
 510   shift
 511   PACKAGE_TYPE=${1:-python}
 512   shift
 513   native_arch="amd64"
 514   if [[ "$HOSTTYPE" == "aarch64" ]]; then
 515     native_arch="arm64"
 516   fi
 517
 518   if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
 519       fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
 520   elif [[ -z "$ARCH" ]]; then
 521     for arch in $native_arch; do
 522       fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
 523     done
 524   else
 525     echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
 526   fi
 527 }
 528
 529 # Build python packages with a virtualenv built-in
 530 fpm_build_virtualenv_worker () {
 531   PKG=$1
 532   shift
 533   PKG_DIR=$1
 534   shift
 535   PACKAGE_TYPE=${1:-python}
 536   shift
 537   arch=${1:-amd64}
 538   shift
 539
 540   # Set up
 541   STDOUT_IF_DEBUG=/dev/null
 542   STDERR_IF_DEBUG=/dev/null
 543   DASHQ_UNLESS_DEBUG=-q
 544   if [[ "$DEBUG" != "0" ]]; then
 545       STDOUT_IF_DEBUG=/dev/stdout
 546       STDERR_IF_DEBUG=/dev/stderr
 547       DASHQ_UNLESS_DEBUG=
 548   fi
 549   if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
 550     ARVADOS_BUILDING_ITERATION=1
 551   fi
 552
 553   local python=""
 554   case "$PACKAGE_TYPE" in
 555     python3)
 556         python=python3
 557         pip=pip3
 558         PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
 559         ;;
 560   esac
 561
 562   if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
 563     PYTHON_PKG=$PACKAGE_PREFIX-$PKG
 564   else
 565     # Exception to our package naming convention
 566     PYTHON_PKG=$PKG
 567   fi
 568
 569   # arvados-python-client sdist should always be built, to be available
 570   # for other dependent packages.
 571   if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
 572     return 0
 573   fi
 574
 575   cd $WORKSPACE/$PKG_DIR
 576
 577   rm -rf dist/*
 578
 579   # Get the latest setuptools
 580   if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
 581     echo "Error, unable to upgrade setuptools with"
 582     echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
 583     exit 1
 584   fi
 585   # filter a useless warning (when building the cwltest package) from the stderr output
 586   if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
 587     echo "Error, unable to run $python setup.py sdist for $PKG"
 588     exit 1
 589   fi
 590
 591   PACKAGE_PATH=`(cd dist; ls *tar.gz)`
 592
 593   if [[ "arvados-python-client" == "$PKG" ]]; then
 594     PYSDK_PATH=`pwd`/dist/
 595   fi
 596
 597   if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
 598     return 0
 599   fi
 600
 601   # Determine the package version from the generated sdist archive
 602   if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
 603       UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
 604       PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
 605   else
 606       PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
 607       UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
 608   fi
 609
 610   # See if we actually need to build this package; does it exist already?
 611   # We can't do this earlier than here, because we need PYTHON_VERSION...
 612   # This isn't so bad; the sdist call above is pretty quick compared to
 613   # the invocation of virtualenv and fpm, below.
 614   if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
 615     return 0
 616   fi
 617
 618   echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
 619
 620   # Package the sdist in a virtualenv
 621   echo "Creating virtualenv..."
 622
 623   cd dist
 624
 625   rm -rf build
 626   rm -f $PYTHON_PKG*deb
 627   echo "virtualenv version: `virtualenv --version`"
 628   virtualenv_command="virtualenv --python `which $python` $DASHQ_UNLESS_DEBUG build/usr/share/$python/dist/$PYTHON_PKG"
 629
 630   if ! $virtualenv_command; then
 631     echo "Error, unable to run"
 632     echo "  $virtualenv_command"
 633     exit 1
 634   fi
 635
 636   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
 637     echo "Error, unable to upgrade pip with"
 638     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
 639     exit 1
 640   fi
 641   echo "pip version:        `build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip --version`"
 642
 643   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
 644     echo "Error, unable to upgrade setuptools with"
 645     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
 646     exit 1
 647   fi
 648   echo "setuptools version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/$python -c 'import setuptools; print(setuptools.__version__)'`"
 649
 650   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
 651     echo "Error, unable to upgrade wheel with"
 652     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
 653     exit 1
 654   fi
 655   echo "wheel version:      `build/usr/share/$python/dist/$PYTHON_PKG/bin/wheel version`"
 656
 657   if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
 658     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
 659   else
 660     # centos7 needs these special tweaks to install python-arvados-fuse
 661     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
 662     PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
 663   fi
 664
 665   if [[ "$?" != "0" ]]; then
 666     echo "Error, unable to run"
 667     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH"
 668     exit 1
 669   fi
 670
 671   cd build/usr/share/$python/dist/$PYTHON_PKG/
 672
 673   # Replace the shebang lines in all python scripts, and handle the activate
 674   # scripts too. This is a functional replacement of the 237 line
 675   # virtualenv_tools.py script that doesn't work in python3 without serious
 676   # patching, minus the parts we don't need (modifying pyc files, etc).
 677   for binfile in `ls bin/`; do
 678     if ! file --mime bin/$binfile |grep -q binary; then
 679       # Not a binary file
 680       if [[ "$binfile" =~ ^activate(.csh|.fish|)$ ]]; then
 681         # these 'activate' scripts need special treatment
 682         sed -i "s/VIRTUAL_ENV=\".*\"/VIRTUAL_ENV=\"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
 683         sed -i "s/VIRTUAL_ENV \".*\"/VIRTUAL_ENV \"\/usr\/share\/$python\/dist\/$PYTHON_PKG\"/" bin/$binfile
 684       else
 685         if grep -q -E '^#!.*/bin/python\d?' bin/$binfile; then
 686           # Replace shebang line
 687           sed -i "1 s/^.*$/#!\/usr\/share\/$python\/dist\/$PYTHON_PKG\/bin\/python/" bin/$binfile
 688         fi
 689       fi
 690     fi
 691   done
 692
 693   cd - >$STDOUT_IF_DEBUG
 694
 695   find build -iname '*.pyc' -exec rm {} \;
 696   find build -iname '*.pyo' -exec rm {} \;
 697
 698   # Finally, generate the package
 699   echo "Creating package..."
 700
 701   declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
 702
 703   if [[ "${arch}" != "amd64" ]]; then
 704     COMMAND_ARR+=("-a${arch}")
 705   fi
 706
 707   if [[ "$MAINTAINER" != "" ]]; then
 708     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
 709   fi
 710
 711   if [[ "$VENDOR" != "" ]]; then
 712     COMMAND_ARR+=('--vendor' "$VENDOR")
 713   fi
 714
 715   COMMAND_ARR+=('--url' 'https://arvados.org')
 716
 717   # Get description
 718   DESCRIPTION=`grep '\sdescription' $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
 719   COMMAND_ARR+=('--description' "$DESCRIPTION")
 720
 721   # Get license string
 722   LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
 723   COMMAND_ARR+=('--license' "$LICENSE_STRING")
 724
 725   if [[ "$FORMAT" == "rpm" ]]; then
 726     # Make sure to conflict with the old rh-python36 packages we used to publish
 727     COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
 728   fi
 729
 730   if [[ "$DEBUG" != "0" ]]; then
 731     COMMAND_ARR+=('--verbose' '--log' 'info')
 732   fi
 733
 734   COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g))
 735   COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
 736   COMMAND_ARR+=('-n' "$PYTHON_PKG")
 737   COMMAND_ARR+=('-C' "build")
 738
 739   systemd_unit="$WORKSPACE/$PKG_DIR/$PKG.service"
 740   if [[ -e "${systemd_unit}" ]]; then
 741     COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
 742     COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
 743   fi
 744
 745   COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
 746
 747   # avoid warning
 748   COMMAND_ARR+=('--deb-no-default-config-files')
 749
 750   # Append --depends X and other arguments specified by fpm-info.sh in
 751   # the package source dir. These are added last so they can override
 752   # the arguments added by this script.
 753   declare -a fpm_args=()
 754   declare -a fpm_depends=()
 755
 756   fpminfo="$WORKSPACE/$PKG_DIR/fpm-info.sh"
 757   if [[ -e "$fpminfo" ]]; then
 758     echo "Loading fpm overrides from $fpminfo"
 759     if ! source "$fpminfo"; then
 760       echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
 761       exit 1
 762     fi
 763   fi
 764
 765   for i in "${fpm_depends[@]}"; do
 766     COMMAND_ARR+=('--depends' "$i")
 767   done
 768
 769   for i in "${fpm_depends[@]}"; do
 770     COMMAND_ARR+=('--replaces' "python-$PKG")
 771   done
 772
 773   # make sure the systemd service file ends up in the right place
 774   # used by arvados-docker-cleaner
 775   if [[ -e "${systemd_unit}" ]]; then
 776     COMMAND_ARR+=("usr/share/$python/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
 777   fi
 778
 779   COMMAND_ARR+=("${fpm_args[@]}")
 780
 781   # Make sure to install all our package binaries in /usr/bin.
 782   # We have to walk $WORKSPACE/$PKG_DIR/bin rather than
 783   # $WORKSPACE/build/usr/share/$python/dist/$PYTHON_PKG/bin/ to get the list
 784   # because the latter also includes all the python binaries for the virtualenv.
 785   # We have to take the copies of our binaries from the latter directory, though,
 786   # because those are the ones we rewrote the shebang line of, above.
 787   if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
 788     for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
 789       COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/$binary=/usr/bin/")
 790     done
 791   fi
 792
 793   # the python3-arvados-cwl-runner package comes with cwltool, expose that version
 794   if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
 795     COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
 796   fi
 797
 798   COMMAND_ARR+=(".")
 799
 800   debug_echo -e "\n${COMMAND_ARR[@]}\n"
 801
 802   FPM_RESULTS=$("${COMMAND_ARR[@]}")
 803   FPM_EXIT_CODE=$?
 804
 805   # if something went wrong and debug is off, print out the fpm command that errored
 806   if ! fpm_verify $FPM_EXIT_CODE $FPM_RESULTS && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
 807     echo "fpm returned an error executing the command:"
 808     echo
 809     echo -e "\n${COMMAND_ARR[@]}\n"
 810   else
 811     echo `ls *$FORMAT`
 812     mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/
 813   fi
 814   echo
 815 }
 816
 817 # build_metapackage builds meta packages that help with the python to python 3 package migration
 818 build_metapackage() {
 819   # base package name (e.g. arvados-python-client)
 820   BASE_NAME=$1
 821   shift
 822   PKG_DIR=$1
 823   shift
 824
 825   if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then
 826     return 0
 827   fi
 828
 829   if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
 830     ARVADOS_BUILDING_ITERATION=1
 831   fi
 832
 833   if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
 834     cd $WORKSPACE/$PKG_DIR
 835     pwd
 836     rm -rf dist/*
 837
 838     # Get the latest setuptools
 839     if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
 840       echo "Error, unable to upgrade setuptools with XY"
 841       echo "  pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
 842       exit 1
 843     fi
 844     # filter a useless warning (when building the cwltest package) from the stderr output
 845     if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
 846       echo "Error, unable to run python3 setup.py sdist for $PKG"
 847       exit 1
 848     fi
 849
 850     PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
 851     UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
 852
 853   else
 854     UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
 855     PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
 856   fi
 857
 858   cd - >$STDOUT_IF_DEBUG
 859   if [[ -d "$BASE_NAME" ]]; then
 860     rm -rf $BASE_NAME
 861   fi
 862   mkdir $BASE_NAME
 863   cd $BASE_NAME
 864
 865   if [[ "$FORMAT" == "deb" ]]; then
 866     cat >ns-control <<EOF
 867 Section: misc
 868 Priority: optional
 869 Standards-Version: 3.9.2
 870
 871 Package: python-${BASE_NAME}
 872 Version: ${PYTHON_VERSION}-${ARVADOS_BUILDING_ITERATION}
 873 Maintainer: Arvados Package Maintainers <packaging@arvados.org>
 874 Depends: python3-${BASE_NAME}
 875 Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME}
 876  This package is a metapackage that will automatically install the new version of
 877  ${BASE_NAME} which is Python 3 based and has a different name.
 878 EOF
 879
 880     /usr/bin/equivs-build ns-control
 881     if [[ $? -ne 0 ]]; then
 882       echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?"
 883       return 1
 884     fi
 885   elif [[ "$FORMAT" == "rpm" ]]; then
 886     cat >meta.spec <<EOF
 887 Summary: metapackage to ease the upgrade to the Python 3 version of ${BASE_NAME}
 888 Name: python-${BASE_NAME}
 889 Version: ${PYTHON_VERSION}
 890 Release: ${ARVADOS_BUILDING_ITERATION}
 891 License: distributable
 892
 893 Requires: python3-${BASE_NAME}
 894
 895 %description
 896 This package is a metapackage that will automatically install the new version of
 897 python-${BASE_NAME} which is Python 3 based and has a different name.
 898
 899 %prep
 900
 901 %build
 902
 903 %clean
 904
 905 %install
 906
 907 %post
 908
 909 %files
 910
 911
 912 %changelog
 913 * Mon Apr 12 2021 Arvados Package Maintainers <packaging@arvados.org>
 914 - initial release
 915 EOF
 916
 917     /usr/bin/rpmbuild -ba meta.spec
 918     if [[ $? -ne 0 ]]; then
 919       echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?"
 920       return 1
 921     else
 922       mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} .
 923       if [[ $? -ne 0 ]]; then
 924         echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'"
 925         return 1
 926       fi
 927     fi
 928   else
 929     echo "Unknown format"
 930     return 1
 931   fi
 932
 933   if [[ $EXITCODE -ne 0 ]]; then
 934     return 1
 935   else
 936     echo `ls *$FORMAT`
 937     mv *$FORMAT $WORKSPACE/packages/$TARGET/
 938   fi
 939
 940   # clean up
 941   cd - >$STDOUT_IF_DEBUG
 942   if [[ -d "$BASE_NAME" ]]; then
 943     rm -rf $BASE_NAME
 944   fi
 945 }
 946
 947 # Build packages for everything
 948 fpm_build () {
 949   # Source dir where fpm-info.sh (if any) will be found.
 950   SRC_DIR=$1
 951   shift
 952   # The package source.  Depending on the source type, this can be a
 953   # path, or the name of the package in an upstream repository (e.g.,
 954   # pip).
 955   PACKAGE=$1
 956   shift
 957   # The name of the package to build.
 958   PACKAGE_NAME=$1
 959   shift
 960   # The type of source package.  Passed to fpm -s.  Default "dir".
 961   PACKAGE_TYPE=${1:-dir}
 962   shift
 963   # Optional: the package version number.  Passed to fpm -v.
 964   VERSION=$1
 965   shift
 966
 967   if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
 968     # arvados-workbench depends on arvados-server at build time, so even when
 969     # only arvados-workbench is being built, we need to build arvados-server too
 970     if [[ "$PACKAGE_NAME" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
 971       return 0
 972     fi
 973   fi
 974
 975   local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
 976
 977   declare -a COMMAND_ARR=("fpm" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
 978   if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
 979       # Dependencies are built from setup.py.  Since setup.py will never
 980       # refer to Debian package iterations, it doesn't make sense to
 981       # enforce those in the .deb dependencies.
 982       COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
 983   fi
 984
 985   if [[ "$DEBUG" != "0" ]]; then
 986     COMMAND_ARR+=('--verbose' '--log' 'info')
 987   fi
 988
 989   if [[ -n "$PACKAGE_NAME" ]]; then
 990     COMMAND_ARR+=('-n' "$PACKAGE_NAME")
 991   fi
 992
 993   if [[ "$MAINTAINER" != "" ]]; then
 994     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
 995   fi
 996
 997   if [[ "$VENDOR" != "" ]]; then
 998     COMMAND_ARR+=('--vendor' "$VENDOR")
 999   fi
1000
1001   if [[ "$VERSION" != "" ]]; then
1002     COMMAND_ARR+=('-v' "$VERSION")
1003   fi
1004   if [[ -n "$default_iteration_value" ]]; then
1005       # We can always add an --iteration here.  If another one is specified in $@,
1006       # that will take precedence, as desired.
1007       COMMAND_ARR+=(--iteration "$default_iteration_value")
1008   fi
1009
1010   # Append --depends X and other arguments specified by fpm-info.sh in
1011   # the package source dir. These are added last so they can override
1012   # the arguments added by this script.
1013   declare -a fpm_args=()
1014   declare -a build_depends=()
1015   declare -a fpm_depends=()
1016   declare -a fpm_conflicts=()
1017   declare -a fpm_exclude=()
1018   if [[ ! -d "$SRC_DIR" ]]; then
1019       echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
1020       exit 1
1021   fi
1022   fpminfo="${SRC_DIR}/fpm-info.sh"
1023   if [[ -e "$fpminfo" ]]; then
1024       debug_echo "Loading fpm overrides from $fpminfo"
1025       source "$fpminfo"
1026   fi
1027   for pkg in "${build_depends[@]}"; do
1028       if [[ $TARGET =~ debian|ubuntu ]]; then
1029           pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
1030           if [[ -e $pkg_deb ]]; then
1031               echo "Installing build_dep $pkg from $pkg_deb"
1032               dpkg -i "$pkg_deb"
1033           else
1034               echo "Attemping to install build_dep $pkg using apt-get"
1035               apt-get install -y "$pkg"
1036           fi
1037           apt-get -y -f install
1038       else
1039           pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
1040           if [[ -e $pkg_rpm ]]; then
1041               echo "Installing build_dep $pkg from $pkg_rpm"
1042               rpm -i "$pkg_rpm"
1043           else
1044               echo "Attemping to install build_dep $pkg"
1045               rpm -i "$pkg"
1046           fi
1047       fi
1048   done
1049   for i in "${fpm_depends[@]}"; do
1050     COMMAND_ARR+=('--depends' "$i")
1051   done
1052   for i in "${fpm_conflicts[@]}"; do
1053     COMMAND_ARR+=('--conflicts' "$i")
1054   done
1055   for i in "${fpm_exclude[@]}"; do
1056     COMMAND_ARR+=('--exclude' "$i")
1057   done
1058
1059   COMMAND_ARR+=("${fpm_args[@]}")
1060
1061   # Append remaining function arguments directly to fpm's command line.
1062   for i; do
1063     COMMAND_ARR+=("$i")
1064   done
1065
1066   COMMAND_ARR+=("$PACKAGE")
1067
1068   debug_echo -e "\n${COMMAND_ARR[@]}\n"
1069
1070   FPM_RESULTS=$("${COMMAND_ARR[@]}")
1071   FPM_EXIT_CODE=$?
1072
1073   fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
1074
1075   # if something went wrong and debug is off, print out the fpm command that errored
1076   if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
1077     echo -e "\n${COMMAND_ARR[@]}\n"
1078   fi
1079 }
1080
1081 # verify build results
1082 fpm_verify () {
1083   FPM_EXIT_CODE=$1
1084   shift
1085   FPM_RESULTS=$@
1086
1087   FPM_PACKAGE_NAME=''
1088   if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
1089     FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
1090   fi
1091
1092   if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
1093     EXITCODE=1
1094     echo
1095     echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
1096     echo
1097     echo $FPM_RESULTS
1098     echo
1099     return 1
1100   elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
1101     echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
1102     return 0
1103   elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
1104     EXITCODE=1
1105     echo "Error building package for $1:\n $FPM_RESULTS"
1106     return 1
1107   fi
1108 }
1109
1110 install_package() {
1111   PACKAGES=$@
1112   if [[ "$FORMAT" == "deb" ]]; then
1113     $SUDO apt-get install $PACKAGES --yes
1114   elif [[ "$FORMAT" == "rpm" ]]; then
1115     $SUDO yum -q -y install $PACKAGES
1116   fi
1117 }
1118
1119 title() {
1120     printf '%s %s\n' "=======" "$1"
1121 }
1122
1123 checkexit() {
1124     if [[ "$1" != "0" ]]; then
1125         title "$2 -- FAILED"
1126         failures+=("$2 (`timer`)")
1127     else
1128         successes+=("$2 (`timer`)")
1129     fi
1130 }
1131
1132 timer_reset() {
1133     t0=$SECONDS
1134 }
1135
1136 timer() {
1137     if [[ -n "$t0" ]]; then
1138         echo -n "$(($SECONDS - $t0))s"
1139     fi
1140 }
1141
1142 report_outcomes() {
1143     for x in "${successes[@]}"
1144     do
1145         echo "Pass: $x"
1146     done
1147
1148     if [[ ${#failures[@]} == 0 ]]
1149     then
1150         if [[ ${#successes[@]} != 0 ]]; then
1151            echo "All test suites passed."
1152         fi
1153     else
1154         echo "Failures (${#failures[@]}):"
1155         for x in "${failures[@]}"
1156         do
1157             echo "Fail: $x"
1158         done
1159     fi
1160 }