X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/85f8ab2ef898f8d0bb7187db6db59a93ac528ec5..198a3409881f5b9f1fe77ae551efde1968bfb639:/build/run-library.sh diff --git a/build/run-library.sh b/build/run-library.sh index 23e09d43d5..83d069c260 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -1,4 +1,7 @@ -#!/bin/bash + #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 # A library of functions shared by the various scripts in this directory. @@ -7,6 +10,12 @@ # older packages. LICENSE_PACKAGE_TS=20151208015500 +if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + RAILS_PACKAGE_ITERATION=8 +else + RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION" +fi + debug_echo () { echo "$@" >"$STDOUT_IF_DEBUG" } @@ -33,20 +42,27 @@ format_last_commit_here() { } version_from_git() { - # Generates a version number from the git log for the current working - # directory, and writes it to stdout. - local git_ts git_hash prefix - if [[ -n "$1" ]] ; then - prefix="$1" - else - prefix="0.1" - fi + # Output the version being built, or if we're building a + # dev/prerelease, output a version number based on the git log for + # the current working directory. + if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then + echo "$ARVADOS_BUILDING_VERSION" + return + fi + + ARVADOS_BUILDING_VERSION=`git describe --abbrev=0` + local git_ts - declare $(format_last_commit_here "git_ts=%ct git_hash=%h") - echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash" + declare $(format_last_commit_here "git_ts=%ct") + ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION.$(date -ud "@$git_ts" +%Y%m%d%H%M%S)" + echo "$ARVADOS_BUILDING_VERSION" } nohash_version_from_git() { + if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then + echo "$ARVADOS_BUILDING_VERSION" + return + fi version_from_git $1 | cut -d. -f1-3 } @@ -56,7 +72,7 @@ timestamp_from_git() { handle_python_package () { # This function assumes the current working directory is the python package directory - if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then + if [ -n "$(find dist -name "*-$ARVADOS_BUILDING_VERSION.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).tar.gz" -print -quit)" ]; then # This package doesn't need rebuilding. return fi @@ -98,24 +114,41 @@ package_go_binary() { mkdir -p "$GOPATH/src/git.curoverse.com" ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" + (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v) cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path" local version="$(version_from_git)" local timestamp="$(timestamp_from_git)" - # If the command imports anything from the Arvados SDK, bump the - # version number and build a new package whenever the SDK changes. + # Update the version number and build a new package if the vendor + # bundle has changed, or the command imports anything from the + # Arvados SDK and the SDK has changed. + declare -a checkdirs=(vendor) if grep -qr git.curoverse.com/arvados .; then - cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go" - if [[ $(timestamp_from_git) -gt "$timestamp" ]]; then - version=$(version_from_git) + checkdirs+=(sdk/go) + if [[ "$prog" -eq "crunch-dispatch-slurm" ]]; then + checkdirs+=(lib/dispatchcloud) fi fi + for dir in ${checkdirs[@]}; do + cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir" + ts="$(timestamp_from_git)" + if [[ "$ts" -gt "$timestamp" ]]; then + version=$(version_from_git) + timestamp="$ts" + fi + done cd $WORKSPACE/packages/$TARGET - go get "git.curoverse.com/arvados.git/$src_path" + test_package_presence $prog $version go + + if [[ "$?" != "0" ]]; then + return 1 + fi + + go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path" - declare -a switches=() + local -a switches=() systemd_unit="$WORKSPACE/${src_path}/${prog}.service" if [[ -e "${systemd_unit}" ]]; then switches+=( @@ -129,13 +162,22 @@ package_go_binary() { } default_iteration() { + if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then + echo "$ARVADOS_BUILDING_ITERATION" + return + fi local package_name="$1"; shift local package_version="$1"; shift + local package_type="$1"; shift local iteration=1 if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \ [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then iteration=2 fi + if [[ $package_type =~ ^python ]]; then + # Fix --iteration for #9242. + iteration=2 + fi echo $iteration } @@ -149,39 +191,119 @@ _build_rails_package_scripts() { done } +test_rails_package_presence() { + local pkgname="$1"; shift + local srcdir="$1"; shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then + return 1 + fi + + tmppwd=`pwd` + + cd $srcdir + + local version="$(version_from_git)" + + cd $tmppwd + + test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION" +} + +test_package_presence() { + local pkgname="$1"; shift + local version="$1"; shift + local pkgtype="$1"; shift + local iteration="$1"; shift + local arch="$1"; shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then + return 1 + fi + + if [[ "$iteration" == "" ]]; then + iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")" + fi + + if [[ "$arch" == "" ]]; then + rpm_architecture="x86_64" + deb_architecture="amd64" + + if [[ "$pkgtype" =~ ^(python|python3)$ ]]; then + rpm_architecture="noarch" + deb_architecture="all" + fi + + if [[ "$pkgtype" =~ ^(src)$ ]]; then + rpm_architecture="noarch" + deb_architecture="all" + fi + + # These python packages have binary components + if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then + rpm_architecture="x86_64" + deb_architecture="amd64" + fi + else + rpm_architecture=$arch + deb_architecture=$arch + fi + + if [[ "$FORMAT" == "deb" ]]; then + local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb" + else + # rpm packages get iteration 1 if we don't supply one + iteration=${iteration:-1} + local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm" + fi + + # See if we can skip building the package, only if it already exists in the + # processed/ directory. If so, move it back to the packages directory to make + # sure it gets picked up by the test and/or upload steps. + if [[ -e "processed/$complete_pkgname" ]]; then + echo "Package $complete_pkgname exists, not rebuilding!" + mv processed/$complete_pkgname . + return 1 + else + echo "Package $complete_pkgname not found, building" + return 0 + fi +} + handle_rails_package() { local pkgname="$1"; shift if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then return 0 fi - local srcdir="$1"; shift + cd "$srcdir" local license_path="$1"; shift + local version="$(version_from_git)" local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \ - local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && ( + ( set -e _build_rails_package_scripts "$pkgname" "$scripts_dir" cd "$srcdir" mkdir -p tmp - version_from_git >"$version_file" git rev-parse HEAD >git-commit.version bundle package --all ) if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then echo "ERROR: $pkgname package prep failed" >&2 - rm -rf "$scripts_dir" "$version_file" + rm -rf "$scripts_dir" EXITCODE=1 return 1 fi local railsdir="/var/www/${pkgname%-server}/current" - local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir - "$(cat "$version_file")") + local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version") local license_arg="$license_path=$railsdir/$(basename "$license_path")" - local -a switches=(--iteration=7 - --after-install "$scripts_dir/postinst" + local -a switches=(--after-install "$scripts_dir/postinst" --before-remove "$scripts_dir/prerm" --after-remove "$scripts_dir/postrm") + if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + switches+=(--iteration $RAILS_PACKAGE_ITERATION) + fi # For some reason fpm excludes need to not start with /. local exclude_root="${railsdir#/}" # .git and packages are for the SSO server, which is built from its @@ -196,8 +318,9 @@ handle_rails_package() { switches+=(-x "$exclude_root/$exclude") done fpm_build "${pos_args[@]}" "${switches[@]}" \ + -x "$exclude_root/vendor/cache-*" \ -x "$exclude_root/vendor/bundle" "$@" "$license_arg" - rm -rf "$scripts_dir" "$version_file" + rm -rf "$scripts_dir" } # Build packages for everything @@ -225,12 +348,14 @@ fpm_build () { return 0 fi - local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")" + local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")" + local python="" case "$PACKAGE_TYPE" in python) # All Arvados Python2 packages depend on Python 2.7. # Make sure we build with that for consistency. + python=python2.7 set -- "$@" --python-bin python2.7 \ --python-easyinstall "$EASY_INSTALL2" \ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \ @@ -239,8 +364,6 @@ fpm_build () { --python-install-data . \ --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \ --depends "$PYTHON2_PACKAGE" - # Fix --iteration for #9242. - default_iteration_value=$(($default_iteration_value + 1)) ;; python3) # fpm does not actually support a python3 package type. Instead @@ -248,6 +371,7 @@ fpm_build () { # necessary arguments to fpm's command line later, after we're # done handling positional arguments. PACKAGE_TYPE=python + python=python3 set -- "$@" --python-bin python3 \ --python-easyinstall "$EASY_INSTALL3" \ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \ @@ -256,8 +380,6 @@ fpm_build () { --python-install-data . \ --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \ --depends "$PYTHON3_PACKAGE" - # Fix --iteration for #9242. - default_iteration_value=$(($default_iteration_value + 1)) ;; esac @@ -269,6 +391,17 @@ fpm_build () { COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies) fi + # 12271 - As FPM-generated packages don't include scripts by default, the + # packages cleanup on upgrade depends on files being listed on the %files + # section in the generated SPEC files. To remove DIRECTORIES, they need to + # be listed in that sectiontoo, so we need to add this parameter to properly + # remove lingering dirs. But this only works for python2: if used on + # python33, it includes dirs like /opt/rh/python33 that belong to + # other packages. + if [[ "$FORMAT" = rpm ]] && [[ "$python" = python2.7 ]]; then + COMMAND_ARR+=('--rpm-auto-add-directories') + fi + if [[ "${DEBUG:-0}" != "0" ]]; then COMMAND_ARR+=('--verbose' '--log' 'info') fi @@ -284,9 +417,11 @@ fpm_build () { if [[ "$VERSION" != "" ]]; then COMMAND_ARR+=('-v' "$VERSION") fi - # We can always add an --iteration here. If another one is specified in $@, - # that will take precedence, as desired. - COMMAND_ARR+=(--iteration "$default_iteration_value") + if [[ -n "$default_iteration_value" ]]; then + # We can always add an --iteration here. If another one is specified in $@, + # that will take precedence, as desired. + COMMAND_ARR+=(--iteration "$default_iteration_value") + fi if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]] then @@ -363,6 +498,11 @@ fpm_build () { FPM_EXIT_CODE=$? fpm_verify $FPM_EXIT_CODE $FPM_RESULTS + + # if something went wrong and debug is off, print out the fpm command that errored + if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then + echo -e "\n${COMMAND_ARR[@]}\n" + fi } # verify build results @@ -378,14 +518,19 @@ fpm_verify () { if [[ "$FPM_PACKAGE_NAME" == "" ]]; then EXITCODE=1 + echo echo "Error: $PACKAGE: Unable to figure out package name from fpm results:" echo echo $FPM_RESULTS echo + return 1 elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then echo "Package $FPM_PACKAGE_NAME exists, not rebuilding" + return 0 elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then + EXITCODE=1 echo "Error building package for $1:\n $FPM_RESULTS" + return 1 fi }