X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c0cbdeb1567d4a4f190a01d3fe89aa975e51e47b..964c763379e2ea98e46584267342ced694e3349a:/build/run-library.sh diff --git a/build/run-library.sh b/build/run-library.sh index a085843d29..92140bbdc8 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -61,11 +61,12 @@ version_from_git() { } nohash_version_from_git() { + local subdir="$1"; shift if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then echo "$ARVADOS_BUILDING_VERSION" return fi - version_from_git | cut -d. -f1-4 + version_from_git $subdir | cut -d. -f1-4 } timestamp_from_git() { @@ -74,25 +75,8 @@ timestamp_from_git() { } calculate_python_sdk_cwl_package_versions() { - python_sdk_ts=$(cd sdk/python && timestamp_from_git) - cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git) - - python_sdk_version=$(cd sdk/python && nohash_version_from_git) - cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git) - - if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then - cwl_runner_version=$python_sdk_version - fi -} - -handle_python_package () { - # This function assumes the current working directory is the python package directory - if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then - # This package doesn't need rebuilding. - return - fi - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - python setup.py $DASHQ_UNLESS_DEBUG sdist + python_sdk_version=$(cd sdk/python && python3 arvados_version.py) + cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py) } handle_ruby_gem() { @@ -130,9 +114,9 @@ calculate_go_package_version() { checkdirs+=("$1") shift done - if grep -qr git.arvados.org/arvados .; then - checkdirs+=(sdk/go lib) - fi + # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server) + # Everything depends on the build directory. + checkdirs+=(sdk/go lib build) local timestamp=0 for dir in ${checkdirs[@]}; do cd "$WORKSPACE" @@ -146,10 +130,12 @@ calculate_go_package_version() { __returnvar="$version" } -# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt] +# Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt] package_go_binary() { local src_path="$1"; shift local prog="$1"; shift + local package_format="$1"; shift + local target_arch="$1"; shift local description="$1"; shift local license_file="${1:-agpl-3.0.txt}"; shift @@ -161,21 +147,78 @@ package_go_binary() { fi fi - debug_echo "package_go_binary $src_path as $prog" + # Only amd64 and aarch64 are supported + native_arch="amd64" + if [[ "$HOSTTYPE" == "aarch64" ]]; then + native_arch="arm64" + fi + + if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then + echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch" + return 1 + fi + if [[ -n "$target_arch" ]]; then + # A target architecture has been specified + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file" + return $? + else + # No target architecture specified, default to native target. When on amd64 also crosscompile arm64 + # but only when building deb packages (centos does not have support for crosscompiling userspace). + archs=($native_arch) + if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then + archs=('amd64' 'arm64') + fi + for ta in $archs; do + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file" + retval=$? + if [[ "$retval" != 0 ]]; then + return $retval + fi + done + fi +} + +# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt] +package_go_binary_worker() { + local src_path="$1"; shift + local prog="$1"; shift + local package_format="$1"; shift + local description="$1"; shift + local native_arch="${1:-amd64}"; shift + local target_arch="${1:-amd64}"; shift + local license_file="${1:-agpl-3.0.txt}"; shift + + if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then + echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch" + return 1 + fi + + debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)" local basename="${src_path##*/}" calculate_go_package_version go_package_version $src_path cd $WORKSPACE/packages/$TARGET - test_package_presence $prog $go_package_version go - + test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch" if [[ "$?" != "0" ]]; then return 1 fi - go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" + echo "Building $pachage_format ($target_arch) package for $prog from $src_path" + if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then + CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" + else + GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" + fi local -a switches=() + + binpath=$GOPATH/bin/${basename} + if [[ "${target_arch}" != "${native_arch}" ]]; then + switches+=("-a${target_arch}") + binpath="$GOPATH/bin/linux_${target_arch}/${basename}" + fi + systemd_unit="$WORKSPACE/${src_path}/${prog}.service" if [[ -e "${systemd_unit}" ]]; then switches+=( @@ -185,16 +228,22 @@ package_go_binary() { fi switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}") - fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}" + fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}" } -# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library" +# Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library" package_go_so() { local src_path="$1"; shift local sofile="$1"; shift local pkg="$1"; shift + local package_format="$1"; shift + local target_arch="$1"; shift # supported: amd64, arm64 local description="$1"; shift + if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then + return 0 + fi + debug_echo "package_go_so $src_path as $pkg" calculate_go_package_version go_package_version $src_path @@ -253,7 +302,7 @@ rails_package_version() { fi local version="$(version_from_git)" if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then - calculate_go_package_version version cmd/arvados-server "$srcdir" + calculate_go_package_version version cmd/arvados-server "$srcdir" fi echo $version } @@ -298,8 +347,14 @@ get_complete_package_name() { fi if [[ "$arch" == "" ]]; then - rpm_architecture="x86_64" - deb_architecture="amd64" + native_arch="amd64" + rpm_native_arch="x86_64" + if [[ "$HOSTTYPE" == "aarch64" ]]; then + native_arch="arm64" + rpm_native_arch="arm64" + fi + rpm_architecture="$rpm_native_arch" + deb_architecture="$native_arch" if [[ "$pkgtype" =~ ^(src)$ ]]; then rpm_architecture="noarch" @@ -308,8 +363,8 @@ get_complete_package_name() { # These python packages have binary components if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then - rpm_architecture="x86_64" - deb_architecture="amd64" + rpm_architecture="$rpm_native_arch" + deb_architecture="$native_arch" fi else rpm_architecture=$arch @@ -341,7 +396,7 @@ test_package_presence() { fi local full_pkgname - get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch + get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch" # See if we can skip building the package, only if it already exists in the # processed/ directory. If so, move it back to the packages directory to make @@ -352,10 +407,10 @@ test_package_presence() { echo "Package $full_pkgname build forced with --force-build, building" elif [[ "$FORMAT" == "deb" ]]; then declare -A dd - dd[debian9]=stretch dd[debian10]=buster - dd[ubuntu1604]=xenial + dd[debian11]=bullseye dd[ubuntu1804]=bionic + dd[ubuntu2004]=focal D=${dd[$TARGET]} if [ ${pkgname:0:3} = "lib" ]; then repo_subdir=${pkgname:0:4} @@ -363,11 +418,11 @@ test_package_presence() { repo_subdir=${pkgname:0:1} fi - repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/) + repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/) echo "${repo_pkg_list}" |grep -q ${full_pkgname} if [ $? -eq 0 ] ; then echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!" - curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname} + curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname} return 1 elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then echo "Package $full_pkgname exists, not rebuilding!" @@ -432,9 +487,7 @@ handle_rails_package() { fi # For some reason fpm excludes need to not start with /. local exclude_root="${railsdir#/}" - # .git and packages are for the SSO server, which is built from its - # repository root. - local -a exclude_list=(.git packages tmp log coverage Capfile\* \ + local -a exclude_list=(tmp log coverage Capfile\* \ config/deploy\* config/application.yml) # for arvados-workbench, we need to have the (dummy) config/database.yml in the package if [[ "$pkgname" != "arvados-workbench" ]]; then @@ -457,6 +510,32 @@ fpm_build_virtualenv () { shift PACKAGE_TYPE=${1:-python} shift + native_arch="amd64" + if [[ "$HOSTTYPE" == "aarch64" ]]; then + native_arch="arm64" + fi + + if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then + fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH" + elif [[ -z "$ARCH" ]]; then + for arch in $native_arch; do + fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch" + done + else + echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH" + fi +} + +# Build python packages with a virtualenv built-in +fpm_build_virtualenv_worker () { + PKG=$1 + shift + PKG_DIR=$1 + shift + PACKAGE_TYPE=${1:-python} + shift + arch=${1:-amd64} + shift # Set up STDOUT_IF_DEBUG=/dev/null @@ -475,12 +554,7 @@ fpm_build_virtualenv () { case "$PACKAGE_TYPE" in python3) python=python3 - if [[ "$FORMAT" != "rpm" ]]; then - pip=pip3 - else - # In CentOS, we use a different mechanism to get the right version of pip - pip=pip - fi + pip=pip3 PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX ;; esac @@ -526,22 +600,22 @@ fpm_build_virtualenv () { # Determine the package version from the generated sdist archive if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then - UNFILTERED_PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION) - PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~rc/rc/g) + UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) else - UNFILTERED_PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') fi # See if we actually need to build this package; does it exist already? # We can't do this earlier than here, because we need PYTHON_VERSION... # This isn't so bad; the sdist call above is pretty quick compared to # the invocation of virtualenv and fpm, below. - if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then + if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then return 0 fi - echo "Building $FORMAT package for $PKG from $PKG_DIR" + echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR" # Package the sdist in a virtualenv echo "Creating virtualenv..." @@ -626,6 +700,10 @@ fpm_build_virtualenv () { declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT") + if [[ "${arch}" != "amd64" ]]; then + COMMAND_ARR+=("-a${arch}") + fi + if [[ "$MAINTAINER" != "" ]]; then COMMAND_ARR+=('--maintainer' "$MAINTAINER") fi @@ -644,11 +722,16 @@ fpm_build_virtualenv () { LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"` COMMAND_ARR+=('--license' "$LICENSE_STRING") + if [[ "$FORMAT" == "rpm" ]]; then + # Make sure to conflict with the old rh-python36 packages we used to publish + COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG") + fi + if [[ "$DEBUG" != "0" ]]; then COMMAND_ARR+=('--verbose' '--log' 'info') fi - COMMAND_ARR+=('-v' $(echo "$PYTHON_VERSION" | sed s/rc/~rc/g)) + COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g)) COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION") COMMAND_ARR+=('-n' "$PYTHON_PKG") COMMAND_ARR+=('-C' "build") @@ -683,6 +766,10 @@ fpm_build_virtualenv () { COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_depends[@]}"; do + COMMAND_ARR+=('--replaces' "python-$PKG") + done + # make sure the systemd service file ends up in the right place # used by arvados-docker-cleaner if [[ -e "${systemd_unit}" ]]; then @@ -703,9 +790,9 @@ fpm_build_virtualenv () { done fi - # the python-arvados-cwl-runner package comes with cwltool, expose that version - if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then - COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/") + # the python3-arvados-cwl-runner package comes with cwltool, expose that version + if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then + COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/") fi COMMAND_ARR+=(".") @@ -727,6 +814,136 @@ fpm_build_virtualenv () { echo } +# build_metapackage builds meta packages that help with the python to python 3 package migration +build_metapackage() { + # base package name (e.g. arvados-python-client) + BASE_NAME=$1 + shift + PKG_DIR=$1 + shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then + return 0 + fi + + if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then + ARVADOS_BUILDING_ITERATION=1 + fi + + if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + cd $WORKSPACE/$PKG_DIR + pwd + rm -rf dist/* + + # Get the latest setuptools + if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then + echo "Error, unable to upgrade setuptools with XY" + echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'" + exit 1 + fi + # filter a useless warning (when building the cwltest package) from the stderr output + if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then + echo "Error, unable to run python3 setup.py sdist for $PKG" + exit 1 + fi + + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') + + else + UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) + fi + + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi + mkdir $BASE_NAME + cd $BASE_NAME + + if [[ "$FORMAT" == "deb" ]]; then + cat >ns-control < +Depends: python3-${BASE_NAME} +Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME} + This package is a metapackage that will automatically install the new version of + ${BASE_NAME} which is Python 3 based and has a different name. +EOF + + /usr/bin/equivs-build ns-control + if [[ $? -ne 0 ]]; then + echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?" + return 1 + fi + elif [[ "$FORMAT" == "rpm" ]]; then + cat >meta.spec < +- initial release +EOF + + /usr/bin/rpmbuild -ba meta.spec + if [[ $? -ne 0 ]]; then + echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?" + return 1 + else + mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} . + if [[ $? -ne 0 ]]; then + echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'" + return 1 + fi + fi + else + echo "Unknown format" + return 1 + fi + + if [[ $EXITCODE -ne 0 ]]; then + return 1 + else + echo `ls *$FORMAT` + mv *$FORMAT $WORKSPACE/packages/$TARGET/ + fi + + # clean up + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi +} + # Build packages for everything fpm_build () { # Source dir where fpm-info.sh (if any) will be found. @@ -796,6 +1013,7 @@ fpm_build () { declare -a fpm_args=() declare -a build_depends=() declare -a fpm_depends=() + declare -a fpm_conflicts=() declare -a fpm_exclude=() if [[ ! -d "$SRC_DIR" ]]; then echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir" @@ -831,6 +1049,9 @@ fpm_build () { for i in "${fpm_depends[@]}"; do COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_conflicts[@]}"; do + COMMAND_ARR+=('--conflicts' "$i") + done for i in "${fpm_exclude[@]}"; do COMMAND_ARR+=('--exclude' "$i") done