__returnvar="$version"
}
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt]
+# Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
package_go_binary() {
local src_path="$1"; shift
local prog="$1"; shift
+ local package_format="$1"; shift
+ local target_arch="$1"; shift
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
fi
fi
- debug_echo "package_go_binary $src_path as $prog"
+ # Only amd64 and aarch64 are supported
+ native_arch="amd64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ fi
+
+ if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
+ echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
+ return 1
+ fi
+
+ if [[ -n "$target_arch" ]]; then
+ # A target architecture has been specified
+ package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
+ return $?
+ else
+ # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
+ # but only when building deb packages (centos does not have support for crosscompiling userspace).
+ archs=($native_arch)
+ if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+ archs=('amd64' 'arm64')
+ fi
+ for ta in $archs; do
+ package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
+ retval=$?
+ if [[ "$retval" != 0 ]]; then
+ return $retval
+ fi
+ done
+ fi
+}
+
+# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
+package_go_binary_worker() {
+ local src_path="$1"; shift
+ local prog="$1"; shift
+ local package_format="$1"; shift
+ local description="$1"; shift
+ local native_arch="${1:-amd64}"; shift
+ local target_arch="${1:-amd64}"; shift
+ local license_file="${1:-agpl-3.0.txt}"; shift
+
+ if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then
+ echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch"
+ return 1
+ fi
+ debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
local basename="${src_path##*/}"
calculate_go_package_version go_package_version $src_path
cd $WORKSPACE/packages/$TARGET
- test_package_presence $prog $go_package_version go
-
+ test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
if [[ "$?" != "0" ]]; then
return 1
fi
- go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ echo "Building $pachage_format ($target_arch) package for $prog from $src_path"
+ if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
+ CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ else
+ GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ fi
local -a switches=()
+
+ binpath=$GOPATH/bin/${basename}
+ if [[ "${target_arch}" != "${native_arch}" ]]; then
+ switches+=("-a${target_arch}")
+ binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
+ fi
+
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
-# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
+# Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
package_go_so() {
local src_path="$1"; shift
local sofile="$1"; shift
local pkg="$1"; shift
+ local package_format="$1"; shift
+ local target_arch="$1"; shift # supported: amd64, arm64
local description="$1"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
debug_echo "package_go_so $src_path as $pkg"
calculate_go_package_version go_package_version $src_path
fi
if [[ "$arch" == "" ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ native_arch="amd64"
+ rpm_native_arch="x86_64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ rpm_native_arch="arm64"
+ fi
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
if [[ "$pkgtype" =~ ^(src)$ ]]; then
rpm_architecture="noarch"
# These python packages have binary components
if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
fi
else
rpm_architecture=$arch
fi
local full_pkgname
- get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch
+ get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
# See if we can skip building the package, only if it already exists in the
# processed/ directory. If so, move it back to the packages directory to make
elif [[ "$FORMAT" == "deb" ]]; then
declare -A dd
dd[debian10]=buster
- dd[ubuntu1604]=xenial
+ dd[debian11]=bullseye
dd[ubuntu1804]=bionic
dd[ubuntu2004]=focal
D=${dd[$TARGET]}
repo_subdir=${pkgname:0:1}
fi
- repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/)
+ repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
echo "${repo_pkg_list}" |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
- curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname}
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
echo "Package $full_pkgname exists, not rebuilding!"
shift
PACKAGE_TYPE=${1:-python}
shift
+ native_arch="amd64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ fi
+
+ if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
+ fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
+ elif [[ -z "$ARCH" ]]; then
+ for arch in $native_arch; do
+ fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
+ done
+ else
+ echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+ fi
+}
+
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv_worker () {
+ PKG=$1
+ shift
+ PKG_DIR=$1
+ shift
+ PACKAGE_TYPE=${1:-python}
+ shift
+ arch=${1:-amd64}
+ shift
# Set up
STDOUT_IF_DEBUG=/dev/null
# We can't do this earlier than here, because we need PYTHON_VERSION...
# This isn't so bad; the sdist call above is pretty quick compared to
# the invocation of virtualenv and fpm, below.
- if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+ if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
return 0
fi
- echo "Building $FORMAT package for $PKG from $PKG_DIR"
+ echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
# Package the sdist in a virtualenv
echo "Creating virtualenv..."
declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
+ if [[ "${arch}" != "amd64" ]]; then
+ COMMAND_ARR+=("-a${arch}")
+ fi
+
if [[ "$MAINTAINER" != "" ]]; then
COMMAND_ARR+=('--maintainer' "$MAINTAINER")
fi
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # Make sure to conflict with the old rh-python36 packages we used to publish
+ COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
+ fi
+
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--replaces' "python-$PKG")
+ done
+
# make sure the systemd service file ends up in the right place
# used by arvados-docker-cleaner
if [[ -e "${systemd_unit}" ]]; then
fi
# the python3-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
echo
}
+# build_metapackage builds meta packages that help with the python to python 3 package migration
+build_metapackage() {
+ # base package name (e.g. arvados-python-client)
+ BASE_NAME=$1
+ shift
+ PKG_DIR=$1
+ shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ cd $WORKSPACE/$PKG_DIR
+ pwd
+ rm -rf dist/*
+
+ # Get the latest setuptools
+ if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ echo "Error, unable to upgrade setuptools with XY"
+ echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ exit 1
+ fi
+ # filter a useless warning (when building the cwltest package) from the stderr output
+ if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
+ echo "Error, unable to run python3 setup.py sdist for $PKG"
+ exit 1
+ fi
+
+ PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
+
+ else
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+ fi
+
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+ mkdir $BASE_NAME
+ cd $BASE_NAME
+
+ if [[ "$FORMAT" == "deb" ]]; then
+ cat >ns-control <<EOF
+Section: misc
+Priority: optional
+Standards-Version: 3.9.2
+
+Package: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}-${ARVADOS_BUILDING_ITERATION}
+Maintainer: Arvados Package Maintainers <packaging@arvados.org>
+Depends: python3-${BASE_NAME}
+Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME}
+ This package is a metapackage that will automatically install the new version of
+ ${BASE_NAME} which is Python 3 based and has a different name.
+EOF
+
+ /usr/bin/equivs-build ns-control
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?"
+ return 1
+ fi
+ elif [[ "$FORMAT" == "rpm" ]]; then
+ cat >meta.spec <<EOF
+Summary: metapackage to ease the upgrade to the Python 3 version of ${BASE_NAME}
+Name: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}
+Release: ${ARVADOS_BUILDING_ITERATION}
+License: distributable
+
+Requires: python3-${BASE_NAME}
+
+%description
+This package is a metapackage that will automatically install the new version of
+python-${BASE_NAME} which is Python 3 based and has a different name.
+
+%prep
+
+%build
+
+%clean
+
+%install
+
+%post
+
+%files
+
+
+%changelog
+* Mon Apr 12 2021 Arvados Package Maintainers <packaging@arvados.org>
+- initial release
+EOF
+
+ /usr/bin/rpmbuild -ba meta.spec
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?"
+ return 1
+ else
+ mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} .
+ if [[ $? -ne 0 ]]; then
+ echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'"
+ return 1
+ fi
+ fi
+ else
+ echo "Unknown format"
+ return 1
+ fi
+
+ if [[ $EXITCODE -ne 0 ]]; then
+ return 1
+ else
+ echo `ls *$FORMAT`
+ mv *$FORMAT $WORKSPACE/packages/$TARGET/
+ fi
+
+ # clean up
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+}
+
# Build packages for everything
fpm_build () {
# Source dir where fpm-info.sh (if any) will be found.
declare -a fpm_args=()
declare -a build_depends=()
declare -a fpm_depends=()
+ declare -a fpm_conflicts=()
declare -a fpm_exclude=()
if [[ ! -d "$SRC_DIR" ]]; then
echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
for i in "${fpm_depends[@]}"; do
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_conflicts[@]}"; do
+ COMMAND_ARR+=('--conflicts' "$i")
+ done
for i in "${fpm_exclude[@]}"; do
COMMAND_ARR+=('--exclude' "$i")
done
projects / arvados.git / blobdiff