}
nohash_version_from_git() {
+ local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git | cut -d. -f1-4
+ version_from_git $subdir | cut -d. -f1-4
}
timestamp_from_git() {
}
calculate_python_sdk_cwl_package_versions() {
- python_sdk_ts=$(cd sdk/python && timestamp_from_git)
- cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
- python_sdk_version=$(cd sdk/python && nohash_version_from_git)
- cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
-
- if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$python_sdk_version
- fi
-}
-
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
+ python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
+ cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
handle_ruby_gem() {
checkdirs+=("$1")
shift
done
- if grep -qr git.arvados.org/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
+ # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
+ # Everything depends on the build directory.
+ checkdirs+=(sdk/go lib build)
local timestamp=0
for dir in ${checkdirs[@]}; do
cd "$WORKSPACE"
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
+ for arch in 'amd64' 'arm64'; do
+ package_go_binary_worker "$src_path" "$prog" "$description" "$arch" "$license_file"
+ done
+}
+
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [apache-2.0.txt]
+package_go_binary_worker() {
+ local src_path="$1"; shift
+ local prog="$1"; shift
+ local description="$1"; shift
+ local arch="${1:-amd64}"; shift
+ local license_file="${1:-agpl-3.0.txt}"; shift
+
if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
# arvados-workbench depends on arvados-server at build time, so even when
# only arvados-workbench is being built, we need to build arvados-server too
return 1
fi
- go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ echo "BUILDING ${arch}"
+ GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
local -a switches=()
+
+ binpath=$GOPATH/bin/${basename}
+ if [[ "${arch}" != "amd64" ]]; then
+ switches+=("-a${arch}")
+ binpath="$GOPATH/bin/linux_${arch}/${basename}"
+ fi
+
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
fi
local version="$(version_from_git)"
if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then
- calculate_go_package_version version cmd/arvados-server "$srcdir"
+ calculate_go_package_version version cmd/arvados-server "$srcdir"
fi
echo $version
}
echo "Package $full_pkgname build forced with --force-build, building"
elif [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian9]=stretch
dd[debian10]=buster
- dd[ubuntu1604]=xenial
+ dd[debian11]=bullseye
dd[ubuntu1804]=bionic
+ dd[ubuntu2004]=focal
D=${dd[$TARGET]}
if [ ${pkgname:0:3} = "lib" ]; then
repo_subdir=${pkgname:0:4}
repo_subdir=${pkgname:0:1}
fi
- repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/)
+ repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
echo "${repo_pkg_list}" |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
- curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname}
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
echo "Package $full_pkgname exists, not rebuilding!"
fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
- # .git and packages are for the SSO server, which is built from its
- # repository root.
- local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+ local -a exclude_list=(tmp log coverage Capfile\* \
config/deploy\* config/application.yml)
# for arvados-workbench, we need to have the (dummy) config/database.yml in the package
if [[ "$pkgname" != "arvados-workbench" ]]; then
case "$PACKAGE_TYPE" in
python3)
python=python3
- if [[ "$FORMAT" != "rpm" ]]; then
- pip=pip3
- else
- # In CentOS, we use a different mechanism to get the right version of pip
- pip=pip
- fi
+ pip=pip3
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
;;
esac
# Determine the package version from the generated sdist archive
if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
- UNFILTERED_PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION)
- PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~rc/rc/g)
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
else
- UNFILTERED_PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
fi
# See if we actually need to build this package; does it exist already?
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # Make sure to conflict with the old rh-python36 packages we used to publish
+ COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
+ fi
+
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- COMMAND_ARR+=('-v' $(echo "$PYTHON_VERSION" | sed s/rc/~rc/g))
+ COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g))
COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
COMMAND_ARR+=('-n' "$PYTHON_PKG")
COMMAND_ARR+=('-C' "build")
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--replaces' "python-$PKG")
+ done
+
# make sure the systemd service file ends up in the right place
# used by arvados-docker-cleaner
if [[ -e "${systemd_unit}" ]]; then
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
echo
}
+# build_metapackage builds meta packages that help with the python to python 3 package migration
+build_metapackage() {
+ # base package name (e.g. arvados-python-client)
+ BASE_NAME=$1
+ shift
+ PKG_DIR=$1
+ shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ cd $WORKSPACE/$PKG_DIR
+ pwd
+ rm -rf dist/*
+
+ # Get the latest setuptools
+ if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ echo "Error, unable to upgrade setuptools with XY"
+ echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ exit 1
+ fi
+ # filter a useless warning (when building the cwltest package) from the stderr output
+ if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
+ echo "Error, unable to run python3 setup.py sdist for $PKG"
+ exit 1
+ fi
+
+ PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
+
+ else
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+ fi
+
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+ mkdir $BASE_NAME
+ cd $BASE_NAME
+
+ if [[ "$FORMAT" == "deb" ]]; then
+ cat >ns-control <<EOF
+Section: misc
+Priority: optional
+Standards-Version: 3.9.2
+
+Package: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}-${ARVADOS_BUILDING_ITERATION}
+Maintainer: Arvados Package Maintainers <packaging@arvados.org>
+Depends: python3-${BASE_NAME}
+Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME}
+ This package is a metapackage that will automatically install the new version of
+ ${BASE_NAME} which is Python 3 based and has a different name.
+EOF
+
+ /usr/bin/equivs-build ns-control
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?"
+ return 1
+ fi
+ elif [[ "$FORMAT" == "rpm" ]]; then
+ cat >meta.spec <<EOF
+Summary: metapackage to ease the upgrade to the Python 3 version of ${BASE_NAME}
+Name: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}
+Release: ${ARVADOS_BUILDING_ITERATION}
+License: distributable
+
+Requires: python3-${BASE_NAME}
+
+%description
+This package is a metapackage that will automatically install the new version of
+python-${BASE_NAME} which is Python 3 based and has a different name.
+
+%prep
+
+%build
+
+%clean
+
+%install
+
+%post
+
+%files
+
+
+%changelog
+* Mon Apr 12 2021 Arvados Package Maintainers <packaging@arvados.org>
+- initial release
+EOF
+
+ /usr/bin/rpmbuild -ba meta.spec
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?"
+ return 1
+ else
+ mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} .
+ if [[ $? -ne 0 ]]; then
+ echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'"
+ return 1
+ fi
+ fi
+ else
+ echo "Unknown format"
+ return 1
+ fi
+
+ if [[ $EXITCODE -ne 0 ]]; then
+ return 1
+ else
+ echo `ls *$FORMAT`
+ mv *$FORMAT $WORKSPACE/packages/$TARGET/
+ fi
+
+ # clean up
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+}
+
# Build packages for everything
fpm_build () {
# Source dir where fpm-info.sh (if any) will be found.
declare -a fpm_args=()
declare -a build_depends=()
declare -a fpm_depends=()
+ declare -a fpm_conflicts=()
declare -a fpm_exclude=()
if [[ ! -d "$SRC_DIR" ]]; then
echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
for i in "${fpm_depends[@]}"; do
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_conflicts[@]}"; do
+ COMMAND_ARR+=('--conflicts' "$i")
+ done
for i in "${fpm_exclude[@]}"; do
COMMAND_ARR+=('--exclude' "$i")
done
projects / arvados.git / blobdiff