}
nohash_version_from_git() {
+ local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git | cut -d. -f1-4
+ version_from_git $subdir | cut -d. -f1-4
}
timestamp_from_git() {
}
calculate_python_sdk_cwl_package_versions() {
- python_sdk_ts=$(cd sdk/python && timestamp_from_git)
- cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
- python_sdk_version=$(cd sdk/python && nohash_version_from_git)
- cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
-
- if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$python_sdk_version
- fi
-}
-
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
+ python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
+ cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
handle_ruby_gem() {
checkdirs+=("$1")
shift
done
- if grep -qr git.arvados.org/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
+ # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
+ # Everything depends on the build directory.
+ checkdirs+=(sdk/go lib build)
local timestamp=0
for dir in ${checkdirs[@]}; do
cd "$WORKSPACE"
"$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
)
if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
- fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/pam-configs/arvados-go")
+ fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go")
fi
if [[ -e "$WORKSPACE/$src_path/README" ]]; then
fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
fi
local version="$(version_from_git)"
if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then
- calculate_go_package_version version cmd/arvados-server "$srcdir"
+ calculate_go_package_version version cmd/arvados-server "$srcdir"
fi
echo $version
}
echo "Package $full_pkgname build forced with --force-build, building"
elif [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian9]=stretch
dd[debian10]=buster
- dd[ubuntu1604]=xenial
+ dd[debian11]=bullseye
dd[ubuntu1804]=bionic
+ dd[ubuntu2004]=focal
D=${dd[$TARGET]}
if [ ${pkgname:0:3} = "lib" ]; then
repo_subdir=${pkgname:0:4}
repo_subdir=${pkgname:0:1}
fi
- repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/)
+ repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
echo "${repo_pkg_list}" |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
- curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname}
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
echo "Package $full_pkgname exists, not rebuilding!"
fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
- # .git and packages are for the SSO server, which is built from its
- # repository root.
- local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+ local -a exclude_list=(tmp log coverage Capfile\* \
config/deploy\* config/application.yml)
# for arvados-workbench, we need to have the (dummy) config/database.yml in the package
if [[ "$pkgname" != "arvados-workbench" ]]; then
case "$PACKAGE_TYPE" in
python3)
python=python3
- if [[ "$FORMAT" != "rpm" ]]; then
- pip=pip3
- else
- # In CentOS, we use a different mechanism to get the right version of pip
- pip=pip
- fi
+ pip=pip3
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
;;
esac
fi
# Determine the package version from the generated sdist archive
- PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)}
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+ else
+ PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
+ fi
# See if we actually need to build this package; does it exist already?
# We can't do this earlier than here, because we need PYTHON_VERSION...
# This isn't so bad; the sdist call above is pretty quick compared to
# the invocation of virtualenv and fpm, below.
- if ! test_package_presence "$PYTHON_PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+ if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
return 0
fi
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # Make sure to conflict with the old rh-python36 packages we used to publish
+ COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
+ fi
+
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- COMMAND_ARR+=('-v' "$PYTHON_VERSION")
+ COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g))
COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
COMMAND_ARR+=('-n' "$PYTHON_PKG")
COMMAND_ARR+=('-C' "build")
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--replaces' "python-$PKG")
+ done
+
# make sure the systemd service file ends up in the right place
# used by arvados-docker-cleaner
if [[ -e "${systemd_unit}" ]]; then
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
+ debug_echo -e "\n${COMMAND_ARR[@]}\n"
+
FPM_RESULTS=$("${COMMAND_ARR[@]}")
FPM_EXIT_CODE=$?
echo
}
+# build_metapackage builds meta packages that help with the python to python 3 package migration
+build_metapackage() {
+ # base package name (e.g. arvados-python-client)
+ BASE_NAME=$1
+ shift
+ PKG_DIR=$1
+ shift
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then
+ return 0
+ fi
+
+ if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+ ARVADOS_BUILDING_ITERATION=1
+ fi
+
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ cd $WORKSPACE/$PKG_DIR
+ pwd
+ rm -rf dist/*
+
+ # Get the latest setuptools
+ if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+ echo "Error, unable to upgrade setuptools with XY"
+ echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+ exit 1
+ fi
+ # filter a useless warning (when building the cwltest package) from the stderr output
+ if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
+ echo "Error, unable to run python3 setup.py sdist for $PKG"
+ exit 1
+ fi
+
+ PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
+
+ else
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+ fi
+
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+ mkdir $BASE_NAME
+ cd $BASE_NAME
+
+ if [[ "$FORMAT" == "deb" ]]; then
+ cat >ns-control <<EOF
+Section: misc
+Priority: optional
+Standards-Version: 3.9.2
+
+Package: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}-${ARVADOS_BUILDING_ITERATION}
+Maintainer: Arvados Package Maintainers <packaging@arvados.org>
+Depends: python3-${BASE_NAME}
+Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME}
+ This package is a metapackage that will automatically install the new version of
+ ${BASE_NAME} which is Python 3 based and has a different name.
+EOF
+
+ /usr/bin/equivs-build ns-control
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?"
+ return 1
+ fi
+ elif [[ "$FORMAT" == "rpm" ]]; then
+ cat >meta.spec <<EOF
+Summary: metapackage to ease the upgrade to the Python 3 version of ${BASE_NAME}
+Name: python-${BASE_NAME}
+Version: ${PYTHON_VERSION}
+Release: ${ARVADOS_BUILDING_ITERATION}
+License: distributable
+
+Requires: python3-${BASE_NAME}
+
+%description
+This package is a metapackage that will automatically install the new version of
+python-${BASE_NAME} which is Python 3 based and has a different name.
+
+%prep
+
+%build
+
+%clean
+
+%install
+
+%post
+
+%files
+
+
+%changelog
+* Mon Apr 12 2021 Arvados Package Maintainers <packaging@arvados.org>
+- initial release
+EOF
+
+ /usr/bin/rpmbuild -ba meta.spec
+ if [[ $? -ne 0 ]]; then
+ echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?"
+ return 1
+ else
+ mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} .
+ if [[ $? -ne 0 ]]; then
+ echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'"
+ return 1
+ fi
+ fi
+ else
+ echo "Unknown format"
+ return 1
+ fi
+
+ if [[ $EXITCODE -ne 0 ]]; then
+ return 1
+ else
+ echo `ls *$FORMAT`
+ mv *$FORMAT $WORKSPACE/packages/$TARGET/
+ fi
+
+ # clean up
+ cd - >$STDOUT_IF_DEBUG
+ if [[ -d "$BASE_NAME" ]]; then
+ rm -rf $BASE_NAME
+ fi
+}
+
# Build packages for everything
fpm_build () {
# Source dir where fpm-info.sh (if any) will be found.
declare -a fpm_args=()
declare -a build_depends=()
declare -a fpm_depends=()
+ declare -a fpm_conflicts=()
declare -a fpm_exclude=()
if [[ ! -d "$SRC_DIR" ]]; then
echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
for i in "${fpm_depends[@]}"; do
COMMAND_ARR+=('--depends' "$i")
done
+ for i in "${fpm_conflicts[@]}"; do
+ COMMAND_ARR+=('--conflicts' "$i")
+ done
for i in "${fpm_exclude[@]}"; do
COMMAND_ARR+=('--exclude' "$i")
done
+ COMMAND_ARR+=("${fpm_args[@]}")
+
# Append remaining function arguments directly to fpm's command line.
for i; do
COMMAND_ARR+=("$i")
done
- COMMAND_ARR+=("${fpm_args[@]}")
-
COMMAND_ARR+=("$PACKAGE")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
projects / arvados.git / blobdiff