X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c6c2f3518bc745eed95b5f5b81db5d17db4366ff..54781f3d3c1dea0e14542d129b1c8e061ad406fc:/build/run-library.sh diff --git a/build/run-library.sh b/build/run-library.sh index 079a34313f..d0ddcd6c67 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -61,11 +61,12 @@ version_from_git() { } nohash_version_from_git() { + local subdir="$1"; shift if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then echo "$ARVADOS_BUILDING_VERSION" return fi - version_from_git | cut -d. -f1-4 + version_from_git $subdir | cut -d. -f1-4 } timestamp_from_git() { @@ -74,25 +75,8 @@ timestamp_from_git() { } calculate_python_sdk_cwl_package_versions() { - python_sdk_ts=$(cd sdk/python && timestamp_from_git) - cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git) - - python_sdk_version=$(cd sdk/python && nohash_version_from_git) - cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git) - - if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then - cwl_runner_version=$python_sdk_version - fi -} - -handle_python_package () { - # This function assumes the current working directory is the python package directory - if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then - # This package doesn't need rebuilding. - return - fi - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - python setup.py $DASHQ_UNLESS_DEBUG sdist + python_sdk_version=$(cd sdk/python && python3 arvados_version.py) + cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py) } handle_ruby_gem() { @@ -130,9 +114,9 @@ calculate_go_package_version() { checkdirs+=("$1") shift done - if grep -qr git.arvados.org/arvados .; then - checkdirs+=(sdk/go lib) - fi + # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server) + # Everything depends on the build directory. + checkdirs+=(sdk/go lib build) local timestamp=0 for dir in ${checkdirs[@]}; do cd "$WORKSPACE" @@ -253,7 +237,7 @@ rails_package_version() { fi local version="$(version_from_git)" if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then - calculate_go_package_version version cmd/arvados-server "$srcdir" + calculate_go_package_version version cmd/arvados-server "$srcdir" fi echo $version } @@ -352,10 +336,10 @@ test_package_presence() { echo "Package $full_pkgname build forced with --force-build, building" elif [[ "$FORMAT" == "deb" ]]; then declare -A dd - dd[debian9]=stretch dd[debian10]=buster - dd[ubuntu1604]=xenial + dd[debian11]=bullseye dd[ubuntu1804]=bionic + dd[ubuntu2004]=focal D=${dd[$TARGET]} if [ ${pkgname:0:3} = "lib" ]; then repo_subdir=${pkgname:0:4} @@ -363,11 +347,11 @@ test_package_presence() { repo_subdir=${pkgname:0:1} fi - repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/) + repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/) echo "${repo_pkg_list}" |grep -q ${full_pkgname} if [ $? -eq 0 ] ; then echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!" - curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname} + curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname} return 1 elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then echo "Package $full_pkgname exists, not rebuilding!" @@ -432,9 +416,7 @@ handle_rails_package() { fi # For some reason fpm excludes need to not start with /. local exclude_root="${railsdir#/}" - # .git and packages are for the SSO server, which is built from its - # repository root. - local -a exclude_list=(.git packages tmp log coverage Capfile\* \ + local -a exclude_list=(tmp log coverage Capfile\* \ config/deploy\* config/application.yml) # for arvados-workbench, we need to have the (dummy) config/database.yml in the package if [[ "$pkgname" != "arvados-workbench" ]]; then @@ -475,12 +457,7 @@ fpm_build_virtualenv () { case "$PACKAGE_TYPE" in python3) python=python3 - if [[ "$FORMAT" != "rpm" ]]; then - pip=pip3 - else - # In CentOS, we use a different mechanism to get the right version of pip - pip=pip - fi + pip=pip3 PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX ;; esac @@ -529,8 +506,8 @@ fpm_build_virtualenv () { UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) else - UNFILTERED_PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) - PYTHON_VERSION=$UNFILTERED_PYTHON_VERSION + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') fi # See if we actually need to build this package; does it exist already? @@ -644,6 +621,11 @@ fpm_build_virtualenv () { LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"` COMMAND_ARR+=('--license' "$LICENSE_STRING") + if [[ "$FORMAT" == "rpm" ]]; then + # Make sure to conflict with the old rh-python36 packages we used to publish + COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG") + fi + if [[ "$DEBUG" != "0" ]]; then COMMAND_ARR+=('--verbose' '--log' 'info') fi @@ -683,6 +665,10 @@ fpm_build_virtualenv () { COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_depends[@]}"; do + COMMAND_ARR+=('--replaces' "python-$PKG") + done + # make sure the systemd service file ends up in the right place # used by arvados-docker-cleaner if [[ -e "${systemd_unit}" ]]; then @@ -703,9 +689,9 @@ fpm_build_virtualenv () { done fi - # the python-arvados-cwl-runner package comes with cwltool, expose that version - if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then - COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/") + # the python3-arvados-cwl-runner package comes with cwltool, expose that version + if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then + COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/") fi COMMAND_ARR+=(".") @@ -727,6 +713,136 @@ fpm_build_virtualenv () { echo } +# build_metapackage builds meta packages that help with the python to python 3 package migration +build_metapackage() { + # base package name (e.g. arvados-python-client) + BASE_NAME=$1 + shift + PKG_DIR=$1 + shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then + return 0 + fi + + if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then + ARVADOS_BUILDING_ITERATION=1 + fi + + if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + cd $WORKSPACE/$PKG_DIR + pwd + rm -rf dist/* + + # Get the latest setuptools + if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then + echo "Error, unable to upgrade setuptools with XY" + echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'" + exit 1 + fi + # filter a useless warning (when building the cwltest package) from the stderr output + if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then + echo "Error, unable to run python3 setup.py sdist for $PKG" + exit 1 + fi + + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') + + else + UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) + fi + + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi + mkdir $BASE_NAME + cd $BASE_NAME + + if [[ "$FORMAT" == "deb" ]]; then + cat >ns-control < +Depends: python3-${BASE_NAME} +Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME} + This package is a metapackage that will automatically install the new version of + ${BASE_NAME} which is Python 3 based and has a different name. +EOF + + /usr/bin/equivs-build ns-control + if [[ $? -ne 0 ]]; then + echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?" + return 1 + fi + elif [[ "$FORMAT" == "rpm" ]]; then + cat >meta.spec < +- initial release +EOF + + /usr/bin/rpmbuild -ba meta.spec + if [[ $? -ne 0 ]]; then + echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?" + return 1 + else + mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} . + if [[ $? -ne 0 ]]; then + echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'" + return 1 + fi + fi + else + echo "Unknown format" + return 1 + fi + + if [[ $EXITCODE -ne 0 ]]; then + return 1 + else + echo `ls *$FORMAT` + mv *$FORMAT $WORKSPACE/packages/$TARGET/ + fi + + # clean up + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi +} + # Build packages for everything fpm_build () { # Source dir where fpm-info.sh (if any) will be found. @@ -796,6 +912,7 @@ fpm_build () { declare -a fpm_args=() declare -a build_depends=() declare -a fpm_depends=() + declare -a fpm_conflicts=() declare -a fpm_exclude=() if [[ ! -d "$SRC_DIR" ]]; then echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir" @@ -831,6 +948,9 @@ fpm_build () { for i in "${fpm_depends[@]}"; do COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_conflicts[@]}"; do + COMMAND_ARR+=('--conflicts' "$i") + done for i in "${fpm_exclude[@]}"; do COMMAND_ARR+=('--exclude' "$i") done