X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3f18ac306f64ef81a9173c6ae452a428ac816309..f30e59af90558e47a4156566d0209bc8efacd85a:/build/run-library.sh diff --git a/build/run-library.sh b/build/run-library.sh index df551455c1..4a3e756ad9 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -61,11 +61,12 @@ version_from_git() { } nohash_version_from_git() { + local subdir="$1"; shift if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then echo "$ARVADOS_BUILDING_VERSION" return fi - version_from_git | cut -d. -f1-4 + version_from_git $subdir | cut -d. -f1-4 } timestamp_from_git() { @@ -74,25 +75,8 @@ timestamp_from_git() { } calculate_python_sdk_cwl_package_versions() { - python_sdk_ts=$(cd sdk/python && timestamp_from_git) - cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git) - - python_sdk_version=$(cd sdk/python && nohash_version_from_git) - cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git) - - if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then - cwl_runner_version=$python_sdk_version - fi -} - -handle_python_package () { - # This function assumes the current working directory is the python package directory - if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then - # This package doesn't need rebuilding. - return - fi - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - python setup.py $DASHQ_UNLESS_DEBUG sdist + python_sdk_version=$(cd sdk/python && python3 arvados_version.py) + cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py) } handle_ruby_gem() { @@ -130,9 +114,9 @@ calculate_go_package_version() { checkdirs+=("$1") shift done - if grep -qr git.arvados.org/arvados .; then - checkdirs+=(sdk/go lib) - fi + # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server) + # Everything depends on the build directory. + checkdirs+=(sdk/go lib build) local timestamp=0 for dir in ${checkdirs[@]}; do cd "$WORKSPACE" @@ -173,7 +157,7 @@ package_go_binary() { return 1 fi - go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" + go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" local -a switches=() systemd_unit="$WORKSPACE/${src_path}/${prog}.service" @@ -210,7 +194,7 @@ package_go_so() { "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt" ) if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then - fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/pam-configs/arvados-go") + fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go") fi if [[ -e "$WORKSPACE/$src_path/README" ]]; then fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README") @@ -253,7 +237,7 @@ rails_package_version() { fi local version="$(version_from_git)" if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then - calculate_go_package_version version cmd/arvados-server "$srcdir" + calculate_go_package_version version cmd/arvados-server "$srcdir" fi echo $version } @@ -352,10 +336,10 @@ test_package_presence() { echo "Package $full_pkgname build forced with --force-build, building" elif [[ "$FORMAT" == "deb" ]]; then declare -A dd - dd[debian9]=stretch dd[debian10]=buster - dd[ubuntu1604]=xenial + dd[debian11]=bullseye dd[ubuntu1804]=bionic + dd[ubuntu2004]=focal D=${dd[$TARGET]} if [ ${pkgname:0:3} = "lib" ]; then repo_subdir=${pkgname:0:4} @@ -363,11 +347,11 @@ test_package_presence() { repo_subdir=${pkgname:0:1} fi - repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/) + repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/) echo "${repo_pkg_list}" |grep -q ${full_pkgname} if [ $? -eq 0 ] ; then echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!" - curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname} + curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname} return 1 elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then echo "Package $full_pkgname exists, not rebuilding!" @@ -432,9 +416,7 @@ handle_rails_package() { fi # For some reason fpm excludes need to not start with /. local exclude_root="${railsdir#/}" - # .git and packages are for the SSO server, which is built from its - # repository root. - local -a exclude_list=(.git packages tmp log coverage Capfile\* \ + local -a exclude_list=(tmp log coverage Capfile\* \ config/deploy\* config/application.yml) # for arvados-workbench, we need to have the (dummy) config/database.yml in the package if [[ "$pkgname" != "arvados-workbench" ]]; then @@ -475,12 +457,7 @@ fpm_build_virtualenv () { case "$PACKAGE_TYPE" in python3) python=python3 - if [[ "$FORMAT" != "rpm" ]]; then - pip=pip3 - else - # In CentOS, we use a different mechanism to get the right version of pip - pip=pip - fi + pip=pip3 PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX ;; esac @@ -525,13 +502,19 @@ fpm_build_virtualenv () { fi # Determine the package version from the generated sdist archive - PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)} + if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then + UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) + else + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') + fi # See if we actually need to build this package; does it exist already? # We can't do this earlier than here, because we need PYTHON_VERSION... # This isn't so bad; the sdist call above is pretty quick compared to # the invocation of virtualenv and fpm, below. - if ! test_package_presence "$PYTHON_PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then + if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then return 0 fi @@ -638,11 +621,16 @@ fpm_build_virtualenv () { LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"` COMMAND_ARR+=('--license' "$LICENSE_STRING") + if [[ "$FORMAT" == "rpm" ]]; then + # Make sure to conflict with the old rh-python36 packages we used to publish + COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG") + fi + if [[ "$DEBUG" != "0" ]]; then COMMAND_ARR+=('--verbose' '--log' 'info') fi - COMMAND_ARR+=('-v' "$PYTHON_VERSION") + COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g)) COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION") COMMAND_ARR+=('-n' "$PYTHON_PKG") COMMAND_ARR+=('-C' "build") @@ -677,6 +665,10 @@ fpm_build_virtualenv () { COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_depends[@]}"; do + COMMAND_ARR+=('--replaces' "python-$PKG") + done + # make sure the systemd service file ends up in the right place # used by arvados-docker-cleaner if [[ -e "${systemd_unit}" ]]; then @@ -697,9 +689,9 @@ fpm_build_virtualenv () { done fi - # the python-arvados-cwl-runner package comes with cwltool, expose that version - if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then - COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/") + # the python3-arvados-cwl-runner package comes with cwltool, expose that version + if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then + COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/") fi COMMAND_ARR+=(".") @@ -721,6 +713,136 @@ fpm_build_virtualenv () { echo } +# build_metapackage builds meta packages that help with the python to python 3 package migration +build_metapackage() { + # base package name (e.g. arvados-python-client) + BASE_NAME=$1 + shift + PKG_DIR=$1 + shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "python-$BASE_NAME" != "$ONLY_BUILD" ]]; then + return 0 + fi + + if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then + ARVADOS_BUILDING_ITERATION=1 + fi + + if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + cd $WORKSPACE/$PKG_DIR + pwd + rm -rf dist/* + + # Get the latest setuptools + if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then + echo "Error, unable to upgrade setuptools with XY" + echo " pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'" + exit 1 + fi + # filter a useless warning (when building the cwltest package) from the stderr output + if ! python3 setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then + echo "Error, unable to run python3 setup.py sdist for $PKG" + exit 1 + fi + + PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO) + UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g') + + else + UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION + PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g) + fi + + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi + mkdir $BASE_NAME + cd $BASE_NAME + + if [[ "$FORMAT" == "deb" ]]; then + cat >ns-control < +Depends: python3-${BASE_NAME} +Description: metapackage to ease the upgrade to the Pyhon 3 version of ${BASE_NAME} + This package is a metapackage that will automatically install the new version of + ${BASE_NAME} which is Python 3 based and has a different name. +EOF + + /usr/bin/equivs-build ns-control + if [[ $? -ne 0 ]]; then + echo "Error running 'equivs-build ns-control', is the 'equivs' package installed?" + return 1 + fi + elif [[ "$FORMAT" == "rpm" ]]; then + cat >meta.spec < +- initial release +EOF + + /usr/bin/rpmbuild -ba meta.spec + if [[ $? -ne 0 ]]; then + echo "Error running 'rpmbuild -ba meta.spec', is the 'rpm-build' package installed?" + return 1 + else + mv /root/rpmbuild/RPMS/x86_64/python-${BASE_NAME}*.${FORMAT} . + if [[ $? -ne 0 ]]; then + echo "Error finding rpm file output of 'rpmbuild -ba meta.spec'" + return 1 + fi + fi + else + echo "Unknown format" + return 1 + fi + + if [[ $EXITCODE -ne 0 ]]; then + return 1 + else + echo `ls *$FORMAT` + mv *$FORMAT $WORKSPACE/packages/$TARGET/ + fi + + # clean up + cd - >$STDOUT_IF_DEBUG + if [[ -d "$BASE_NAME" ]]; then + rm -rf $BASE_NAME + fi +} + # Build packages for everything fpm_build () { # Source dir where fpm-info.sh (if any) will be found. @@ -790,6 +912,7 @@ fpm_build () { declare -a fpm_args=() declare -a build_depends=() declare -a fpm_depends=() + declare -a fpm_conflicts=() declare -a fpm_exclude=() if [[ ! -d "$SRC_DIR" ]]; then echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir" @@ -825,6 +948,9 @@ fpm_build () { for i in "${fpm_depends[@]}"; do COMMAND_ARR+=('--depends' "$i") done + for i in "${fpm_conflicts[@]}"; do + COMMAND_ARR+=('--conflicts' "$i") + done for i in "${fpm_exclude[@]}"; do COMMAND_ARR+=('--exclude' "$i") done