From 758d39fd7aee4d1989395eba2113f0f2538b30d1 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Tue, 31 May 2016 11:13:41 -0400 Subject: [PATCH] 9242: Python packages install libraries to the distro path. This avoids breaking dependent packages that expect to find files in the same place. --- backports/python-ciso8601/fpm-info.sh | 2 +- backports/python-llfuse/fpm-info.sh | 2 +- backports/python-pycrypto/fpm-info.sh | 2 +- backports/python-pycurl/fpm-info.sh | 2 +- build/run-build-packages.sh | 24 +++++++++++++++--------- build/run-library.sh | 27 ++++++++++++++++++--------- 6 files changed, 37 insertions(+), 22 deletions(-) diff --git a/backports/python-ciso8601/fpm-info.sh b/backports/python-ciso8601/fpm-info.sh index 42f31e365c..c9665be096 100644 --- a/backports/python-ciso8601/fpm-info.sh +++ b/backports/python-ciso8601/fpm-info.sh @@ -8,4 +8,4 @@ case "$TARGET" in esac # FIXME: Remove this line after #6885 is done. -fpm_args+=(--iteration 2) +fpm_args+=(--iteration 3) diff --git a/backports/python-llfuse/fpm-info.sh b/backports/python-llfuse/fpm-info.sh index a7d9398701..9fc00987e6 100644 --- a/backports/python-llfuse/fpm-info.sh +++ b/backports/python-llfuse/fpm-info.sh @@ -10,4 +10,4 @@ case "$TARGET" in esac # FIXME: Remove this line after #6885 is done. -fpm_args+=(--iteration 2) +fpm_args+=(--iteration 3) diff --git a/backports/python-pycrypto/fpm-info.sh b/backports/python-pycrypto/fpm-info.sh index 52df46573d..65a83a06c7 100644 --- a/backports/python-pycrypto/fpm-info.sh +++ b/backports/python-pycrypto/fpm-info.sh @@ -11,4 +11,4 @@ case "$TARGET" in esac # FIXME: Remove this line after #6885 is done. -fpm_args+=(--iteration 2) +fpm_args+=(--iteration 3) diff --git a/backports/python-pycurl/fpm-info.sh b/backports/python-pycurl/fpm-info.sh index e4fbf59344..5bfb010f0e 100644 --- a/backports/python-pycurl/fpm-info.sh +++ b/backports/python-pycurl/fpm-info.sh @@ -1,5 +1,5 @@ # FIXME: Remove this line after #6885 is done. -fpm_args+=(--iteration 2) +fpm_args+=(--iteration 3) case "$TARGET" in centos6) diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index cf12b6552a..a51198f9ef 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -86,9 +86,13 @@ PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=s # You can customize them as needed in distro sections below. PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python +PYTHON2_PREFIX=/usr +PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/dist-packages PYTHON3_PACKAGE=python$PYTHON3_VERSION PYTHON3_PKG_PREFIX=python3 +PYTHON3_PREFIX=/usr +PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages ## End Debian Python defaults. case "$TARGET" in @@ -134,8 +138,12 @@ case "$TARGET" in FORMAT=rpm PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE + PYTHON2_PREFIX=/opt/rh/python27/root/usr + PYTHON2_INSTALL_LIB=lib64/python$PYTHON2_VERSION PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE + PYTHON3_PREFIX=/opt/rh/python33/root/usr + PYTHON3_INSTALL_LIB=lib64/python$PYTHON3_VERSION PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ @@ -405,7 +413,7 @@ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'C # cwl-runner cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/sdk/cwl/build" -fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 2 +fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3 # schema_salad. This is a python dependency of arvados-cwl-runner, # but we can't use the usual PYTHONPACKAGES way to build this package due to the @@ -422,17 +430,17 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover # So we build this thing separately. # # Ward, 2016-03-17 -fpm --maintainer='Ward Vandewege ' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-schema-salad" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.11.20160506154702 schema_salad +fpm_build schema_salad schema_salad "" python 1.7.20160316203940 -# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh. +# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh. # Ward, 2016-05-26 -fpm --maintainer='Ward Vandewege ' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" --python-setup-py-arguments "--single-version-externally-managed" ruamel.yaml +fpm_build ruamel.yaml ruamel.yaml "" python "" --python-setup-py-arguments "--single-version-externally-managed" # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17 -fpm --maintainer='Ward Vandewege ' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-cwltool" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.0.20160519182434 cwltool +fpm_build cwltool cwltool "" python 1.0.20160427142240 # FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25 -fpm --maintainer='Ward Vandewege ' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --verbose --log info -n "${PYTHON2_PKG_PREFIX}-rdflib-jsonld" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 0.3.0 rdflib-jsonld +fpm_build rdflib-jsonld rdflib-jsonld "" python 0.3.0 # The PAM module if [[ $TARGET =~ debian|ubuntu ]]; then @@ -506,9 +514,7 @@ for deppkg in "${PYTHON_BACKPORTS[@]}"; do "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build chmod -R go+rX . set +e - # --iteration 2 provides an upgrade for previously built - # buggy packages. - fpm_build . "$outname" "" python "" --iteration 2 + fpm_build . "$outname" "" python "" --iteration 3 # The upload step uses the package timestamp to determine # whether it's new. --no-clobber plays nice with that. mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET" diff --git a/build/run-library.sh b/build/run-library.sh index 8d97adab81..c7bc9538d1 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -192,6 +192,8 @@ fpm_build () { VERSION=$1 shift + local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")" + case "$PACKAGE_TYPE" in python) # All Arvados Python2 packages depend on Python 2.7. @@ -199,7 +201,12 @@ fpm_build () { set -- "$@" --python-bin python2.7 \ --python-easyinstall "$EASY_INSTALL2" \ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \ + --prefix "$PYTHON2_PREFIX" \ + --python-install-lib "$PYTHON2_INSTALL_LIB" \ + --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \ --depends "$PYTHON2_PACKAGE" + # Fix --iteration for #9242. + default_iteration_value=$(($default_iteration_value + 1)) ;; python3) # fpm does not actually support a python3 package type. Instead @@ -210,19 +217,21 @@ fpm_build () { set -- "$@" --python-bin python3 \ --python-easyinstall "$EASY_INSTALL3" \ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \ + --prefix "$PYTHON3_PREFIX" \ + --python-install-lib "$PYTHON3_INSTALL_LIB" \ + --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \ --depends "$PYTHON3_PACKAGE" + # Fix --iteration for #9242. + default_iteration_value=$(($default_iteration_value + 1)) ;; esac declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT") - if [ python = "$PACKAGE_TYPE" ]; then - COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*) - if [ deb = "$FORMAT" ]; then - # Dependencies are built from setup.py. Since setup.py will never - # refer to Debian package iterations, it doesn't make sense to - # enforce those in the .deb dependencies. - COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies) - fi + if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then + # Dependencies are built from setup.py. Since setup.py will never + # refer to Debian package iterations, it doesn't make sense to + # enforce those in the .deb dependencies. + COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies) fi if [[ "${DEBUG:-0}" != "0" ]]; then @@ -242,7 +251,7 @@ fpm_build () { fi # We can always add an --iteration here. If another one is specified in $@, # that will take precedence, as desired. - COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")") + COMMAND_ARR+=(--iteration "$default_iteration_value") # Append --depends X and other arguments specified by fpm-info.sh in # the package source dir. These are added last so they can override -- 2.30.2