9242: Python packages install libraries to the distro path.

This avoids breaking dependent packages that expect to find files in
the same place.

diff --git a/backports/python-ciso8601/fpm-info.sh b/backports/python-ciso8601/fpm-info.sh
index 42f31e365ca58f3add83cbba0bd73855d1190e30..c9665be0967d205d16e28f453df94bb22cb4f223 100644 (file)
--- a/backports/python-ciso8601/fpm-info.sh
+++ b/backports/python-ciso8601/fpm-info.sh
@@ -8,4 +8,4 @@ case "$TARGET" in
 esac
 
 # FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)

diff --git a/backports/python-llfuse/fpm-info.sh b/backports/python-llfuse/fpm-info.sh
index a7d9398701b7bc4c405027c26f5133fe7b23d383..9fc00987e6f9561f37528368264adf15718b3afd 100644 (file)
--- a/backports/python-llfuse/fpm-info.sh
+++ b/backports/python-llfuse/fpm-info.sh
@@ -10,4 +10,4 @@ case "$TARGET" in
 esac
 
 # FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)

diff --git a/backports/python-pycrypto/fpm-info.sh b/backports/python-pycrypto/fpm-info.sh
index 52df46573da5fae9d1a87d14589f598bde198f0e..65a83a06c7496c5da09315807965272b8d0ff460 100644 (file)
--- a/backports/python-pycrypto/fpm-info.sh
+++ b/backports/python-pycrypto/fpm-info.sh
@@ -11,4 +11,4 @@ case "$TARGET" in
 esac
 
 # FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)

diff --git a/backports/python-pycurl/fpm-info.sh b/backports/python-pycurl/fpm-info.sh
index e4fbf59344ec041875f016b8e7bf227420ce4fef..5bfb010f0e56a3391146482fa5b9d5b51658ee02 100644 (file)
--- a/backports/python-pycurl/fpm-info.sh
+++ b/backports/python-pycurl/fpm-info.sh
@@ -1,5 +1,5 @@
 # FIXME: Remove this line after #6885 is done.
-fpm_args+=(--iteration 2)
+fpm_args+=(--iteration 3)
 
 case "$TARGET" in
     centos6)

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index cf12b6552a00188747770fdb6d7852d98d7b6d8e..a51198f9ef5c657fbcc8c199097080c1bd875004 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -86,9 +86,13 @@ PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=s
 # You can customize them as needed in distro sections below.
 PYTHON2_PACKAGE=python$PYTHON2_VERSION
 PYTHON2_PKG_PREFIX=python
+PYTHON2_PREFIX=/usr
+PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/dist-packages
 
 PYTHON3_PACKAGE=python$PYTHON3_VERSION
 PYTHON3_PKG_PREFIX=python3
+PYTHON3_PREFIX=/usr
+PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 ## End Debian Python defaults.
 
 case "$TARGET" in
@@ -134,8 +138,12 @@ case "$TARGET" in
         FORMAT=rpm
         PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
         PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+        PYTHON2_PREFIX=/opt/rh/python27/root/usr
+        PYTHON2_INSTALL_LIB=lib64/python$PYTHON2_VERSION
         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+        PYTHON3_PREFIX=/opt/rh/python33/root/usr
+        PYTHON3_INSTALL_LIB=lib64/python$PYTHON3_VERSION
         PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
             oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
@@ -405,7 +413,7 @@ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'C
 # cwl-runner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 2
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -422,17 +430,17 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover
 # So we build this thing separately.
 #
 # Ward, 2016-03-17
-fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-schema-salad" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.11.20160506154702 schema_salad
+fpm_build schema_salad schema_salad "" python 1.7.20160316203940
 
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh. 
+# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
 # Ward, 2016-05-26
-fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" --python-setup-py-arguments "--single-version-externally-managed" ruamel.yaml
+fpm_build ruamel.yaml ruamel.yaml "" python "" --python-setup-py-arguments "--single-version-externally-managed"
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n "${PYTHON2_PKG_PREFIX}-cwltool" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 1.0.20160519182434 cwltool
+fpm_build cwltool cwltool "" python 1.0.20160427142240
 
 # FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t $FORMAT --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies --verbose --log info -n "${PYTHON2_PKG_PREFIX}-rdflib-jsonld" --iteration 1 --python-bin python2.7 --python-easyinstall "$EASY_INSTALL2" --python-package-name-prefix "$PYTHON2_PKG_PREFIX" --depends "$PYTHON2_PACKAGE" -v 0.3.0 rdflib-jsonld
+fpm_build rdflib-jsonld rdflib-jsonld "" python 0.3.0
 
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
@@ -506,9 +514,7 @@ for deppkg in "${PYTHON_BACKPORTS[@]}"; do
                 "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
                 chmod -R go+rX .
                 set +e
-                # --iteration 2 provides an upgrade for previously built
-                # buggy packages.
-                fpm_build . "$outname" "" python "" --iteration 2
+                fpm_build . "$outname" "" python "" --iteration 3
                 # The upload step uses the package timestamp to determine
                 # whether it's new.  --no-clobber plays nice with that.
                 mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"

diff --git a/build/run-library.sh b/build/run-library.sh
index 8d97adab8132ba296b95261ac6ce4ab18db359ad..c7bc9538d1c7a54432a0fe840952acfdcea67c6b 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -192,6 +192,8 @@ fpm_build () {
   VERSION=$1
   shift
 
+  local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")"
+
   case "$PACKAGE_TYPE" in
       python)
           # All Arvados Python2 packages depend on Python 2.7.
@@ -199,7 +201,12 @@ fpm_build () {
           set -- "$@" --python-bin python2.7 \
               --python-easyinstall "$EASY_INSTALL2" \
               --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+              --prefix "$PYTHON2_PREFIX" \
+              --python-install-lib "$PYTHON2_INSTALL_LIB" \
+              --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
               --depends "$PYTHON2_PACKAGE"
+          # Fix --iteration for #9242.
+          default_iteration_value=$(($default_iteration_value + 1))
           ;;
       python3)
           # fpm does not actually support a python3 package type.  Instead
@@ -210,19 +217,21 @@ fpm_build () {
           set -- "$@" --python-bin python3 \
               --python-easyinstall "$EASY_INSTALL3" \
               --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+              --prefix "$PYTHON3_PREFIX" \
+              --python-install-lib "$PYTHON3_INSTALL_LIB" \
+              --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
               --depends "$PYTHON3_PACKAGE"
+          # Fix --iteration for #9242.
+          default_iteration_value=$(($default_iteration_value + 1))
           ;;
   esac
 
   declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
-  if [ python = "$PACKAGE_TYPE" ]; then
-    COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
-    if [ deb = "$FORMAT" ]; then
-        # Dependencies are built from setup.py.  Since setup.py will never
-        # refer to Debian package iterations, it doesn't make sense to
-        # enforce those in the .deb dependencies.
-        COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
-    fi
+  if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
+      # Dependencies are built from setup.py.  Since setup.py will never
+      # refer to Debian package iterations, it doesn't make sense to
+      # enforce those in the .deb dependencies.
+      COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
   fi
 
   if [[ "${DEBUG:-0}" != "0" ]]; then
@@ -242,7 +251,7 @@ fpm_build () {
   fi
   # We can always add an --iteration here.  If another one is specified in $@,
   # that will take precedence, as desired.
-  COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")")
+  COMMAND_ARR+=(--iteration "$default_iteration_value")
 
   # Append --depends X and other arguments specified by fpm-info.sh in
   # the package source dir. These are added last so they can override