Merge branch '14853-chapmanb-subprocess-merge'

refs #14853

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --combined build/run-library.sh
index 58e6f0d888130ce242cd8f9f3dffe0b36ac117c1,8bf2166d5e2a3c29e46494cba956f1cec47eeeb4..40589fd565c258240fed5fe1057fad5ab38993b1
--- 1/build/run-library.sh
--- 2/build/run-library.sh
+++ b/build/run-library.sh
@@@ -274,7 -274,7 +274,7 @@@ test_package_presence() 
          echo ${repo_pkg_list} |grep -q ${complete_pkgname}
          if [ $? -eq 0 ] ; then
            echo "Package $complete_pkgname exists, not rebuilding!"
 -          curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
 +          curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
            return 1
          elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
            echo "Package $complete_pkgname exists, not rebuilding!"
@@@ -287,11 -287,11 +287,11 @@@
      else
        centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
  
 -      repo_pkg_list=$(curl -o - ${centos_repo})
 +      repo_pkg_list=$(curl -s -o - ${centos_repo})
        echo ${repo_pkg_list} |grep -q ${complete_pkgname}
        if [ $? -eq 0 ]; then
          echo "Package $complete_pkgname exists, not rebuilding!"
 -        curl -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
 +        curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
          return 1
        elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
          echo "Package $complete_pkgname exists, not rebuilding!"
@@@ -381,16 -381,23 +381,23 @@@ fpm_build_virtualenv () 
  
    local python=""
    case "$PACKAGE_TYPE" in
+     python3)
+         python=python3
+         if [[ "$FORMAT" != "rpm" ]]; then
+           pip=pip3
+         else
+           # In CentOS, we use a different mechanism to get the right version of pip
+           pip=pip
+         fi
+         PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
+         ;;
      python)
          # All Arvados Python2 packages depend on Python 2.7.
          # Make sure we build with that for consistency.
          python=python2.7
+         pip=pip
          PACKAGE_PREFIX=$PYTHON2_PKG_PREFIX
          ;;
-     python3)
-         PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
-         python=python3
-         ;;
    esac
  
    if [[ "$PKG" != "libpam-arvados" ]] &&
@@@ -410,8 -417,14 +417,14 @@@
  
    rm -rf dist/*
  
+   # Get the latest setuptools
+   if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+     echo "Error, unable to upgrade setuptools with"
+     echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+     exit 1
+   fi
    if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist; then
-     echo "Error, unable to run python setup.py sdist for $PKG"
+     echo "Error, unable to run $python setup.py sdist for $PKG"
      exit 1
    fi
  
@@@ -446,28 -459,33 +459,33 @@@
      exit 1
    fi
  
-   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
+   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
      echo "Error, unable to upgrade pip with"
-     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
+     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
+     exit 1
+   fi
+   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+     echo "Error, unable to upgrade setuptools with"
+     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
      exit 1
    fi
-   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
+   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
      echo "Error, unable to upgrade wheel with"
-     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
+     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
      exit 1
    fi
  
    if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
-     build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
    else
      # centos7 needs these special tweaks to install python-arvados-fuse
-     build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
-     PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
+     PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
    fi
  
    if [[ "$?" != "0" ]]; then
      echo "Error, unable to run"
-     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
      exit 1
    fi