Merge branch '14291-cdc-aws' refs #14291

[arvados.git] / build / run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 9df62adee0dd18e751cf680998148162c8b7d5a9..40589fd565c258240fed5fe1057fad5ab38993b1 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -274,7 +274,7 @@ test_package_presence() {
         echo ${repo_pkg_list} |grep -q ${complete_pkgname}
         if [ $? -eq 0 ] ; then
           echo "Package $complete_pkgname exists, not rebuilding!"
-          curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+          curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
           return 1
         elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
           echo "Package $complete_pkgname exists, not rebuilding!"
@@ -287,11 +287,11 @@ test_package_presence() {
     else
       centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
 
-      repo_pkg_list=$(curl -o - ${centos_repo})
+      repo_pkg_list=$(curl -s -o - ${centos_repo})
       echo ${repo_pkg_list} |grep -q ${complete_pkgname}
       if [ $? -eq 0 ]; then
         echo "Package $complete_pkgname exists, not rebuilding!"
-        curl -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+        curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
         return 1
       elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
         echo "Package $complete_pkgname exists, not rebuilding!"
@@ -381,16 +381,23 @@ fpm_build_virtualenv () {
 
   local python=""
   case "$PACKAGE_TYPE" in
+    python3)
+        python=python3
+        if [[ "$FORMAT" != "rpm" ]]; then
+          pip=pip3
+        else
+          # In CentOS, we use a different mechanism to get the right version of pip
+          pip=pip
+        fi
+        PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
+        ;;
     python)
         # All Arvados Python2 packages depend on Python 2.7.
         # Make sure we build with that for consistency.
         python=python2.7
+        pip=pip
         PACKAGE_PREFIX=$PYTHON2_PKG_PREFIX
         ;;
-    python3)
-        PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
-        python=python3
-        ;;
   esac
 
   if [[ "$PKG" != "libpam-arvados" ]] &&
@@ -410,8 +417,14 @@ fpm_build_virtualenv () {
 
   rm -rf dist/*
 
+  # Get the latest setuptools
+  if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+    echo "Error, unable to upgrade setuptools with"
+    echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+    exit 1
+  fi
   if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist; then
-    echo "Error, unable to run python setup.py sdist for $PKG"
+    echo "Error, unable to run $python setup.py sdist for $PKG"
     exit 1
   fi
 
@@ -446,28 +459,33 @@ fpm_build_virtualenv () {
     exit 1
   fi
 
-  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
     echo "Error, unable to upgrade pip with"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
     exit 1
   fi
-  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+    echo "Error, unable to upgrade setuptools with"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+    exit 1
+  fi
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
     echo "Error, unable to upgrade wheel with"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
     exit 1
   fi
 
   if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
   else
     # centos7 needs these special tweaks to install python-arvados-fuse
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
-    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
+    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
   fi
 
   if [[ "$?" != "0" ]]; then
     echo "Error, unable to run"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
     exit 1
   fi
 
@@ -602,6 +620,11 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/data/lib/security/libpam_arvados.py=/usr/data/lib/security/")
   fi
 
+  # the python-arvados-cwl-runner package comes with cwltool, expose that version
+  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
+    COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+  fi
+
   COMMAND_ARR+=(".")
 
   FPM_RESULTS=$("${COMMAND_ARR[@]}")