X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9f57bcbab5b244365ca4d0ece8490d7540c72b86..b3a016e9a47d453b5ae4d287d8b6eaafd69971df:/build/run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 40589fd565..ea81bbf17c 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -11,7 +11,7 @@
 LICENSE_PACKAGE_TS=20151208015500
 
 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
-    RAILS_PACKAGE_ITERATION=8
+    RAILS_PACKAGE_ITERATION=1
 else
     RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
 fi
@@ -261,29 +261,33 @@ test_package_presence() {
     # Get the list of packages from the repos
 
     if [[ "$FORMAT" == "deb" ]]; then
-      debian_distros="jessie precise stretch trusty wheezy xenial bionic"
-
-      for D in ${debian_distros}; do
-        if [ ${pkgname:0:3} = "lib" ]; then
-          repo_subdir=${pkgname:0:4}
-        else
-          repo_subdir=${pkgname:0:1}
-        fi
+      declare -A dd
+      dd[debian8]=jessie
+      dd[debian9]=stretch
+      dd[debian10]=buster
+      dd[ubuntu1404]=trusty
+      dd[ubuntu1604]=xenial
+      dd[ubuntu1804]=bionic
+      D=${dd[$TARGET]}
+      if [ ${pkgname:0:3} = "lib" ]; then
+        repo_subdir=${pkgname:0:4}
+      else
+        repo_subdir=${pkgname:0:1}
+      fi
 
-        repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
-        echo ${repo_pkg_list} |grep -q ${complete_pkgname}
-        if [ $? -eq 0 ] ; then
-          echo "Package $complete_pkgname exists, not rebuilding!"
-          curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
-          return 1
-        elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
-          echo "Package $complete_pkgname exists, not rebuilding!"
-          return 1
-        else
-          echo "Package $complete_pkgname not found, building"
-          return 0
-        fi
-      done
+      repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
+      echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+      if [ $? -eq 0 ] ; then
+        echo "Package $complete_pkgname exists, not rebuilding!"
+        curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+        return 1
+      elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
+        echo "Package $complete_pkgname exists, not rebuilding!"
+        return 1
+      else
+        echo "Package $complete_pkgname not found, building"
+        return 0
+      fi
     else
       centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
 
@@ -409,7 +413,9 @@ fpm_build_virtualenv () {
     PYTHON_PKG=$PKG
   fi
 
-  if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+  # arvados-python-client sdist should always be built, to be available
+  # for other dependant packages.
+  if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0
   fi
 
@@ -423,13 +429,22 @@ fpm_build_virtualenv () {
     echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
     exit 1
   fi
-  if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist; then
+  # filter a useless warning (when building the cwltest package) from the stderr output
+  if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist 2> >(grep -v 'warning: no previously-included files matching'); then
     echo "Error, unable to run $python setup.py sdist for $PKG"
     exit 1
   fi
 
   PACKAGE_PATH=`(cd dist; ls *tar.gz)`
 
+  if [[ "arvados-python-client" == "$PKG" ]]; then
+    PYSDK_PATH=`pwd`/dist/
+  fi
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
+    return 0
+  fi
+
   # Determine the package version from the generated sdist archive
   PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)}
 
@@ -450,7 +465,7 @@ fpm_build_virtualenv () {
 
   rm -rf build
   rm -f $PYTHON_PKG*deb
-
+  echo "virtualenv version: `virtualenv --version`"
   virtualenv_command="virtualenv --python `which $python` $DASHQ_UNLESS_DEBUG build/usr/share/$python/dist/$PYTHON_PKG"
 
   if ! $virtualenv_command; then
@@ -464,28 +479,33 @@ fpm_build_virtualenv () {
     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
     exit 1
   fi
+  echo "pip version:        `build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip --version`"
+
   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
     echo "Error, unable to upgrade setuptools with"
     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
     exit 1
   fi
+  echo "setuptools version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/$python -c 'import setuptools; print(setuptools.__version__)'`"
+
   if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
     echo "Error, unable to upgrade wheel with"
     echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
     exit 1
   fi
+  echo "wheel version:      `build/usr/share/$python/dist/$PYTHON_PKG/bin/wheel version`"
 
   if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
   else
     # centos7 needs these special tweaks to install python-arvados-fuse
     build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
-    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
   fi
 
   if [[ "$?" != "0" ]]; then
     echo "Error, unable to run"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH"
     exit 1
   fi
 
@@ -614,10 +634,13 @@ fpm_build_virtualenv () {
     done
   fi
 
-  # the libpam module should place this file in the historically correct place
-  # so as not to break backwards compatibility
-  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/libpam-arvados/lib/security/libpam_arvados.py" ]]; then
-    COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/data/lib/security/libpam_arvados.py=/usr/data/lib/security/")
+  # the libpam module should place a few files in the correct place for the pam
+  # subsystem
+  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/lib/security/libpam_arvados.py" ]]; then
+    COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/lib/security/libpam_arvados.py=/usr/lib/security/")
+  fi
+  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/share/pam-configs/arvados" ]]; then
+    COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/share/pam-configs/arvados=/usr/share/pam-configs/")
   fi
 
   # the python-arvados-cwl-runner package comes with cwltool, expose that version
@@ -816,14 +839,13 @@ install_package() {
   fi
 }
 
-title () {
-    txt="********** $1 **********"
-    printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
+title() {
+    printf '%s %s\n' "=======" "$1"
 }
 
 checkexit() {
     if [[ "$1" != "0" ]]; then
-        title "!!!!!! $2 FAILED !!!!!!"
+        title "$2 -- FAILED"
         failures+=("$2 (`timer`)")
     else
         successes+=("$2 (`timer`)")
@@ -846,7 +868,9 @@ report_outcomes() {
 
     if [[ ${#failures[@]} == 0 ]]
     then
-        echo "All test suites passed."
+        if [[ ${#successes[@]} != 0 ]]; then
+           echo "All test suites passed."
+        fi
     else
         echo "Failures (${#failures[@]}):"
         for x in "${failures[@]}"