X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d0d60bbbe1c60516334e5d56f3c923b79e611b21..0bd8810e6ea29ab242472b9dae11c621cf50953c:/build/run-build-packages.sh

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 29b7e8cc7e..7763ca5454 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin
+. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
@@ -103,34 +103,15 @@ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 case "$TARGET" in
     debian8)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six \
-            ciso8601 pycrypto backports.ssl_match_hostname 'llfuse>=1.0' \
-            'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol 'pathlib2>=2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1204)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six \
-            ciso8601 pycrypto backports.ssl_match_hostname 'llfuse>=1.0' \
-            contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro isodate ruamel.ordereddict
-            cachecontrol requests 'pathlib2>=2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1404)
         FORMAT=deb
-        PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 'llfuse>=1.0' ciso8601 \
-            google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
-            rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol 'pathlib2>=2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
+        ;;
+    ubuntu1604)
+        FORMAT=deb
         ;;
     centos7)
         FORMAT=rpm
@@ -141,15 +122,6 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
         PYTHON3_PREFIX=/opt/rh/python33/root/usr
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka  \
-            ciso8601 pycrypto 'pycurl<7.21.5' \
-            python-daemon==2.1.1 'llfuse>=1.0' 'pbr<1.0' pyyaml contextlib2 \
-            'rdflib>=4.2.0' shellescape mistune typing avro \
-            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
-            ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -222,21 +194,29 @@ fi
 debug_echo -e "\nPerl packages\n"
 
 if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
-cd "$WORKSPACE/sdk/perl"
+  cd "$WORKSPACE/sdk/perl"
+  libarvados_perl_version="$(version_from_git)"
 
-if [[ -e Makefile ]]; then
-  make realclean >"$STDOUT_IF_DEBUG"
-fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-    -delete
-rm -rf install
-
-perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
-    make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
-    fpm_build install/lib/=/usr/share libarvados-perl \
-    "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
-    "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
-    mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  cd $WORKSPACE/packages/$TARGET
+  test_package_presence libarvados-perl "$libarvados_perl_version"
+
+  if [[ "$?" == "0" ]]; then
+    cd "$WORKSPACE/sdk/perl"
+
+    if [[ -e Makefile ]]; then
+      make realclean >"$STDOUT_IF_DEBUG"
+    fi
+    find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+        -delete
+    rm -rf install
+
+    perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+        make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+        fpm_build install/lib/=/usr/share libarvados-perl \
+        "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+        "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+        mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  fi
 fi
 
 # Ruby gems
@@ -273,27 +253,33 @@ handle_python_package
 
 # arvados-src
 (
-    set -e
-
     cd "$WORKSPACE"
     COMMIT_HASH=$(format_last_commit_here "%H")
+    arvados_src_version="$(version_from_git)"
 
-    SRC_BUILD_DIR=$(mktemp -d)
-    # mktemp creates the directory with 0700 permissions by default
-    chmod 755 $SRC_BUILD_DIR
-    git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
-    cd "$SRC_BUILD_DIR"
+    cd $WORKSPACE/packages/$TARGET
+    test_package_presence arvados-src $arvados_src_version src ""
 
-    # go into detached-head state
-    git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
-    echo "$COMMIT_HASH" >git-commit.version
+    if [[ "$?" == "0" ]]; then
+      cd "$WORKSPACE"
+      SRC_BUILD_DIR=$(mktemp -d)
+      # mktemp creates the directory with 0700 permissions by default
+      chmod 755 $SRC_BUILD_DIR
+      git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+      cd "$SRC_BUILD_DIR"
 
-    cd "$SRC_BUILD_DIR"
-    PKG_VERSION=$(version_from_git)
-    cd $WORKSPACE/packages/$TARGET
-    fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+      # go into detached-head state
+      git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+      echo "$COMMIT_HASH" >git-commit.version
+
+      cd "$SRC_BUILD_DIR"
+      PKG_VERSION=$(version_from_git)
+      cd $WORKSPACE/packages/$TARGET
+      fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+      rm -rf "$SRC_BUILD_DIR"
 
-    rm -rf "$SRC_BUILD_DIR"
+    fi
 )
 
 # On older platforms we need to publish a backport of libfuse >=2.9.2,
@@ -386,14 +372,21 @@ package_go_binary tools/keep-exercise keep-exercise \
 # 2014-05-15
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+fi
 
 # cwl-runner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
-
-fpm_build lockfile "" "" python 0.12.2 --epoch 1
+arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
+arvados_cwl_runner_iteration=3
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+fi
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -411,28 +404,27 @@ fpm_build lockfile "" "" python 0.12.2 --epoch 1
 #
 # Ward, 2016-03-17
 saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
-
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
-# Ward, 2016-05-26
-fpm_build ruamel.yaml "" "" python 0.13.7 --python-setup-py-arguments "--single-version-externally-managed"
-
-# Dependency of cwltool.  Fpm doesn't produce a package with the correct version
-# number unless we build it explicitly
-fpm_build cwltest "" "" python 1.0.20160907111242
+test_package_presence python-schema-salad "$saladversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+fi
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
 cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build cwltool "" "" python $cwltoolversion
-
-# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm_build rdflib-jsonld "" "" python 0.4.0
+test_package_presence python-cwltool "$cwltoolversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build cwltool "" "" python $cwltoolversion
+fi
 
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
     cd $WORKSPACE/packages/$TARGET
     rm -rf "$WORKSPACE/sdk/pam/build"
-    fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    libpam_arvados_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)
+    test_package_presence libpam-arvados "$libpam_arvados_version" python
+    if [[ "$?" == "0" ]]; then
+      fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$libpam_arvados_version" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    fi
 fi
 
 # The FUSE driver
@@ -440,37 +432,56 @@ fi
 # not omit the python- prefix first.
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)
+test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The node manager
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)
+test_package_presence arvados-node-manager "$nodemanager_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The Docker image cleaner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
+dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
+dockercleaner_iteration=3
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+fi
 
 # The Arvados crunchstat-summary tool
 cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+if [[ "$?" == "0" ]]; then
+  rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
+  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+fi
 
-# Forked libcloud
-LIBCLOUD_DIR=$(mktemp -d)
-(
-    cd $LIBCLOUD_DIR
-    git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
-    git checkout apache-libcloud-$LIBCLOUD_PIN
-    # libcloud is absurdly noisy without -q, so force -q here
-    OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
-    DASHQ_UNLESS_DEBUG=-q
-    handle_python_package
-    DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
-)
-fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
-rm -rf $LIBCLOUD_DIR
+if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
+  # Forked libcloud
+  LIBCLOUD_DIR=$(mktemp -d)
+  (
+      cd $LIBCLOUD_DIR
+      git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+      git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
+      # libcloud is absurdly noisy without -q, so force -q here
+      OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+      DASHQ_UNLESS_DEBUG=-q
+      handle_python_package
+      DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+  )
+  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+  rm -rf $LIBCLOUD_DIR
+fi
 
 # Python 2 dependencies
 declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
@@ -482,99 +493,144 @@ case "$?" in
     *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
 esac
 
-for deppkg in "${PYTHON_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+while read -r line || [[ -n "$line" ]]; do
+#  echo "Text read from file: $line"
+  if [[ "$line" =~ ^# ]]; then
+    continue
+  fi
+  IFS='|'; arr=($line); unset IFS
 
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
-        continue
-    fi
+  dist=${arr[0]}
 
-    case "$deppkg" in
-        httplib2|google-api-python-client)
+  IFS=',';dists=($dist); unset IFS
+
+  MATCH=0
+  for d in "${dists[@]}"; do
+    if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
+      MATCH=1
+    fi
+  done
+
+  if [[ "$MATCH" != "1" ]]; then
+    continue
+  fi
+  name=${arr[1]}
+  version=${arr[2]}
+  iteration=${arr[3]}
+  pkgtype=${arr[4]}
+  arch=${arr[5]}
+  extra=${arr[6]}
+  declare -a 'extra_arr=('"$extra"')'
+
+  if [[ "$FORMAT" == "rpm" ]]; then
+    if [[ "$arch" == "all" ]]; then
+      arch="noarch"
+    fi
+    if [[ "$arch" == "amd64" ]]; then
+      arch="x86_64"
+    fi
+  fi
+
+  if [[ "$pkgtype" == "python" ]]; then
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+  else
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+  fi
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
+      continue
+  fi
+
+  case "$name" in
+      httplib2|google-api-python-client)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
             # Work around 0640 permissions on some package files.
             # See #7591 and #7991.
             pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
                 set -e
                 cd "$pyfpm_workdir"
-                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
                 # Sometimes pip gives us a tarball, sometimes a zip file...
-                DOWNLOADED=`ls $deppkg-*`
+                DOWNLOADED=`ls $name-*`
                 [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
                 [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
-                cd "$deppkg"-*/
+                cd "$name"-*/
                 "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
                 chmod -R go+rX .
                 set +e
-                fpm_build . "$outname" "" python "" --iteration 3
+                fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
                 # The upload step uses the package timestamp to determine
-                # whether it's new.  --no-clobber plays nice with that.
+                # if it is new.  --no-clobber plays nice with that.
                 mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
             )
             if [ 0 != "$?" ]; then
-                echo "ERROR: $deppkg build process failed"
+                echo "ERROR: $name build process failed"
                 EXITCODE=1
             fi
             if [ -n "$pyfpm_workdir" ]; then
                 rm -rf "$pyfpm_workdir"
             fi
-            ;;
-        *)
-            fpm_build "$deppkg" "$outname"
-            ;;
-    esac
-done
-
-# Python 3 dependencies
-for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
-    # The empty string is the vendor argument: these aren't Curoverse software.
-    fpm_build "$deppkg" "$outname" "" python3
-done
+          fi
+          ;;
+      *)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
+            fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
+          fi
+          ;;
+  esac
+
+done <`dirname "$(readlink -f "$0")"`"/build.list"
 
 # Build the API server package
-handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
-    "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-    --description="Arvados API server - Arvados is a free and open source platform for big data science." \
-    --license="GNU Affero General Public License, version 3.0"
-
-# Build the workbench server package
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then
-(
-    set -e
-    cd "$WORKSPACE/apps/workbench"
-
-    # We need to bundle to be ready even when we build a package without vendor directory
-    # because asset compilation requires it.
-    bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
-
-    # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-    # and we want that in the package, so it's easier to not exclude the tmp directory
-    # from the package - empty it instead.
-    rm -rf tmp
-    mkdir tmp
-
-    # Set up application.yml and production.rb so that asset precompilation works
-    \cp config/application.yml.example config/application.yml -f
-    \cp config/environments/production.rb.example config/environments/production.rb -f
-    sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
-
-    RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
-
-    # Remove generated configuration files so they don't go in the package.
-    rm config/application.yml config/environments/production.rb
-)
-fi
-
-if [[ "$?" != "0" ]]; then
-  echo "ERROR: Asset precompilation failed"
-  EXITCODE=1
-else
-  handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+if [[ "$?" == "0" ]]; then
+  handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
       "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-      --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+      --description="Arvados API server - Arvados is a free and open source platform for big data science." \
       --license="GNU Affero General Public License, version 3.0"
 fi
 
+# Build the workbench server package
+test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+if [[ "$?" == "0" ]] ; then
+  (
+      set -e
+      cd "$WORKSPACE/apps/workbench"
+
+      # We need to bundle to be ready even when we build a package without vendor directory
+      # because asset compilation requires it.
+      bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+      # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+      # and we want that in the package, so it's easier to not exclude the tmp directory
+      # from the package - empty it instead.
+      rm -rf tmp
+      mkdir tmp
+
+      # Set up application.yml and production.rb so that asset precompilation works
+      \cp config/application.yml.example config/application.yml -f
+      \cp config/environments/production.rb.example config/environments/production.rb -f
+      sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+      # Remove generated configuration files so they don't go in the package.
+      rm config/application.yml config/environments/production.rb
+  )
+
+  if [[ "$?" != "0" ]]; then
+    echo "ERROR: Asset precompilation failed"
+    EXITCODE=1
+  else
+    handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+        --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+        --license="GNU Affero General Public License, version 3.0"
+  fi
+fi
+
 # clean up temporary GOPATH
 rm -rf "$GOPATH"