X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0936c2e7630c0e4316684c5b79965ad7dfaf9fa4..0eb72b526bf8bbb011551ecf019f604e17a534f1:/build/run-build-packages.sh

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ddf93832b3..a8e730c27f 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin
+. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
@@ -16,7 +19,7 @@ Options:
 --debug
     Output debug information (default: false)
 --target <target>
-    Distribution to build packages for (default: debian7)
+    Distribution to build packages for (default: debian8)
 --only-build <package>
     Build only a specific package (or $ONLY_BUILD from environment)
 --command
@@ -29,7 +32,7 @@ EOF
 
 EXITCODE=0
 DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian7
+TARGET=debian8
 COMMAND=
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
@@ -101,68 +104,17 @@ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 ## End Debian Python defaults.
 
 case "$TARGET" in
-    debian7)
-        FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six  \
-            ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol requests)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
-        ;;
     debian8)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six  \
-            ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1204)
         FORMAT=deb
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six  \
-            ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro isodate ruamel.ordereddict
-            cachecontrol requests)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1404)
         FORMAT=deb
-        PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
-            google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
-            rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
-            shellescape mistune typing avro ruamel.ordereddict
-            cachecontrol)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
         ;;
-    centos6)
-        FORMAT=rpm
-        PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
-        PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
-        PYTHON2_PREFIX=/opt/rh/python27/root/usr
-        PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
-        PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
-        PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
-        PYTHON3_PREFIX=/opt/rh/python33/root/usr
-        PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six  \
-            ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
-            python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
-            'rdflib>=4.2.0' shellescape mistune typing avro requests \
-            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
-            ruamel.ordereddict cachecontrol)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
-        export PYCURL_SSL_LIBRARY=nss
+    ubuntu1604)
+        FORMAT=deb
         ;;
     centos7)
         FORMAT=rpm
@@ -173,15 +125,6 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
         PYTHON3_PREFIX=/opt/rh/python33/root/usr
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
-        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
-            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka  \
-            ciso8601 pycrypto 'pycurl<7.21.5' \
-            python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml \
-            'rdflib>=4.2.0' shellescape mistune typing avro \
-            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
-            ruamel.ordereddict cachecontrol)
-        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -254,21 +197,29 @@ fi
 debug_echo -e "\nPerl packages\n"
 
 if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
-cd "$WORKSPACE/sdk/perl"
+  cd "$WORKSPACE/sdk/perl"
+  libarvados_perl_version="$(version_from_git)"
 
-if [[ -e Makefile ]]; then
-  make realclean >"$STDOUT_IF_DEBUG"
-fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-    -delete
-rm -rf install
-
-perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
-    make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
-    fpm_build install/lib/=/usr/share libarvados-perl \
-    "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
-    "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
-    mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  cd $WORKSPACE/packages/$TARGET
+  test_package_presence libarvados-perl "$libarvados_perl_version"
+
+  if [[ "$?" == "0" ]]; then
+    cd "$WORKSPACE/sdk/perl"
+
+    if [[ -e Makefile ]]; then
+      make realclean >"$STDOUT_IF_DEBUG"
+    fi
+    find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+        -delete
+    rm -rf install
+
+    perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+        make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+        fpm_build install/lib/=/usr/share libarvados-perl \
+        "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+        "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+        mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+  fi
 fi
 
 # Ruby gems
@@ -305,27 +256,33 @@ handle_python_package
 
 # arvados-src
 (
-    set -e
-
     cd "$WORKSPACE"
     COMMIT_HASH=$(format_last_commit_here "%H")
+    arvados_src_version="$(version_from_git)"
 
-    SRC_BUILD_DIR=$(mktemp -d)
-    # mktemp creates the directory with 0700 permissions by default
-    chmod 755 $SRC_BUILD_DIR
-    git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
-    cd "$SRC_BUILD_DIR"
+    cd $WORKSPACE/packages/$TARGET
+    test_package_presence arvados-src $arvados_src_version src ""
 
-    # go into detached-head state
-    git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
-    echo "$COMMIT_HASH" >git-commit.version
+    if [[ "$?" == "0" ]]; then
+      cd "$WORKSPACE"
+      SRC_BUILD_DIR=$(mktemp -d)
+      # mktemp creates the directory with 0700 permissions by default
+      chmod 755 $SRC_BUILD_DIR
+      git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+      cd "$SRC_BUILD_DIR"
 
-    cd "$SRC_BUILD_DIR"
-    PKG_VERSION=$(version_from_git)
-    cd $WORKSPACE/packages/$TARGET
-    fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+      # go into detached-head state
+      git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+      echo "$COMMIT_HASH" >git-commit.version
+
+      cd "$SRC_BUILD_DIR"
+      PKG_VERSION=$(version_from_git)
+      cd $WORKSPACE/packages/$TARGET
+      fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
 
-    rm -rf "$SRC_BUILD_DIR"
+      rm -rf "$SRC_BUILD_DIR"
+
+    fi
 )
 
 # On older platforms we need to publish a backport of libfuse >=2.9.2,
@@ -375,38 +332,6 @@ if [[ $TARGET =~ ubuntu1204 ]]; then
         "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
     apt-get -y --no-install-recommends -f install
     rm -rf $LIBFUSE_DIR
-elif [[ $TARGET =~ centos6 ]]; then
-    # port fuse 2.9.2 to centos 6
-    # install tools to build rpm from source
-    yum install -y rpm-build redhat-rpm-config
-    LIBFUSE_DIR=$(mktemp -d)
-    (
-        cd "$LIBFUSE_DIR"
-        # download fuse 2.9.2 centos 7 source rpm
-        file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
-        (
-            # modify source rpm spec to remove conflict on filesystem version
-            mkdir -p /root/rpmbuild/SOURCES
-            cd /root/rpmbuild/SOURCES
-            rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
-            perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
-        )
-        # build rpms from source
-        rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
-        rm -f fuse-2.9.2-6.el7.src.rpm
-        # move built RPMs to LIBFUSE_DIR
-        mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
-        mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
-        mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
-        rm -rf /root/rpmbuild
-    )
-    fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
-    fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
-    fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
-    yum install -y \
-        "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
-        "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
-        "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
 fi
 
 # Go binaries
@@ -424,8 +349,6 @@ package_go_binary services/crunch-run crunch-run \
     "Supervise a single Crunch container"
 package_go_binary services/crunchstat crunchstat \
     "Gather cpu/memory/network statistics of running Crunch jobs"
-package_go_binary services/datamanager arvados-data-manager \
-    "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
 package_go_binary services/keep-balance keep-balance \
     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
 package_go_binary services/keepproxy keepproxy \
@@ -434,6 +357,8 @@ package_go_binary services/keepstore keepstore \
     "Keep storage daemon, accessible to clients on the LAN"
 package_go_binary services/keep-web keep-web \
     "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary services/ws arvados-ws \
+    "Arvados Websocket server"
 package_go_binary tools/keep-block-check keep-block-check \
     "Verify that all data from one set of Keep servers to another was copied"
 package_go_binary tools/keep-rsync keep-rsync \
@@ -450,14 +375,21 @@ package_go_binary tools/keep-exercise keep-exercise \
 # 2014-05-15
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
+arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+fi
 
 # cwl-runner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
-
-fpm_build lockfile "" "" python 0.12.2 --epoch 1
+arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
+arvados_cwl_runner_iteration=3
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+fi
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the
@@ -474,28 +406,28 @@ fpm_build lockfile "" "" python 0.12.2 --epoch 1
 # So we build this thing separately.
 #
 # Ward, 2016-03-17
-fpm_build schema_salad "" "" python 1.20.20161122192122 --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
-
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
-# Ward, 2016-05-26
-fpm_build ruamel.yaml "" "" python 0.12.4 --python-setup-py-arguments "--single-version-externally-managed"
-
-# Dependency of cwltool.  Fpm doesn't produce a package with the correct version
-# number unless we build it explicitly
-fpm_build cwltest "" "" python 1.0.20160907111242
+saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
+test_package_presence python-schema-salad "$saladversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+fi
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(1\.0\..*\)'.*/\1/")
-fpm_build cwltool "" "" python $cwltoolversion
-
-# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm_build rdflib-jsonld "" "" python 0.3.0
+cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
+test_package_presence python-cwltool "$cwltoolversion" python
+if [[ "$?" == "0" ]]; then
+  fpm_build cwltool "" "" python $cwltoolversion
+fi
 
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
     cd $WORKSPACE/packages/$TARGET
     rm -rf "$WORKSPACE/sdk/pam/build"
-    fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    libpam_arvados_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)
+    test_package_presence libpam-arvados "$libpam_arvados_version" python
+    if [[ "$?" == "0" ]]; then
+      fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$libpam_arvados_version" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+    fi
 fi
 
 # The FUSE driver
@@ -503,37 +435,56 @@ fi
 # not omit the python- prefix first.
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)
+test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The node manager
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)
+test_package_presence arvados-node-manager "$nodemanager_version" python
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
 
 # The Docker image cleaner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
+dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
+dockercleaner_iteration=3
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+if [[ "$?" == "0" ]]; then
+  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+fi
 
 # The Arvados crunchstat-summary tool
 cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+if [[ "$?" == "0" ]]; then
+  rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
+  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+fi
 
-# Forked libcloud
-LIBCLOUD_DIR=$(mktemp -d)
-(
-    cd $LIBCLOUD_DIR
-    git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
-    git checkout apache-libcloud-$LIBCLOUD_PIN
-    # libcloud is absurdly noisy without -q, so force -q here
-    OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
-    DASHQ_UNLESS_DEBUG=-q
-    handle_python_package
-    DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
-)
-fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
-rm -rf $LIBCLOUD_DIR
+if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
+  # Forked libcloud
+  LIBCLOUD_DIR=$(mktemp -d)
+  (
+      cd $LIBCLOUD_DIR
+      git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+      git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
+      # libcloud is absurdly noisy without -q, so force -q here
+      OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+      DASHQ_UNLESS_DEBUG=-q
+      handle_python_package
+      DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+  )
+  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+  rm -rf $LIBCLOUD_DIR
+fi
 
 # Python 2 dependencies
 declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
@@ -545,99 +496,144 @@ case "$?" in
     *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
 esac
 
-for deppkg in "${PYTHON_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+while read -r line || [[ -n "$line" ]]; do
+#  echo "Text read from file: $line"
+  if [[ "$line" =~ ^# ]]; then
+    continue
+  fi
+  IFS='|'; arr=($line); unset IFS
 
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
-        continue
-    fi
+  dist=${arr[0]}
 
-    case "$deppkg" in
-        httplib2|google-api-python-client)
+  IFS=',';dists=($dist); unset IFS
+
+  MATCH=0
+  for d in "${dists[@]}"; do
+    if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
+      MATCH=1
+    fi
+  done
+
+  if [[ "$MATCH" != "1" ]]; then
+    continue
+  fi
+  name=${arr[1]}
+  version=${arr[2]}
+  iteration=${arr[3]}
+  pkgtype=${arr[4]}
+  arch=${arr[5]}
+  extra=${arr[6]}
+  declare -a 'extra_arr=('"$extra"')'
+
+  if [[ "$FORMAT" == "rpm" ]]; then
+    if [[ "$arch" == "all" ]]; then
+      arch="noarch"
+    fi
+    if [[ "$arch" == "amd64" ]]; then
+      arch="x86_64"
+    fi
+  fi
+
+  if [[ "$pkgtype" == "python" ]]; then
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+  else
+    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+  fi
+
+  if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
+      continue
+  fi
+
+  case "$name" in
+      httplib2|google-api-python-client)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
             # Work around 0640 permissions on some package files.
             # See #7591 and #7991.
             pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
                 set -e
                 cd "$pyfpm_workdir"
-                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+                pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
                 # Sometimes pip gives us a tarball, sometimes a zip file...
-                DOWNLOADED=`ls $deppkg-*`
+                DOWNLOADED=`ls $name-*`
                 [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
                 [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
-                cd "$deppkg"-*/
+                cd "$name"-*/
                 "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
                 chmod -R go+rX .
                 set +e
-                fpm_build . "$outname" "" python "" --iteration 3
+                fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
                 # The upload step uses the package timestamp to determine
-                # whether it's new.  --no-clobber plays nice with that.
+                # if it is new.  --no-clobber plays nice with that.
                 mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
             )
             if [ 0 != "$?" ]; then
-                echo "ERROR: $deppkg build process failed"
+                echo "ERROR: $name build process failed"
                 EXITCODE=1
             fi
             if [ -n "$pyfpm_workdir" ]; then
                 rm -rf "$pyfpm_workdir"
             fi
-            ;;
-        *)
-            fpm_build "$deppkg" "$outname"
-            ;;
-    esac
-done
-
-# Python 3 dependencies
-for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
-    outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
-    # The empty string is the vendor argument: these aren't Curoverse software.
-    fpm_build "$deppkg" "$outname" "" python3
-done
+          fi
+          ;;
+      *)
+          test_package_presence $outname $version $pkgtype $iteration $arch
+          if [[ "$?" == "0" ]]; then
+            fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
+          fi
+          ;;
+  esac
+
+done <`dirname "$(readlink -f "$0")"`"/build.list"
 
 # Build the API server package
-handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
-    "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-    --description="Arvados API server - Arvados is a free and open source platform for big data science." \
-    --license="GNU Affero General Public License, version 3.0"
-
-# Build the workbench server package
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then
-(
-    set -e
-    cd "$WORKSPACE/apps/workbench"
-
-    # We need to bundle to be ready even when we build a package without vendor directory
-    # because asset compilation requires it.
-    bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
-
-    # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-    # and we want that in the package, so it's easier to not exclude the tmp directory
-    # from the package - empty it instead.
-    rm -rf tmp
-    mkdir tmp
-
-    # Set up application.yml and production.rb so that asset precompilation works
-    \cp config/application.yml.example config/application.yml -f
-    \cp config/environments/production.rb.example config/environments/production.rb -f
-    sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
-
-    RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
-
-    # Remove generated configuration files so they don't go in the package.
-    rm config/application.yml config/environments/production.rb
-)
-fi
-
-if [[ "$?" != "0" ]]; then
-  echo "ERROR: Asset precompilation failed"
-  EXITCODE=1
-else
-  handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+if [[ "$?" == "0" ]]; then
+  handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
       "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-      --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+      --description="Arvados API server - Arvados is a free and open source platform for big data science." \
       --license="GNU Affero General Public License, version 3.0"
 fi
 
+# Build the workbench server package
+test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+if [[ "$?" == "0" ]] ; then
+  (
+      set -e
+      cd "$WORKSPACE/apps/workbench"
+
+      # We need to bundle to be ready even when we build a package without vendor directory
+      # because asset compilation requires it.
+      bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+      # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+      # and we want that in the package, so it's easier to not exclude the tmp directory
+      # from the package - empty it instead.
+      rm -rf tmp
+      mkdir tmp
+
+      # Set up application.yml and production.rb so that asset precompilation works
+      \cp config/application.yml.example config/application.yml -f
+      \cp config/environments/production.rb.example config/environments/production.rb -f
+      sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+      # Remove generated configuration files so they don't go in the package.
+      rm config/application.yml config/environments/production.rb
+  )
+
+  if [[ "$?" != "0" ]]; then
+    echo "ERROR: Asset precompilation failed"
+    EXITCODE=1
+  else
+    handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+        --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+        --license="GNU Affero General Public License, version 3.0"
+  fi
+fi
+
 # clean up temporary GOPATH
 rm -rf "$GOPATH"