9945: use virtualenv for our python packages
authorWard Vandewege <wvandewege@veritasgenetics.com>
Tue, 22 Jan 2019 01:46:50 +0000 (20:46 -0500)
committerWard Vandewege <wvandewege@veritasgenetics.com>
Tue, 22 Jan 2019 02:06:50 +0000 (21:06 -0500)
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege@veritasgenetics.com>

20 files changed:
build/build.list [deleted file]
build/package-build-dockerfiles/centos7/Dockerfile
build/package-build-dockerfiles/debian8/Dockerfile
build/package-build-dockerfiles/debian9/Dockerfile
build/package-build-dockerfiles/ubuntu1404/Dockerfile
build/package-build-dockerfiles/ubuntu1604/Dockerfile
build/package-build-dockerfiles/ubuntu1804/Dockerfile
build/package-testing/deb-common-test-packages.sh
build/package-testing/rpm-common-test-packages.sh
build/package-testing/test-package-arvados-node-manager.sh
build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
build/package-testing/test-package-python27-python-arvados-fuse.sh
build/package-testing/test-package-python27-python-arvados-python-client.sh
build/run-build-packages.sh
build/run-library.sh
sdk/cwl/fpm-info.sh [new file with mode: 0644]
sdk/python/fpm-info.sh [new file with mode: 0644]
services/fuse/fpm-info.sh
services/nodemanager/fpm-info.sh [new file with mode: 0644]
tools/crunchstat-summary/fpm-info.sh [new file with mode: 0644]

diff --git a/build/build.list b/build/build.list
deleted file mode 100644 (file)
index 502460b..0000000
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
-debian8,debian9,centos7|python-gflags|2.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|google-api-python-client|1.6.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
-debian8,debian9,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rsa|3.4.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|uritemplate|3.0.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|httplib2|0.9.2|3|python|all
-debian8,debian9,centos7,ubuntu1404,ubuntu1604|ws4py|0.4.2|2|python|all
-debian8,debian9,centos7|pykka|1.2.1|2|python|all
-debian8,debian9,ubuntu1404,centos7|six|1.10.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|ciso8601|1.0.6|3|python|amd64
-debian8,debian9,centos7|pycrypto|2.6.1|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804|backports.ssl_match_hostname|3.5.0.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|llfuse|1.2|3|python|amd64
-debian8,debian9,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pyyaml|3.12|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rdflib|4.2.2|2|python|all
-debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|mistune|0.8.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|typing|3.6.4|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|avro|1.8.1|2|python|all
-debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|cachecontrol|0.11.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pathlib2|2.3.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|scandir|1.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|python3|all
-debian8,debian9,centos7|six|1.10.0|2|python3|all
-debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
-debian8,ubuntu1404,centos7|requests|2.6.1|2|python|all
-centos7|contextlib2|0.5.4|2|python|all
-centos7|isodate|0.5.4|2|python|all
-centos7|python-daemon|2.1.2|1|python|all
-centos7|pbr|0.11.1|2|python|all
-centos7|pyparsing|2.1.10|2|python|all
-centos7|keepalive|0.5|2|python|all
-centos7|networkx|1.11|0|python|all
-centos7|psutil|5.0.1|0|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.15.77|1|python|amd64|--python-setup-py-arguments --single-version-externally-managed --depends 'python-ruamel.ordereddict >= 0.4.9'
-all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
-all|junit-xml|1.8|3|python|all
-all|rdflib-jsonld|0.4.0|2|python|all
-all|futures|3.0.5|2|python|all
-all|future|0.16.0|2|python|all
-all|future|0.16.0|2|python3|all
-all|mypy-extensions|0.3.0|1|python|all
-all|prov|1.5.1|0|python|all
-all|bagit|1.6.4|0|python|all
-all|typing-extensions|3.6.5|0|python|all
index 23400cf48fdc8afad4fc3433c11a1fde4d96b1fa..9bba1f161c5449edc6f4071940d39f7a0b899b9c 100644 (file)
@@ -29,8 +29,8 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
 # Need to "touch" RPM database to workaround bug in interaction between
 # overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install python33
-RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip
+RUN touch /var/lib/rpm/* && yum -q -y install rh-python35
+RUN scl enable rh-python35 "easy_install-3.5 pip" && easy_install-2.7 pip
 
 # Old versions of setuptools cannot build a schema-salad package.
 RUN pip install --upgrade setuptools
@@ -38,4 +38,4 @@ RUN pip install --upgrade setuptools
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
-CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
+CMD ["scl", "enable", "rh-python35", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
index 493aef7befc6925d5f282b5747f2b1d86b81456d..1e6c62ce7ea72fba1380260bb2c7c07dbc19c87c 100644 (file)
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install dependencies.
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
 
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
 # Install RVM
 ADD generated/mpapis.asc /tmp/
 ADD generated/pkuczynski.asc /tmp/
index 98da1c97ad3c611ec7278da8eda5023450580923..fcaf0bc0a1bb74cb134847eed3cbde9fc14f0c80 100644 (file)
@@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
 
 # Install virtualenv
-RUN /usr/bin/pip install virtualenv virtualenv-tools
+RUN /usr/bin/pip install virtualenv
 
 # Install RVM
 ADD generated/mpapis.asc /tmp/
index 8b0d9bc65820a5ef69d1fd17c37e9ed75e875313..a88435a5b4bb0e98fd646f7ede5e03a45c5490bd 100644 (file)
@@ -8,7 +8,10 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3.4-venv python3.4-dev
+
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
 
 # Install RVM
 ADD generated/mpapis.asc /tmp/
index de15c9ecdd7f862533da2486ef8e47942153855c..be3911eff3e94ee6907efc883a8da047a4622ae4 100644 (file)
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install dependencies.
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
 
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
 # Install RVM
 ADD generated/mpapis.asc /tmp/
 ADD generated/pkuczynski.asc /tmp/
index 0dc431373cad4cd72b92ba4c8a35519f6a218e11..8995d14be0bdc55cbdeb47c005c2323f648b6bc4 100644 (file)
@@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install dependencies.
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
 
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv
+
 # Install RVM
 ADD generated/mpapis.asc /tmp/
 ADD generated/pkuczynski.asc /tmp/
index b5325224ee170bbc6170babfa8c2b95665d2057a..77017ba9702cb870ed83cb982fe3b135a779e6e6 100755 (executable)
@@ -2,9 +2,19 @@
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: AGPL-3.0
-
 set -eu
 
+# Set up
+DEBUG=${ARVADOS_DEBUG:-0}
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQQ_UNLESS_DEBUG=-qq
+if [[ "$DEBUG" != "0" ]]; then
+  STDOUT_IF_DEBUG=/dev/stdout
+  STDERR_IF_DEBUG=/dev/stderr
+  DASHQQ_UNLESS_DEBUG=
+fi
+
 # Multiple .deb based distros symlink to this script, so extract the target
 # from the invocation path.
 target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
@@ -13,8 +23,9 @@ export ARV_PACKAGES_DIR="/arvados/packages/$target"
 
 dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
 
-apt-get -qq update
-apt-get --assume-yes --allow-unauthenticated install "$1"
+apt-get $DASHQQ_UNLESS_DEBUG update
+
+apt-get $DASHQQ_UNLESS_DEBUG -y --allow-unauthenticated install "$1" >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
 
 dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
 
@@ -35,12 +46,14 @@ fi
 
 dpkg-deb -x $debpkg .
 
-while read so && [ -n "$so" ]; do
-    echo
-    echo "== Packages dependencies for $so =="
-    ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
-done <<EOF
+if [[ "$DEBUG" != "0" ]]; then
+  while read so && [ -n "$so" ]; do
+      echo
+      echo "== Packages dependencies for $so =="
+      ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+  done <<EOF
 $(find -name '*.so')
 EOF
+fi
 
 exec /jenkins/package-testing/common-test-packages.sh "$1"
index 268611cb7f50eba879e84c2e10f766b8ea4b6ff5..12450dd4f954acf65a58fe637880697be5918861 100755 (executable)
@@ -5,6 +5,15 @@
 
 set -eu
 
+# Set up
+DEBUG=${ARVADOS_DEBUG:-0}
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+if [[ "$DEBUG" != "0" ]]; then
+  STDOUT_IF_DEBUG=/dev/stdout
+  STDERR_IF_DEBUG=/dev/stderr
+fi
+
 target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b')
 
 yum -q clean all
@@ -14,7 +23,7 @@ export ARV_PACKAGES_DIR="/arvados/packages/$target"
 
 rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
 
-yum install --assumeyes $1
+yum install --assumeyes -e 0 $1
 
 rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
 
@@ -41,10 +50,12 @@ cd /tmp/opts
 
 rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
 
-find -name '*.so' | while read so; do
-    echo -e "\n== Packages dependencies for $so =="
-    ldd "$so" \
-        | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
-done
+if [[ "$DEBUG" != "0" ]]; then
+  find -name '*.so' | while read so; do
+      echo -e "\n== Packages dependencies for $so =="
+      ldd "$so" \
+          | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+  done
+fi
 
 exec /jenkins/package-testing/common-test-packages.sh "$1"
index c699fb45b54ae7ff7f89a8567aa4536147ef0bd8..e2ada8ed62c492ac45517931ad37eb187f0affcb 100755 (executable)
@@ -3,9 +3,20 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-exec python <<EOF
+set -e
+arvados-node-manager --version
+set +e
+
+PYTHON=`ls /usr/share/python2.7/dist/arvados-node-manager/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+  set -e
+  exec $PYTHON <<EOF
 import libcloud.compute.types
 import libcloud.compute.providers
 libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
 print "Successfully imported compatible libcloud library"
 EOF
+else
+       exit 1
+fi
index e499238d89eb2572af6beb6f9d9a05bce1dd8b31..02da021aff8041e5d2499b66093f71a9d998d6e4 100755 (executable)
@@ -4,10 +4,17 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 set -e
-
 arvados-cwl-runner --version
+set +e
+
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-cwl-runner/bin/python2.7`
 
-exec python <<EOF
+if [ "$PYTHON" != "" ]; then
+  set -e
+  exec $PYTHON <<EOF
 import arvados_cwl
 print "arvados-cwl-runner version", arvados_cwl.__version__
 EOF
+else
+       exit 1
+fi
index 152d1eb6979a520eb56fcb60f498752cd4847eb9..8d21951ff5c16af2a9d7f80188f49846acf55d92 100755 (executable)
@@ -3,7 +3,14 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-exec python <<EOF
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-fuse/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+  set -e
+  exec $PYTHON <<EOF
 import arvados_fuse
 print "Successfully imported arvados_fuse"
 EOF
+else
+       exit 1
+fi
index 2603e034aad3cd35b79c54ee3e53d4eedb569df6..3e737ef2bf3ef962852a3838837d5439bfd558ed 100755 (executable)
@@ -3,7 +3,14 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-exec python2.7 <<EOF
+PYTHON=`ls /usr/share/python2.7/dist/python-arvados-python-client/bin/python2.7`
+
+if [ "$PYTHON" != "" ]; then
+  set -e
+  exec $PYTHON <<EOF
 import arvados
 print "Successfully imported arvados"
 EOF
+else
+       exit 1
+fi
index efc20b234df4ec25b0f1eb5483eb4a6317624672..04b4a7956a58d5db9bdeae5188c958030a24ae7c 100755 (executable)
@@ -117,7 +117,7 @@ case "$TARGET" in
         PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
-        PYTHON3_PREFIX=/opt/rh/python33/root/usr
+        PYTHON3_PREFIX=/opt/rh/rh-python35/root/usr
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
         export PYCURL_SSL_LIBRARY=nss
         ;;
@@ -327,27 +327,6 @@ package_go_binary tools/keep-rsync keep-rsync \
 package_go_binary tools/keep-exercise keep-exercise \
     "Performance testing tool for Arvados Keep"
 
-
-# we need explicit debian_revision values in the dependencies for ruamel.yaml, because we have a package iteration
-# greater than zero. So we parse setup.py, get the ruamel.yaml dependencies, tell fpm not to automatically include
-# them in the package being built, and re-add them manually with an appropriate debian_revision value.
-# See #14552 for the reason for this (nasty) workaround. We use ${ruamel_depends[@]} in a few places further down
-# in this script.
-# Ward, 2018-11-28
-IFS=', ' read -r -a deps <<< `grep ruamel.yaml $WORKSPACE/sdk/python/setup.py |cut -f 3 -dl |sed -e "s/'//g"`
-declare -a ruamel_depends=()
-for i in ${deps[@]}; do
-  i=`echo "$i" | sed -e 's!\([0-9]\)! \1!'`
-  if [[ $i =~ .*\>.* ]]; then
-    ruamel_depends+=(--depends "python-ruamel.yaml $i-1")
-  elif [[ $i =~ .*\<.* ]]; then
-    ruamel_depends+=(--depends "python-ruamel.yaml $i-9")
-  else
-    echo "Encountered ruamel dependency that I can't parse. Aborting..."
-    exit 1
-  fi
-done
-
 # The Python SDK
 fpm_build_virtualenv "arvados-python-client" "sdk/python"
 fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
@@ -355,34 +334,6 @@ fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
 # Arvadow cwl runner
 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
 
-# schema_salad. This is a python dependency of arvados-cwl-runner,
-# but we can't use the usual PYTHONPACKAGES way to build this package due to the
-# intricacies of how version numbers get generated in setup.py: we need a specific version,
-# e.g. 1.7.20160316203940. If we don't explicitly list that version with the -v
-# argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get
-# a package with version 1.7. That's because our gittagger hack is not being
-# picked up by self.distribution.get_version(), which is called from
-# https://github.com/jordansissel/fpm/blob/master/lib/fpm/package/pyfpm/get_metadata.py
-# by means of this command:
-#
-# python2.7 setup.py --command-packages=pyfpm get_metadata --output=metadata.json
-#
-# So we build this thing separately.
-#
-# Ward, 2016-03-17
-saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python 2
-if [[ "$?" == "0" ]]; then
-  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
-fi
-
-# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python 3
-if [[ "$?" == "0" ]]; then
-  fpm_build cwltool "" "" python $cwltoolversion --iteration 3 --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
-fi
-
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
     cd $WORKSPACE/packages/$TARGET
@@ -398,34 +349,13 @@ fi
 fpm_build_virtualenv "arvados-fuse" "services/fuse"
 
 # The node manager
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/nodemanager/build"
-nodemanager_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-1}"
-test_package_presence arvados-node-manager "$nodemanager_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration "$iteration"
-fi
+fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
 
 # The Docker image cleaner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/dockercleaner/build"
-dockercleaner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-4}"
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
-fi
+fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
 
 # The Arvados crunchstat-summary tool
-cd $WORKSPACE/packages/$TARGET
-crunchstat_summary_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)}
-iteration="${ARVADOS_BUILDING_ITERATION:-2}"
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
-if [[ "$?" == "0" ]]; then
-  rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
-fi
+fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary"
 
 # Forked libcloud
 if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
@@ -452,112 +382,6 @@ then
   rm -rf $LIBCLOUD_DIR
 fi
 
-# Python 2 dependencies
-declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
-# Add --no-use-wheel if this pip knows it.
-pip install --no-use-wheel >/dev/null 2>&1
-case "$?" in
-    0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
-    1) ;;
-    2) ;;
-    *) echo "WARNING: 'pip install --no-use-wheel' test returned unknown exit code $?" ;;
-esac
-
-while read -r line || [[ -n "$line" ]]; do
-#  echo "Text read from file: $line"
-  if [[ "$line" =~ ^# ]]; then
-    continue
-  fi
-  IFS='|'; arr=($line); unset IFS
-
-  dist=${arr[0]}
-
-  IFS=',';dists=($dist); unset IFS
-
-  MATCH=0
-  for d in "${dists[@]}"; do
-    if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
-      MATCH=1
-    fi
-  done
-
-  if [[ "$MATCH" != "1" ]]; then
-    continue
-  fi
-  name=${arr[1]}
-  version=${arr[2]}
-  iteration=${arr[3]}
-  pkgtype=${arr[4]}
-  arch=${arr[5]}
-  extra=${arr[6]}
-  declare -a 'extra_arr=('"$extra"')'
-
-  if [[ "$FORMAT" == "rpm" ]]; then
-    if [[ "$arch" == "all" ]]; then
-      arch="noarch"
-    fi
-    if [[ "$arch" == "amd64" ]]; then
-      arch="x86_64"
-    fi
-  fi
-
-  if [[ "$pkgtype" == "python" ]]; then
-    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
-  else
-    outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
-  fi
-
-  if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
-      continue
-  fi
-
-  case "$name" in
-      httplib2|google-api-python-client)
-          test_package_presence $outname $version $pkgtype $iteration $arch
-          if [[ "$?" == "0" ]]; then
-            # Work around 0640 permissions on some package files.
-            # See #7591 and #7991.
-            pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
-                set -e
-                cd "$pyfpm_workdir"
-                PIP_VERSION=`python$PYTHON2_VERSION -c "import pip; print(pip.__version__)" |cut -f1 -d.`
-                if (( $PIP_VERSION < 8 )); then
-                  pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
-                else
-                  pip download --no-deps --no-binary :all: "$name==$version"
-                fi
-                # Sometimes pip gives us a tarball, sometimes a zip file...
-                DOWNLOADED=`ls $name-*`
-                [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
-                [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
-                cd "$name"-*/
-                "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
-                chmod -R go+rX .
-                set +e
-                fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
-                # The upload step uses the package timestamp to determine
-                # if it is new.  --no-clobber plays nice with that.
-                mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
-            )
-            if [ 0 != "$?" ]; then
-                echo "ERROR: $name build process failed"
-                EXITCODE=1
-            fi
-            if [ -n "$pyfpm_workdir" ]; then
-                rm -rf "$pyfpm_workdir"
-            fi
-          fi
-          ;;
-      *)
-          test_package_presence $outname $version $pkgtype $iteration $arch
-          if [[ "$?" == "0" ]]; then
-            fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
-          fi
-          ;;
-  esac
-
-done <`dirname "$(readlink -f "$0")"`"/build.list"
-
 # Build the API server package
 test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
 if [[ "$?" == "0" ]]; then
index f079be24dc73beefbdd3aacbaef1518757c6b84d..16cb0bc1eff10689241f0901447080b3e2c334d3 100755 (executable)
@@ -623,8 +623,7 @@ fpm_build_virtualenv () {
   FPM_RESULTS=$("${COMMAND_ARR[@]}")
   FPM_EXIT_CODE=$?
 
-# FIXME
-#  fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+  fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
 
   # if something went wrong and debug is off, print out the fpm command that errored
   if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
diff --git a/sdk/cwl/fpm-info.sh b/sdk/cwl/fpm-info.sh
new file mode 100644 (file)
index 0000000..5bfee91
--- /dev/null
@@ -0,0 +1,12 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+case "$TARGET" in
+    debian8)
+        fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
+        ;;
+    debian* | ubuntu*)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
+esac
diff --git a/sdk/python/fpm-info.sh b/sdk/python/fpm-info.sh
new file mode 100644 (file)
index 0000000..7a89cf0
--- /dev/null
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+case "$TARGET" in
+    debian* | ubuntu*)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
+esac
index 6cf69fb782ee5ce876fe9ae6a64a6c7dbce3239d..f789abe69270c024e73a5294666bc06169b45026 100644 (file)
@@ -3,3 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 fpm_depends+=(fuse)
+
+case "$TARGET" in
+    centos*)
+        fpm_depends+=(fuse-libs)
+        ;;
+    debian* | ubuntu*)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
+esac
diff --git a/services/nodemanager/fpm-info.sh b/services/nodemanager/fpm-info.sh
new file mode 100644 (file)
index 0000000..0abc6a0
--- /dev/null
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+    debian* | ubuntu*)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
+esac
diff --git a/tools/crunchstat-summary/fpm-info.sh b/tools/crunchstat-summary/fpm-info.sh
new file mode 100644 (file)
index 0000000..0abc6a0
--- /dev/null
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+    debian* | ubuntu*)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
+esac