#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
. `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin
+. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados packages
case "$TARGET" in
debian8)
FORMAT=deb
- PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six \
- ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
- 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
- shellescape mistune typing avro ruamel.ordereddict
- cachecontrol 'pathlib2>=2.1.0')
- PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
;;
ubuntu1204)
FORMAT=deb
- PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six \
- ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
- contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
- shellescape mistune typing avro isodate ruamel.ordereddict
- cachecontrol requests 'pathlib2>=2.1.0')
- PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
;;
ubuntu1404)
FORMAT=deb
- PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
- google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
- rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
- shellescape mistune typing avro ruamel.ordereddict
- cachecontrol 'pathlib2>=2.1.0')
- PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
;;
- centos6)
- FORMAT=rpm
- PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
- PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
- PYTHON2_PREFIX=/opt/rh/python27/root/usr
- PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
- PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
- PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
- PYTHON3_PREFIX=/opt/rh/python33/root/usr
- PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
- PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six \
- ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
- python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
- 'rdflib>=4.2.0' shellescape mistune typing avro requests \
- isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
- PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
- export PYCURL_SSL_LIBRARY=nss
+ ubuntu1604)
+ FORMAT=deb
;;
centos7)
FORMAT=rpm
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
PYTHON3_PREFIX=/opt/rh/python33/root/usr
PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
- PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
- oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka \
- ciso8601 pycrypto 'pycurl<7.21.5' \
- python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml \
- 'rdflib>=4.2.0' shellescape mistune typing avro \
- isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
- PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
export PYCURL_SSL_LIBRARY=nss
;;
*)
debug_echo -e "\nPerl packages\n"
if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
-cd "$WORKSPACE/sdk/perl"
+ cd "$WORKSPACE/sdk/perl"
+ libarvados_perl_version="$(version_from_git)"
-if [[ -e Makefile ]]; then
- make realclean >"$STDOUT_IF_DEBUG"
-fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
- -delete
-rm -rf install
-
-perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
- make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
- fpm_build install/lib/=/usr/share libarvados-perl \
- "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
- "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
- mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence libarvados-perl "$libarvados_perl_version"
+
+ if [[ "$?" == "0" ]]; then
+ cd "$WORKSPACE/sdk/perl"
+
+ if [[ -e Makefile ]]; then
+ make realclean >"$STDOUT_IF_DEBUG"
+ fi
+ find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+ -delete
+ rm -rf install
+
+ perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+ fpm_build install/lib/=/usr/share libarvados-perl \
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+ "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+ fi
fi
# Ruby gems
# arvados-src
(
- set -e
-
cd "$WORKSPACE"
COMMIT_HASH=$(format_last_commit_here "%H")
+ arvados_src_version="$(version_from_git)"
+
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence arvados-src $arvados_src_version src ""
- SRC_BUILD_DIR=$(mktemp -d)
- # mktemp creates the directory with 0700 permissions by default
- chmod 755 $SRC_BUILD_DIR
- git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
- cd "$SRC_BUILD_DIR"
+ if [[ "$?" == "0" ]]; then
+ cd "$WORKSPACE"
+ SRC_BUILD_DIR=$(mktemp -d)
+ # mktemp creates the directory with 0700 permissions by default
+ chmod 755 $SRC_BUILD_DIR
+ git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+ cd "$SRC_BUILD_DIR"
- # go into detached-head state
- git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
- echo "$COMMIT_HASH" >git-commit.version
+ # go into detached-head state
+ git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+ echo "$COMMIT_HASH" >git-commit.version
- cd "$SRC_BUILD_DIR"
- PKG_VERSION=$(version_from_git)
- cd $WORKSPACE/packages/$TARGET
- fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+ cd "$SRC_BUILD_DIR"
+ PKG_VERSION=$(version_from_git)
+ cd $WORKSPACE/packages/$TARGET
+ fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+ rm -rf "$SRC_BUILD_DIR"
- rm -rf "$SRC_BUILD_DIR"
+ fi
)
# On older platforms we need to publish a backport of libfuse >=2.9.2,
"$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
apt-get -y --no-install-recommends -f install
rm -rf $LIBFUSE_DIR
-elif [[ $TARGET =~ centos6 ]]; then
- # port fuse 2.9.2 to centos 6
- # install tools to build rpm from source
- yum install -y rpm-build redhat-rpm-config
- LIBFUSE_DIR=$(mktemp -d)
- (
- cd "$LIBFUSE_DIR"
- # download fuse 2.9.2 centos 7 source rpm
- file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
- (
- # modify source rpm spec to remove conflict on filesystem version
- mkdir -p /root/rpmbuild/SOURCES
- cd /root/rpmbuild/SOURCES
- rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
- perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
- )
- # build rpms from source
- rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
- rm -f fuse-2.9.2-6.el7.src.rpm
- # move built RPMs to LIBFUSE_DIR
- mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
- rm -rf /root/rpmbuild
- )
- fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
- fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
- fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
- yum install -y \
- "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
- "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
- "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
fi
# Go binaries
# 2014-05-15
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
+if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+fi
# cwl-runner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
-
-fpm_build lockfile "" "" python 0.12.2 --epoch 1
+arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
+arvados_cwl_runner_iteration=3
+test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
+if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
# but we can't use the usual PYTHONPACKAGES way to build this package due to the
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
-
-# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
-# Ward, 2016-05-26
-fpm_build ruamel.yaml "" "" python 0.13.7 --python-setup-py-arguments "--single-version-externally-managed"
-
-# Dependency of cwltool. Fpm doesn't produce a package with the correct version
-# number unless we build it explicitly
-fpm_build cwltest "" "" python 1.0.20160907111242
+test_package_presence python-schema-salad "$saladversion" python
+if [[ "$?" == "0" ]]; then
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-fpm_build cwltool "" "" python $cwltoolversion
-
-# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
-fpm_build rdflib-jsonld "" "" python 0.4.0
+test_package_presence python-cwltool "$cwltoolversion" python
+if [[ "$?" == "0" ]]; then
+ fpm_build cwltool "" "" python $cwltoolversion
+fi
# The PAM module
if [[ $TARGET =~ debian|ubuntu ]]; then
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/pam/build"
- fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+ libpam_arvados_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)
+ test_package_presence libpam-arvados "$libpam_arvados_version" python
+ if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$libpam_arvados_version" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+ fi
fi
# The FUSE driver
# not omit the python- prefix first.
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)
+test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
+if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
# The node manager
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)
+test_package_presence arvados-node-manager "$nodemanager_version" python
+if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
+fi
# The Docker image cleaner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
+dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
+dockercleaner_iteration=3
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+if [[ "$?" == "0" ]]; then
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+if [[ "$?" == "0" ]]; then
+ rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+fi
-# Forked libcloud
-LIBCLOUD_DIR=$(mktemp -d)
-(
- cd $LIBCLOUD_DIR
- git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
- git checkout apache-libcloud-$LIBCLOUD_PIN
- # libcloud is absurdly noisy without -q, so force -q here
- OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
- DASHQ_UNLESS_DEBUG=-q
- handle_python_package
- DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
-)
-fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
-rm -rf $LIBCLOUD_DIR
+if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
+ # Forked libcloud
+ LIBCLOUD_DIR=$(mktemp -d)
+ (
+ cd $LIBCLOUD_DIR
+ git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+ git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
+ # libcloud is absurdly noisy without -q, so force -q here
+ OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+ DASHQ_UNLESS_DEBUG=-q
+ handle_python_package
+ DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+ )
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ rm -rf $LIBCLOUD_DIR
+fi
# Python 2 dependencies
declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
*) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
esac
-for deppkg in "${PYTHON_BACKPORTS[@]}"; do
- outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+while read -r line || [[ -n "$line" ]]; do
+# echo "Text read from file: $line"
+ if [[ "$line" =~ ^# ]]; then
+ continue
+ fi
+ IFS='|'; arr=($line); unset IFS
- if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
- continue
- fi
+ dist=${arr[0]}
+
+ IFS=',';dists=($dist); unset IFS
- case "$deppkg" in
- httplib2|google-api-python-client)
+ MATCH=0
+ for d in "${dists[@]}"; do
+ if [[ "$d" == "$TARGET" ]] || [[ "$d" == "all" ]]; then
+ MATCH=1
+ fi
+ done
+
+ if [[ "$MATCH" != "1" ]]; then
+ continue
+ fi
+ name=${arr[1]}
+ version=${arr[2]}
+ iteration=${arr[3]}
+ pkgtype=${arr[4]}
+ arch=${arr[5]}
+ extra=${arr[6]}
+ declare -a 'extra_arr=('"$extra"')'
+
+ if [[ "$FORMAT" == "rpm" ]]; then
+ if [[ "$arch" == "all" ]]; then
+ arch="noarch"
+ fi
+ if [[ "$arch" == "amd64" ]]; then
+ arch="x86_64"
+ fi
+ fi
+
+ if [[ "$pkgtype" == "python" ]]; then
+ outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ else
+ outname=$(echo "$name" | sed -e 's/^python-//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+ fi
+
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
+ continue
+ fi
+
+ case "$name" in
+ httplib2|google-api-python-client)
+ test_package_presence $outname $version $pkgtype $iteration $arch
+ if [[ "$?" == "0" ]]; then
# Work around 0640 permissions on some package files.
# See #7591 and #7991.
pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
set -e
cd "$pyfpm_workdir"
- pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+ pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$name==$version"
# Sometimes pip gives us a tarball, sometimes a zip file...
- DOWNLOADED=`ls $deppkg-*`
+ DOWNLOADED=`ls $name-*`
[[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
[[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
- cd "$deppkg"-*/
+ cd "$name"-*/
"python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
chmod -R go+rX .
set +e
- fpm_build . "$outname" "" python "" --iteration 3
+ fpm_build . "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
# The upload step uses the package timestamp to determine
- # whether it's new. --no-clobber plays nice with that.
+ # if it is new. --no-clobber plays nice with that.
mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
)
if [ 0 != "$?" ]; then
- echo "ERROR: $deppkg build process failed"
+ echo "ERROR: $name build process failed"
EXITCODE=1
fi
if [ -n "$pyfpm_workdir" ]; then
rm -rf "$pyfpm_workdir"
fi
- ;;
- *)
- fpm_build "$deppkg" "$outname"
- ;;
- esac
-done
-
-# Python 3 dependencies
-for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
- outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
- # The empty string is the vendor argument: these aren't Curoverse software.
- fpm_build "$deppkg" "$outname" "" python3
-done
+ fi
+ ;;
+ *)
+ test_package_presence $outname $version $pkgtype $iteration $arch
+ if [[ "$?" == "0" ]]; then
+ fpm_build "$name" "$outname" "" "$pkgtype" "$version" --iteration "$iteration" "${extra_arr[@]}"
+ fi
+ ;;
+ esac
+
+done <`dirname "$(readlink -f "$0")"`"/build.list"
# Build the API server package
-handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
-
-# Build the workbench server package
-if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then
-(
- set -e
- cd "$WORKSPACE/apps/workbench"
-
- # We need to bundle to be ready even when we build a package without vendor directory
- # because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
-
- # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
- # and we want that in the package, so it's easier to not exclude the tmp directory
- # from the package - empty it instead.
- rm -rf tmp
- mkdir tmp
-
- # Set up application.yml and production.rb so that asset precompilation works
- \cp config/application.yml.example config/application.yml -f
- \cp config/environments/production.rb.example config/environments/production.rb -f
- sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
-
- RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
-
- # Remove generated configuration files so they don't go in the package.
- rm config/application.yml config/environments/production.rb
-)
-fi
-
-if [[ "$?" != "0" ]]; then
- echo "ERROR: Asset precompilation failed"
- EXITCODE=1
-else
- handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
+if [[ "$?" == "0" ]]; then
+ handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --description="Arvados API server - Arvados is a free and open source platform for big data science." \
--license="GNU Affero General Public License, version 3.0"
fi
+# Build the workbench server package
+test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+if [[ "$?" == "0" ]] ; then
+ (
+ set -e
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up application.yml and production.rb so that asset precompilation works
+ \cp config/application.yml.example config/application.yml -f
+ \cp config/environments/production.rb.example config/environments/production.rb -f
+ sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+ RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+ # Remove generated configuration files so they don't go in the package.
+ rm config/application.yml config/environments/production.rb
+ )
+
+ if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+ else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0"
+ fi
+fi
+
# clean up temporary GOPATH
rm -rf "$GOPATH"
projects / arvados.git / blobdiff