Build api server and workbench packages with vendor/bundle included
--debug
Output debug information (default: false)
---format
- Package format (default: deb)
+--target
+ Distribution to build packages for (default: debian7)
WORKSPACE=path Path to the Arvados source tree to build packages from
DEBUG=0
UPLOAD=0
BUILD_BUNDLE_PACKAGES=0
-FORMAT=deb
+TARGET=debian7
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,format: \
+ help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--scp-user)
SCPUSER="$2"; shift
;;
- --scp-host|--apt-server)
+ --scp-host)
SCPHOST="$2"; shift
;;
- --format)
- FORMAT="$2"; shift
+ --target)
+ TARGET="$2"; shift
;;
--debug)
DEBUG=1
exit 1
fi
-# Sanity check
+declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
+
+PYTHON2_VERSION=2.7
+PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+
+case "$TARGET" in
+ debian7)
+ FORMAT=deb
+ FPM_OUTDIR=tmp
+ REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl)
+ PYTHON3_BACKPORTS=(docker-py six requests)
+ ;;
+ centos6)
+ FORMAT=rpm
+ FPM_OUTDIR=rpm
+ REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/'
+
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl
+ python-daemon lockfile llfuse)
+ PYTHON3_BACKPORTS=(docker-py six requests)
+ export PYCURL_SSL_LIBRARY=nss
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+
if ! [[ -n "$WORKSPACE" ]]; then
echo >&2 "$helpmessage"
echo >&2
exit 1
fi
-if ! easy_install3 --version >/dev/null; then
- cat >&2 <<EOF
+find_easy_install() {
+ for version_suffix in "$@"; do
+ if "easy_install$version_suffix" --version >/dev/null 2>&1; then
+ echo "easy_install$version_suffix"
+ return 0
+ fi
+ done
+ cat >&2 <<EOF
$helpmessage
-Error: easy_install3 (from python3-setuptools) not found
+Error: easy_install$1 (from Python setuptools module) not found
EOF
- exit 1
-fi
+ exit 1
+}
+
+EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
+EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
VERSION=$1
shift
- # fpm does not actually support a python3 package type. Instead we recognize
- # it as a convenience shortcut to add several necessary arguments to
- # fpm's command line later, after we're done handling positional arguments.
- if [ "python3" = "$PACKAGE_TYPE" ]; then
- PACKAGE_TYPE=python
- set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
- --python-package-name-prefix python3 --depends python3
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ set -- "$@" --python-bin python2.7 \
+ --python-easyinstall "$EASY_INSTALL2" \
+ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --depends "$PYTHON2_PACKAGE"
+ ;;
+ python3)
+ # fpm does not actually support a python3 package type. Instead
+ # we recognize it as a convenience shortcut to add several
+ # necessary arguments to fpm's command line later, after we're
+ # done handling positional arguments.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 \
+ --python-easyinstall "$EASY_INSTALL3" \
+ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --depends "$PYTHON3_PACKAGE"
+ ;;
+ esac
+
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ if [ python = "$PACKAGE_TYPE" ]; then
+ COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
fi
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests")
-
if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fi
echo "Error building package for $1:\n $FPM_RESULTS"
else
if [[ "$UPLOAD" != 0 ]]; then
- if [[ "$FORMAT" == 'deb' ]]; then
- scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:tmp/
- else
- scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:rpm/
- fi
+ scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/
CALL_FREIGHT=1
fi
fi
echo "umask is" `umask`
fi
+if [[ ! -d "$WORKSPACE/debs" ]]; then
+ mkdir -p $WORKSPACE/debs
+fi
+
# Perl packages
if [[ "$DEBUG" != 0 ]]; then
echo -e "\nPerl packages\n"
if [[ -e Makefile ]]; then
make realclean >"$PERL_OUT"
fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name 'libarvados-perl_*.deb' \) \
+find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-delete
rm -rf install
-perl Makefile.PL >"$PERL_OUT" && \
- make install PREFIX=install INSTALLDIRS=perl >"$PERL_OUT" && \
- fpm_build_and_scp install/=/usr libarvados-perl "Curoverse, Inc." dir \
- "$(version_from_git)"
+perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \
+ make install INSTALLDIRS=perl >"$PERL_OUT" && \
+ fpm_build_and_scp install/lib/=/usr/share libarvados-perl \
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \
+ mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/"
# Ruby gems
if [[ "$DEBUG" != 0 ]]; then
cd ../../services/nodemanager
handle_python_package
-if [[ ! -d "$WORKSPACE/debs" ]]; then
- mkdir -p $WORKSPACE/debs
-fi
-
# Arvados-src
# We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package
if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then
# whip up a patch and send it upstream, but that will be for another day. Ward,
# 2014-05-15
cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK"
+fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK"
# The FUSE driver
# Please see comment about --no-python-fix-name above; we stay consistent and do
# not omit the python- prefix first.
cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
# The node manager
cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
# The Docker image cleaner
cd $WORKSPACE/debs
fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
# A few dependencies
-for deppkg in python-gflags pyvcf google-api-python-client oauth2client \
- pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py virtualenv \
- pykka apache-libcloud requests six pyexecjs jsonschema ciso8601 \
- pycrypto backports.ssl_match_hostname; do
- fpm_build_and_scp "$deppkg"
+for deppkg in "${PYTHON_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ fpm_build_and_scp "$deppkg" "$outname"
done
# Python 3 dependencies
-for deppkg in docker-py six requests; do
+for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
# The empty string is the vendor argument: these aren't Curoverse software.
- fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3
+ fpm_build_and_scp "$deppkg" "$outname" "" python3
done
# Build the API server package
# Finally, publish the packages, if necessary
if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then
- if [[ "$FORMAT" == 'deb' ]]; then
- ssh -p2222 $SCPUSER@$SCPHOST -t "cd tmp && ls -laF *deb && freight add *deb apt/wheezy && freight cache && rm -f *deb"
- else
- ssh -p2222 $SCPUSER@$SCPHOST -t "cd rpm && ls -laF *rpm && mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/"
- fi
+ ssh -p2222 $SCPUSER@$SCPHOST -t bash - <<EOF
+if [ -n "\$(find -name "$FPM_OUTDIR/*.$FORMAT" -print -quit)" ]; then
+ cd "$FPM_OUTDIR" && $REPO_UPDATE_CMD
+fi
+EOF
else
if [[ "$UPLOAD" != 0 ]]; then
echo "No new packages generated. No freight run necessary."
projects / arvados-dev.git / blobdiff