TARGET=debian7
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,target: \
+ help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--scp-user)
SCPUSER="$2"; shift
;;
- --scp-host|--apt-server)
+ --scp-host)
SCPHOST="$2"; shift
;;
--target)
declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
+PYTHON2_VERSION=2.7
+PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+
case "$TARGET" in
debian7)
FORMAT=deb
FPM_OUTDIR=tmp
- REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb'
+ REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
- oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname)
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ PYTHON3_BACKPORTS=(docker-py six requests)
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ FPM_OUTDIR=tmp
+ REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
PYTHON3_BACKPORTS=(docker-py six requests)
;;
centos6)
FORMAT=rpm
FPM_OUTDIR=rpm
REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/'
+
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
- oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
- virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname)
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl
+ python-daemon lockfile llfuse)
PYTHON3_BACKPORTS=(docker-py six requests)
+ export PYCURL_SSL_LIBRARY=nss
;;
*)
echo -e "$0: Unknown target '$TARGET'.\n" >&2
exit 1
fi
-if ! easy_install3 --version >/dev/null; then
- cat >&2 <<EOF
+find_easy_install() {
+ for version_suffix in "$@"; do
+ if "easy_install$version_suffix" --version >/dev/null 2>&1; then
+ echo "easy_install$version_suffix"
+ return 0
+ fi
+ done
+ cat >&2 <<EOF
$helpmessage
-Error: easy_install3 (from python3-setuptools) not found
+Error: easy_install$1 (from Python setuptools module) not found
EOF
- exit 1
-fi
+ exit 1
+}
+
+EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
+EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
fi
}
-# Build debs for everything
+# Build packages for everything
fpm_build_and_scp () {
# The package source. Depending on the source type, this can be a
# path, or the name of the package in an upstream repository (e.g.,
VERSION=$1
shift
- # fpm does not actually support a python3 package type. Instead we recognize
- # it as a convenience shortcut to add several necessary arguments to
- # fpm's command line later, after we're done handling positional arguments.
- if [ "python3" = "$PACKAGE_TYPE" ]; then
- PACKAGE_TYPE=python
- set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
- --python-package-name-prefix python3 --depends python3
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ set -- "$@" --python-bin python2.7 \
+ --python-easyinstall "$EASY_INSTALL2" \
+ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --depends "$PYTHON2_PACKAGE"
+ ;;
+ python3)
+ # fpm does not actually support a python3 package type. Instead
+ # we recognize it as a convenience shortcut to add several
+ # necessary arguments to fpm's command line later, after we're
+ # done handling positional arguments.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 \
+ --python-easyinstall "$EASY_INSTALL3" \
+ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --depends "$PYTHON3_PACKAGE"
+ ;;
+ esac
+
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ if [ python = "$PACKAGE_TYPE" ]; then
+ COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
fi
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests")
-
if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fi
fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
}
-# verify build results and scp debs, if needed
+# verify build results and scp packages, if needed
fpm_verify_and_scp () {
FPM_EXIT_CODE=$1
shift
FPM_RESULTS=$@
FPM_PACKAGE_NAME=''
- if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then
+ if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
fi
echo "umask is" `umask`
fi
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p $WORKSPACE/packages/$TARGET
+fi
+
# Perl packages
if [[ "$DEBUG" != 0 ]]; then
echo -e "\nPerl packages\n"
if [[ -e Makefile ]]; then
make realclean >"$PERL_OUT"
fi
-find -maxdepth 1 \( -name 'MANIFEST*' -or -name 'libarvados-perl_*.deb' \) \
+find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
-delete
rm -rf install
-perl Makefile.PL >"$PERL_OUT" && \
- make install PREFIX=install INSTALLDIRS=perl >"$PERL_OUT" && \
- fpm_build_and_scp install/=/usr libarvados-perl "Curoverse, Inc." dir \
- "$(version_from_git)"
+perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \
+ make install INSTALLDIRS=perl >"$PERL_OUT" && \
+ fpm_build_and_scp install/lib/=/usr/share libarvados-perl \
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \
+ mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
# Ruby gems
if [[ "$DEBUG" != 0 ]]; then
cd ../../services/nodemanager
handle_python_package
-if [[ ! -d "$WORKSPACE/debs" ]]; then
- mkdir -p $WORKSPACE/debs
-fi
-
# Arvados-src
# We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package
if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then
# Build arvados src deb package
cd "$WORKSPACE"
PKG_VERSION=$(version_from_git)
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
# clean up, check out master and step away from detached-head state
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore"
PKG_VERSION=$(version_from_git)
go get "git.curoverse.com/arvados.git/services/keepstore"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN"
# Get GO SDK version
fi
go get "git.curoverse.com/arvados.git/services/keepproxy"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
# datamanager
fi
go get "git.curoverse.com/arvados.git/services/datamanager"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
# arv-git-httpd
fi
go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
# crunchstat
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
PKG_VERSION=$(version_from_git)
go get "git.curoverse.com/arvados.git/services/crunchstat"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs"
# The Python SDK
# prefix from only one of the dependencies of a package... Maybe I could
# whip up a patch and send it upstream, but that will be for another day. Ward,
# 2014-05-15
-cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK"
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK"
# The FUSE driver
# Please see comment about --no-python-fix-name above; we stay consistent and do
# not omit the python- prefix first.
-cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
# The node manager
-cd $WORKSPACE/debs
-# Python version numbering is obscure. Strip dashes and replace them with dots
-# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440.
-fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
# The Docker image cleaner
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
# A few dependencies
for deppkg in "${PYTHON_BACKPORTS[@]}"; do
- fpm_build_and_scp "$deppkg"
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ fpm_build_and_scp "$deppkg" "$outname"
done
# Python 3 dependencies
for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
# The empty string is the vendor argument: these aren't Curoverse software.
- fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3
+ fpm_build_and_scp "$deppkg" "$outname" "" python3
done
# Build the API server package
/usr/bin/git rev-parse HEAD > git-commit.version
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
# Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now,
# we do that in the upgrade script.
EXITCODE=1
fi
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
# This is the complete package with vendor/bundle included.
# It's big, so we do not build it by default.
# Finally, publish the packages, if necessary
if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then
ssh -p2222 $SCPUSER@$SCPHOST -t bash - <<EOF
-if [ -n "\$(find -name "$FPM_OUTDIR/*.$FORMAT" -print -quit)" ]; then
+if [ -n "\$(find $FPM_OUTDIR -name "*.$FORMAT" -print -quit)" ]; then
cd "$FPM_OUTDIR" && $REPO_UPDATE_CMD
fi
EOF
projects / arvados-dev.git / blobdiff