TARGET=debian7
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,target: \
+ help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--scp-user)
SCPUSER="$2"; shift
;;
- --scp-host|--apt-server)
+ --scp-host)
SCPHOST="$2"; shift
;;
--target)
declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
+PYTHON2_VERSION=2.7
+PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+
case "$TARGET" in
debian7)
FORMAT=deb
FPM_OUTDIR=tmp
- REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb'
+ REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
- oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl)
- PYTHON3_BACKPORTS=(docker-py six requests)
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ debian8)
+ FORMAT=deb
+ FPM_OUTDIR=tmp
+ REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ FPM_OUTDIR=tmp
+ REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb'
+
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
;;
centos6)
FORMAT=rpm
FPM_OUTDIR=rpm
REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/'
+
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
- oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
- virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl)
+ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py \
+ pykka apache-libcloud requests six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname pycurl
+ python-daemon lockfile llfuse)
PYTHON3_BACKPORTS=(docker-py six requests)
+ export PYCURL_SSL_LIBRARY=nss
;;
*)
echo -e "$0: Unknown target '$TARGET'.\n" >&2
exit 1
fi
-if ! easy_install3 --version >/dev/null; then
- cat >&2 <<EOF
+find_easy_install() {
+ for version_suffix in "$@"; do
+ if "easy_install$version_suffix" --version >/dev/null 2>&1; then
+ echo "easy_install$version_suffix"
+ return 0
+ fi
+ done
+ cat >&2 <<EOF
$helpmessage
-Error: easy_install3 (from python3-setuptools) not found
+Error: easy_install$1 (from Python setuptools module) not found
EOF
- exit 1
-fi
+ exit 1
+}
+
+EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
+EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
# Generates a version number from the git log for the current working
# directory, and writes it to stdout.
local git_ts git_hash
- declare $(TZ=UTC git log -n1 --first-parent --max-count=1 \
+ declare $(TZ=UTC git log -n1 --first-parent \
--format=format:"git_ts=%ct git_hash=%h" .)
echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
# Generates a version number from the git log for the current working
# directory, and writes it to stdout.
local git_ts git_hash
- declare $(TZ=UTC git log -n1 --first-parent --max-count=1 \
+ declare $(TZ=UTC git log -n1 --first-parent \
--format=format:"git_ts=%ct git_hash=%h" .)
echo "$git_ts"
}
fi
}
-# Build debs for everything
+# Build packages for everything
fpm_build_and_scp () {
# The package source. Depending on the source type, this can be a
# path, or the name of the package in an upstream repository (e.g.,
# All Arvados Python2 packages depend on Python 2.7.
# Make sure we build with that for consistency.
set -- "$@" --python-bin python2.7 \
- --python-easyinstall easy_install-2.7
+ --python-easyinstall "$EASY_INSTALL2" \
+ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --depends "$PYTHON2_PACKAGE"
;;
python3)
# fpm does not actually support a python3 package type. Instead
# necessary arguments to fpm's command line later, after we're
# done handling positional arguments.
PACKAGE_TYPE=python
- set -- "$@" --python-bin python3 --python-easyinstall easy_install3 \
- --python-package-name-prefix python3 --depends python3
+ set -- "$@" --python-bin python3 \
+ --python-easyinstall "$EASY_INSTALL3" \
+ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --depends "$PYTHON3_PACKAGE"
;;
esac
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests")
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ if [ python = "$PACKAGE_TYPE" ]; then
+ COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
+ fi
if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
}
-# verify build results and scp debs, if needed
+# verify build results and scp packages, if needed
fpm_verify_and_scp () {
FPM_EXIT_CODE=$1
shift
FPM_RESULTS=$@
FPM_PACKAGE_NAME=''
- if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then
+ if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
fi
echo "umask is" `umask`
fi
-if [[ ! -d "$WORKSPACE/debs" ]]; then
- mkdir -p $WORKSPACE/debs
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p $WORKSPACE/packages/$TARGET
fi
# Perl packages
make install INSTALLDIRS=perl >"$PERL_OUT" && \
fpm_build_and_scp install/lib/=/usr/share libarvados-perl \
"Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \
- mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/"
+ mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
# Ruby gems
if [[ "$DEBUG" != 0 ]]; then
git checkout master
git pull
# go into detached-head state
- git checkout `git log --format=format:%h -n1 .`
+ git checkout `git log --format=format:%h -n1 --first-parent .`
else
git checkout -q master
git pull -q
# go into detached-head state
- git checkout -q `git log --format=format:%h -n1 .`
+ git checkout -q `git log --format=format:%h -n1 --first-parent .`
fi
-git log --format=format:%H -n1 . > git-commit.version
+git log --format=format:%H -n1 --first-parent . > git-commit.version
# Build arvados src deb package
cd "$WORKSPACE"
PKG_VERSION=$(version_from_git)
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
# clean up, check out master and step away from detached-head state
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore"
PKG_VERSION=$(version_from_git)
go get "git.curoverse.com/arvados.git/services/keepstore"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN"
# Get GO SDK version
fi
go get "git.curoverse.com/arvados.git/services/keepproxy"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
# datamanager
fi
go get "git.curoverse.com/arvados.git/services/datamanager"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
# arv-git-httpd
fi
go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
# crunchstat
cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
PKG_VERSION=$(version_from_git)
go get "git.curoverse.com/arvados.git/services/crunchstat"
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs"
# The Python SDK
# prefix from only one of the dependencies of a package... Maybe I could
# whip up a patch and send it upstream, but that will be for another day. Ward,
# 2014-05-15
-cd $WORKSPACE/debs
-fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends=python2.7
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK"
# The FUSE driver
# Please see comment about --no-python-fix-name above; we stay consistent and do
# not omit the python- prefix first.
-cd $WORKSPACE/debs
-fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends=python2.7
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
# The node manager
-cd $WORKSPACE/debs
-fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends=python2.7
+cd $WORKSPACE/packages/$TARGET
+fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
# The Docker image cleaner
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
# A few dependencies
for deppkg in "${PYTHON_BACKPORTS[@]}"; do
- fpm_build_and_scp "$deppkg"
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ fpm_build_and_scp "$deppkg" "$outname"
done
# Python 3 dependencies
for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
# The empty string is the vendor argument: these aren't Curoverse software.
- fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3
+ fpm_build_and_scp "$deppkg" "$outname" "" python3
done
+# libpam-arvados
+cd $WORKSPACE/sdk/pam
+PKG_VERSION=$(version_from_git)
+cd $WORKSPACE/packages/$TARGET
+
+if [[ "$FORMAT" == "deb" ]]; then
+ fpm_build_and_scp $WORKSPACE/sdk/pam/debian/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/debian/arvados_pam=/etc/default/arvados_pam
+#else
+ # FIXME enable and test once we have the centos pam.d file
+ #fpm_build_and_scp $WORKSPACE/sdk/pam/centos/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/centos/arvados_pam=/etc/default/arvados_pam
+fi
+
# Build the API server package
cd "$WORKSPACE/services/api"
/usr/bin/git rev-parse HEAD > git-commit.version
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
# Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now,
# we do that in the upgrade script.
# from the package - empty it instead.
rm -rf $WORKSPACE/apps/workbench/tmp/*
-# Set up application.yml so that asset precompilation works
+# Set up application.yml and production.rb so that asset precompilation works
\cp config/application.yml.example config/application.yml -f
+\cp config/environments/production.rb.example config/environments/production.rb -f
sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
EXITCODE=1
fi
-cd $WORKSPACE/debs
+cd $WORKSPACE/packages/$TARGET
# This is the complete package with vendor/bundle included.
# It's big, so we do not build it by default.
# Finally, publish the packages, if necessary
if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then
ssh -p2222 $SCPUSER@$SCPHOST -t bash - <<EOF
-if [ -n "\$(find -name "$FPM_OUTDIR/*.$FORMAT" -print -quit)" ]; then
+if [ -n "\$(find $FPM_OUTDIR -name "*.$FORMAT" -print -quit)" ]; then
cd "$FPM_OUTDIR" && $REPO_UPDATE_CMD
fi
EOF
projects / arvados-dev.git / blobdiff