X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/f0d1f9935eadd303d6b160ae9c349398f6691bd9..598d191de7ae6033ababaefcfcfc973e13ccdcbe:/jenkins/run-build-packages.sh?ds=sidebyside diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index e0584c4..37dc5a4 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -19,8 +19,8 @@ Options: Build api server and workbench packages with vendor/bundle included --debug Output debug information (default: false) ---format - Package format (default: deb) +--target + Distribution to build packages for (default: debian7) WORKSPACE=path Path to the Arvados source tree to build packages from @@ -32,10 +32,10 @@ CALL_FREIGHT=0 DEBUG=0 UPLOAD=0 BUILD_BUNDLE_PACKAGES=0 -FORMAT=deb +TARGET=debian7 PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,format: \ + help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -52,11 +52,11 @@ while [ $# -gt 0 ]; do --scp-user) SCPUSER="$2"; shift ;; - --scp-host|--apt-server) + --scp-host) SCPHOST="$2"; shift ;; - --format) - FORMAT="$2"; shift + --target) + TARGET="$2"; shift ;; --debug) DEBUG=1 @@ -86,7 +86,53 @@ if [[ "$UPLOAD" != '0' && ("$SCPUSER" == '' || "$SCPHOST" == '') ]]; then exit 1 fi -# Sanity check +declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS + +PYTHON2_VERSION=2.7 +PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') + +case "$TARGET" in + debian7) + FORMAT=deb + FPM_OUTDIR=tmp + REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl) + PYTHON3_BACKPORTS=(docker-py six requests) + ;; + centos6) + FORMAT=rpm + FPM_OUTDIR=rpm + REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' + + PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') + PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE + PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl + python-daemon lockfile llfuse) + PYTHON3_BACKPORTS=(docker-py six requests) + export PYCURL_SSL_LIBRARY=nss + ;; + *) + echo -e "$0: Unknown target '$TARGET'.\n" >&2 + exit 1 + ;; +esac + + if ! [[ -n "$WORKSPACE" ]]; then echo >&2 "$helpmessage" echo >&2 @@ -106,15 +152,24 @@ if [[ "$?" != 0 ]]; then exit 1 fi -if ! easy_install3 --version >/dev/null; then - cat >&2 </dev/null 2>&1; then + echo "easy_install$version_suffix" + return 0 + fi + done + cat >&2 <" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT") + if [ python = "$PACKAGE_TYPE" ]; then + COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*) fi - declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests") - if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then COMMAND_ARR+=('-n' "$PACKAGE_NAME") fi @@ -236,7 +307,7 @@ fpm_verify_and_scp () { FPM_RESULTS=$@ FPM_PACKAGE_NAME='' - if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then + if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]} fi @@ -252,11 +323,7 @@ fpm_verify_and_scp () { echo "Error building package for $1:\n $FPM_RESULTS" else if [[ "$UPLOAD" != 0 ]]; then - if [[ "$FORMAT" == 'deb' ]]; then - scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:tmp/ - else - scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:rpm/ - fi + scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/ CALL_FREIGHT=1 fi fi @@ -285,6 +352,10 @@ if [[ "$DEBUG" != 0 ]]; then echo "umask is" `umask` fi +if [[ ! -d "$WORKSPACE/debs" ]]; then + mkdir -p $WORKSPACE/debs +fi + # Perl packages if [[ "$DEBUG" != 0 ]]; then echo -e "\nPerl packages\n" @@ -301,14 +372,15 @@ cd "$WORKSPACE/sdk/perl" if [[ -e Makefile ]]; then make realclean >"$PERL_OUT" fi -find -maxdepth 1 \( -name 'MANIFEST*' -or -name 'libarvados-perl_*.deb' \) \ +find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ -delete rm -rf install -perl Makefile.PL >"$PERL_OUT" && \ - make install PREFIX=install INSTALLDIRS=perl >"$PERL_OUT" && \ - fpm_build_and_scp install/=/usr libarvados-perl "Curoverse, Inc." dir \ - "$(version_from_git)" +perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ + make install INSTALLDIRS=perl >"$PERL_OUT" && \ + fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ + "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ + mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/" # Ruby gems if [[ "$DEBUG" != 0 ]]; then @@ -400,10 +472,6 @@ handle_python_package cd ../../services/nodemanager handle_python_package -if [[ ! -d "$WORKSPACE/debs" ]]; then - mkdir -p $WORKSPACE/debs -fi - # Arvados-src # We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then @@ -523,40 +591,33 @@ fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" +fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" +fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" +fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner cd $WORKSPACE/debs fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies -for deppkg in python-gflags pyvcf google-api-python-client oauth2client \ - pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py virtualenv \ - pykka apache-libcloud requests six pyexecjs jsonschema ciso8601 \ - pycrypto backports.ssl_match_hostname; do - fpm_build_and_scp "$deppkg" +for deppkg in "${PYTHON_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") + fpm_build_and_scp "$deppkg" "$outname" done # Python 3 dependencies -for deppkg in docker-py six requests; do +for deppkg in "${PYTHON3_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3 + fpm_build_and_scp "$deppkg" "$outname" "" python3 done # Build the API server package @@ -689,11 +750,11 @@ fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS # Finally, publish the packages, if necessary if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then - if [[ "$FORMAT" == 'deb' ]]; then - ssh -p2222 $SCPUSER@$SCPHOST -t "cd tmp && ls -laF *deb && freight add *deb apt/wheezy && freight cache && rm -f *deb" - else - ssh -p2222 $SCPUSER@$SCPHOST -t "cd rpm && ls -laF *rpm && mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/" - fi + ssh -p2222 $SCPUSER@$SCPHOST -t bash - <