X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/591ddfb6f369bd37bf8218646219f5f3fcc561bd..3eecec6dc8eea0765a0821526255f682daa3f321:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index e6a89b4..6ebef74 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -12,15 +12,15 @@ Options: --upload Upload packages (default: false) --scp-user USERNAME - Scp user for apt server (only required when --upload is specified) ---apt-server HOSTNAME - Apt server hostname (only required when --upload is specified) + scp user for repository server (only required when --upload is specified) +--scp-host HOSTNAME + scp host for repository server (only required when --upload is specified) --build-bundle-packages (default: false) Build api server and workbench packages with vendor/bundle included --debug Output debug information (default: false) ---format - Package format (default: deb) +--target + Distribution to build packages for (default: debian7) WORKSPACE=path Path to the Arvados source tree to build packages from @@ -32,25 +32,31 @@ CALL_FREIGHT=0 DEBUG=0 UPLOAD=0 BUILD_BUNDLE_PACKAGES=0 -FORMAT=deb +TARGET=debian7 -while [[ -n "$1" ]] -do - arg="$1"; shift - case "$arg" in +PARSEDOPTS=$(getopt --name "$0" --longoptions \ + help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \ + -- "" "$@") +if [ $? -ne 0 ]; then + exit 1 +fi + +eval set -- "$PARSEDOPTS" +while [ $# -gt 0 ]; do + case "$1" in --help) echo >&2 "$helpmessage" echo >&2 exit 1 ;; --scp-user) - APTUSER="$1"; shift + SCPUSER="$2"; shift ;; - --apt-server) - APTSERVER="$1"; shift + --scp-host) + SCPHOST="$2"; shift ;; - --format) - FORMAT="$1"; shift + --target) + TARGET="$2"; shift ;; --debug) DEBUG=1 @@ -61,23 +67,72 @@ do --build-bundle-packages) BUILD_BUNDLE_PACKAGES=1 ;; - *) - echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help" - exit 1 + --) + if [ $# -gt 1 ]; then + echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help" + exit 1 + fi ;; esac + shift done # Sanity checks -if [[ "$UPLOAD" != '0' && ("$APTUSER" == '' || "$APTSERVER" == '') ]]; then +if [[ "$UPLOAD" != '0' && ("$SCPUSER" == '' || "$SCPHOST" == '') ]]; then echo >&2 "$helpmessage" echo >&2 - echo >&2 "Error: please specify --scp-user and --apt-server if --upload is set" + echo >&2 "Error: please specify --scp-user and --scp-host if --upload is set" echo >&2 exit 1 fi -# Sanity check +declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS + +PYTHON2_VERSION=2.7 +PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') + +case "$TARGET" in + debian7) + FORMAT=deb + FPM_OUTDIR=tmp + REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl) + PYTHON3_BACKPORTS=(docker-py six requests) + ;; + centos6) + FORMAT=rpm + FPM_OUTDIR=rpm + REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' + + PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') + PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE + PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl + daemon llfuse) + PYTHON3_BACKPORTS=(docker-py six requests) + export PYCURL_SSL_LIBRARY=nss + ;; + *) + echo -e "$0: Unknown target '$TARGET'.\n" >&2 + exit 1 + ;; +esac + + if ! [[ -n "$WORKSPACE" ]]; then echo >&2 "$helpmessage" echo >&2 @@ -97,15 +152,24 @@ if [[ "$?" != 0 ]]; then exit 1 fi -if ! easy_install3 --version >/dev/null; then - cat >&2 </dev/null 2>&1; then + echo "easy_install$version_suffix" + return 0 + fi + done + cat >&2 <" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT") + if [ python = "$PACKAGE_TYPE" ]; then + COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*) fi - declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests") - if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then COMMAND_ARR+=('-n' "$PACKAGE_NAME") fi @@ -217,11 +297,11 @@ build_and_scp_deb () { FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS } # verify build results and scp debs, if needed -verify_and_scp_deb () { +fpm_verify_and_scp () { FPM_EXIT_CODE=$1 shift FPM_RESULTS=$@ @@ -243,11 +323,7 @@ verify_and_scp_deb () { echo "Error building package for $1:\n $FPM_RESULTS" else if [[ "$UPLOAD" != 0 ]]; then - if [[ "$FORMAT" == 'deb' ]]; then - scp -P2222 $FPM_PACKAGE_NAME $APTUSER@$APTSERVER:tmp/ - else - scp -P2222 $FPM_PACKAGE_NAME $APTUSER@$APTSERVER:rpm/ - fi + scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/ CALL_FREIGHT=1 fi fi @@ -276,6 +352,10 @@ if [[ "$DEBUG" != 0 ]]; then echo "umask is" `umask` fi +if [[ ! -d "$WORKSPACE/debs" ]]; then + mkdir -p $WORKSPACE/debs +fi + # Perl packages if [[ "$DEBUG" != 0 ]]; then echo -e "\nPerl packages\n" @@ -292,14 +372,15 @@ cd "$WORKSPACE/sdk/perl" if [[ -e Makefile ]]; then make realclean >"$PERL_OUT" fi -find -maxdepth 1 \( -name 'MANIFEST*' -or -name 'libarvados-perl_*.deb' \) \ +find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ -delete rm -rf install -perl Makefile.PL >"$PERL_OUT" && \ - make install PREFIX=install INSTALLDIRS=perl >"$PERL_OUT" && \ - build_and_scp_deb install/=/usr libarvados-perl "Curoverse, Inc." dir \ - "$(version_from_git)" +perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ + make install INSTALLDIRS=perl >"$PERL_OUT" && \ + fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ + "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ + mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/" # Ruby gems if [[ "$DEBUG" != 0 ]]; then @@ -335,16 +416,16 @@ if [[ "$?" != "0" ]]; then # -q appears to be broken in gem version 2.2.2 gem build arvados.gemspec -q >/dev/null 2>&1 fi - + if [[ "$UPLOAD" != 0 ]]; then # publish new gem gem push arvados-*gem fi - - build_and_scp_deb arvados-*.gem "" "Curoverse, Inc." gem "" \ + + fpm_build_and_scp arvados-*.gem "" "Curoverse, Inc." gem "" \ --prefix "$FPM_GEM_PREFIX" fi - + # Build arvados-cli GEM cd "$WORKSPACE" cd sdk/cli @@ -359,14 +440,14 @@ gem search arvados-cli -r -a |grep -q $ARVADOS_GEM_VERSION if [[ "$?" != "0" ]]; then # clean up old gems rm -f arvados-cli*gem - + if [[ "$DEBUG" != 0 ]]; then gem build arvados-cli.gemspec else # -q appears to be broken in gem version 2.2.2 gem build arvados-cli.gemspec -q >/dev/null fi - + if [[ "$UPLOAD" != 0 ]]; then # publish new gem gem push arvados-cli*gem @@ -391,10 +472,6 @@ handle_python_package cd ../../services/nodemanager handle_python_package -if [[ ! -d "$WORKSPACE/debs" ]]; then - mkdir -p $WORKSPACE/debs -fi - # Arvados-src # We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then @@ -427,7 +504,7 @@ git log --format=format:%H -n1 . > git-commit.version cd "$WORKSPACE" PKG_VERSION=$(version_from_git) cd $WORKSPACE/debs -build_and_scp_deb $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" +fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" # clean up, check out master and step away from detached-head state cd "$WORKSPACE/src-build-dir" @@ -447,7 +524,7 @@ cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/keepstore" cd $WORKSPACE/debs -build_and_scp_deb $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" +fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" # Get GO SDK version cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go" @@ -467,7 +544,7 @@ fi go get "git.curoverse.com/arvados.git/services/keepproxy" cd $WORKSPACE/debs -build_and_scp_deb $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" +fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" # datamanager cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager" @@ -482,7 +559,7 @@ fi go get "git.curoverse.com/arvados.git/services/datamanager" cd $WORKSPACE/debs -build_and_scp_deb $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." +fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." # arv-git-httpd cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd" @@ -497,14 +574,14 @@ fi go get "git.curoverse.com/arvados.git/services/arv-git-httpd" cd $WORKSPACE/debs -build_and_scp_deb $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." +fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." # crunchstat cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/crunchstat" cd $WORKSPACE/debs -build_and_scp_deb $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" +fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" # The Python SDK # Please resist the temptation to add --no-python-fix-name to the fpm call here @@ -514,40 +591,33 @@ build_and_scp_deb $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -build_and_scp_deb $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" +fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -build_and_scp_deb $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2 }' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" +fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager cd $WORKSPACE/debs -# Python version numbering is obscure. Strip dashes and replace them with dots -# to match our other version numbers. Cf. commit 4afcb8c, compliance with PEP-440. -build_and_scp_deb $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){ gsub(/-/,".",$2); print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" +fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner cd $WORKSPACE/debs -build_and_scp_deb $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" +fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies -for deppkg in python-gflags pyvcf google-api-python-client oauth2client \ - pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py virtualenv \ - pykka apache-libcloud requests six pyexecjs jsonschema ciso8601 \ - pycrypto backports.ssl_match_hostname; do - build_and_scp_deb "$deppkg" +for deppkg in "${PYTHON_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") + fpm_build_and_scp "$deppkg" "$outname" done # Python 3 dependencies -for deppkg in docker-py six requests; do +for deppkg in "${PYTHON3_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - build_and_scp_deb "$deppkg" "python3-$deppkg" "" python3 + fpm_build_and_scp "$deppkg" "$outname" "" python3 done # Build the API server package @@ -590,7 +660,7 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. @@ -604,7 +674,7 @@ fi FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS # API server package build done @@ -632,8 +702,17 @@ fi # from the package - empty it instead. rm -rf $WORKSPACE/apps/workbench/tmp/* +# Set up application.yml so that asset precompilation works +\cp config/application.yml.example config/application.yml -f +sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml + RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null +if [[ "$?" != "0" ]]; then + echo "ERROR: Asset precompilation failed" + EXITCODE=1 +fi + cd $WORKSPACE/debs # This is the complete package with vendor/bundle included. @@ -650,7 +729,7 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. @@ -665,17 +744,17 @@ fi FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS # Workbench package build done # Finally, publish the packages, if necessary if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then - if [[ "$FORMAT" == 'deb' ]]; then - ssh -p2222 $APTUSER@$APTSERVER -t "cd tmp && ls -laF *deb && freight add *deb apt/wheezy && freight cache && rm -f *deb" - else - ssh -p2222 $APTUSER@$APTSERVER -t "cd rpm && ls -laF *rpm && mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/" - fi + ssh -p2222 $SCPUSER@$SCPHOST -t bash - <