X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/e5561b32e943e67eb457b8d7669e005a0338395e..d30c7f8b3210ae1e825033e852c369d16b7e48f1:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index 3d7e55d..42b42da 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -35,7 +35,7 @@ BUILD_BUNDLE_PACKAGES=0 TARGET=debian7 PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,target: \ + help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -52,7 +52,7 @@ while [ $# -gt 0 ]; do --scp-user) SCPUSER="$2"; shift ;; - --scp-host|--apt-server) + --scp-host) SCPHOST="$2"; shift ;; --target) @@ -88,26 +88,75 @@ fi declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS +PYTHON2_VERSION=2.7 +PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') + case "$TARGET" in debian7) FORMAT=deb FPM_OUTDIR=tmp REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl) - PYTHON3_BACKPORTS=(docker-py six requests) + ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + PYTHON3_BACKPORTS=(docker-py six requests websocket-client) + ;; + debian8) + FORMAT=deb + FPM_OUTDIR=tmp + REPO_UPDATE_CMD='freight add *deb apt/jessie && freight cache && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + PYTHON3_BACKPORTS=(docker-py six requests websocket-client) + ;; + ubuntu1204) + FORMAT=deb + FPM_OUTDIR=tmp + REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; centos6) FORMAT=rpm FPM_OUTDIR=rpm REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' + + PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') + PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE + PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \ - virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl) + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl + python-daemon lockfile llfuse) PYTHON3_BACKPORTS=(docker-py six requests) + export PYCURL_SSL_LIBRARY=nss ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 @@ -135,15 +184,24 @@ if [[ "$?" != 0 ]]; then exit 1 fi -if ! easy_install3 --version >/dev/null; then - cat >&2 </dev/null 2>&1; then + echo "easy_install$version_suffix" + return 0 + fi + done + cat >&2 <" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests") + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT") + if [ python = "$PACKAGE_TYPE" ]; then + COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*) + fi if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then COMMAND_ARR+=('-n' "$PACKAGE_NAME") @@ -267,14 +332,14 @@ fpm_build_and_scp () { fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS } -# verify build results and scp debs, if needed +# verify build results and scp packages, if needed fpm_verify_and_scp () { FPM_EXIT_CODE=$1 shift FPM_RESULTS=$@ FPM_PACKAGE_NAME='' - if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then + if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]} fi @@ -319,6 +384,10 @@ if [[ "$DEBUG" != 0 ]]; then echo "umask is" `umask` fi +if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then + mkdir -p $WORKSPACE/packages/$TARGET +fi + # Perl packages if [[ "$DEBUG" != 0 ]]; then echo -e "\nPerl packages\n" @@ -335,14 +404,15 @@ cd "$WORKSPACE/sdk/perl" if [[ -e Makefile ]]; then make realclean >"$PERL_OUT" fi -find -maxdepth 1 \( -name 'MANIFEST*' -or -name 'libarvados-perl_*.deb' \) \ +find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ -delete rm -rf install -perl Makefile.PL >"$PERL_OUT" && \ - make install PREFIX=install INSTALLDIRS=perl >"$PERL_OUT" && \ - fpm_build_and_scp install/=/usr libarvados-perl "Curoverse, Inc." dir \ - "$(version_from_git)" +perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ + make install INSTALLDIRS=perl >"$PERL_OUT" && \ + fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ + "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ + mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" # Ruby gems if [[ "$DEBUG" != 0 ]]; then @@ -434,10 +504,6 @@ handle_python_package cd ../../services/nodemanager handle_python_package -if [[ ! -d "$WORKSPACE/debs" ]]; then - mkdir -p $WORKSPACE/debs -fi - # Arvados-src # We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then @@ -456,20 +522,20 @@ if [[ "$DEBUG" != 0 ]]; then git checkout master git pull # go into detached-head state - git checkout `git log --format=format:%h -n1 .` + git checkout `git log --format=format:%h -n1 --first-parent .` else git checkout -q master git pull -q # go into detached-head state - git checkout -q `git log --format=format:%h -n1 .` + git checkout -q `git log --format=format:%h -n1 --first-parent .` fi -git log --format=format:%H -n1 . > git-commit.version +git log --format=format:%H -n1 --first-parent . > git-commit.version # Build arvados src deb package cd "$WORKSPACE" PKG_VERSION=$(version_from_git) -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" # clean up, check out master and step away from detached-head state @@ -489,7 +555,7 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/keepstore" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" # Get GO SDK version @@ -509,7 +575,7 @@ else fi go get "git.curoverse.com/arvados.git/services/keepproxy" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" # datamanager @@ -524,7 +590,7 @@ else fi go get "git.curoverse.com/arvados.git/services/datamanager" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." # arv-git-httpd @@ -539,14 +605,14 @@ else fi go get "git.curoverse.com/arvados.git/services/arv-git-httpd" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." # crunchstat cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/crunchstat" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" # The Python SDK @@ -556,34 +622,48 @@ fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # prefix from only one of the dependencies of a package... Maybe I could # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends=python2.7 +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends=python2.7 +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends=python2.7 +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies for deppkg in "${PYTHON_BACKPORTS[@]}"; do - fpm_build_and_scp "$deppkg" + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") + fpm_build_and_scp "$deppkg" "$outname" done # Python 3 dependencies for deppkg in "${PYTHON3_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3 + fpm_build_and_scp "$deppkg" "$outname" "" python3 done +# libpam-arvados +cd $WORKSPACE/sdk/pam +PKG_VERSION=$(version_from_git) +cd $WORKSPACE/packages/$TARGET + +if [[ "$FORMAT" == "deb" ]]; then + fpm_build_and_scp $WORKSPACE/sdk/pam/debian/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/debian/arvados_pam=/etc/default/arvados_pam +#else + # FIXME enable and test once we have the centos pam.d file + #fpm_build_and_scp $WORKSPACE/sdk/pam/centos/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/centos/arvados_pam=/etc/default/arvados_pam +fi + # Build the API server package cd "$WORKSPACE/services/api" @@ -603,7 +683,7 @@ fi /usr/bin/git rev-parse HEAD > git-commit.version -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET # Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now, # we do that in the upgrade script. @@ -666,8 +746,9 @@ fi # from the package - empty it instead. rm -rf $WORKSPACE/apps/workbench/tmp/* -# Set up application.yml so that asset precompilation works +# Set up application.yml and production.rb so that asset precompilation works \cp config/application.yml.example config/application.yml -f +\cp config/environments/production.rb.example config/environments/production.rb -f sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null @@ -677,7 +758,7 @@ if [[ "$?" != "0" ]]; then EXITCODE=1 fi -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET # This is the complete package with vendor/bundle included. # It's big, so we do not build it by default. @@ -715,7 +796,7 @@ fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS # Finally, publish the packages, if necessary if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then ssh -p2222 $SCPUSER@$SCPHOST -t bash - <