X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/82807388a1f7dd901e619419690253a5fd58663b..refs/heads/6388-package-scripts-redhat-support-wip:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index b54761c..0eb2244 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -35,7 +35,7 @@ BUILD_BUNDLE_PACKAGES=0 TARGET=debian7 PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,upload,scp-user:,scp-host:,apt-server:,build-bundle-packages,debug,target: \ + help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -52,7 +52,7 @@ while [ $# -gt 0 ]; do --scp-user) SCPUSER="$2"; shift ;; - --scp-host|--apt-server) + --scp-host) SCPHOST="$2"; shift ;; --target) @@ -95,14 +95,33 @@ case "$TARGET" in debian7) FORMAT=deb FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' + REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb' PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl) + ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + PYTHON3_BACKPORTS=(docker-py six requests) + ;; + ubuntu1204) + FORMAT=deb + FPM_OUTDIR=tmp + REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rsync -AaHX /var/www/freight/ /var/www/freight-production/ --delete && rm -f *deb' + + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) PYTHON3_BACKPORTS=(docker-py six requests) ;; centos6) @@ -111,12 +130,17 @@ case "$TARGET" in REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') + PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \ - virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl) + oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py \ + pykka apache-libcloud requests six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname pycurl + python-daemon lockfile llfuse) PYTHON3_BACKPORTS=(docker-py six requests) + export PYCURL_SSL_LIBRARY=nss ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 @@ -213,7 +237,7 @@ handle_python_package () { fi } -# Build debs for everything +# Build packages for everything fpm_build_and_scp () { # The package source. Depending on the source type, this can be a # path, or the name of the package in an upstream repository (e.g., @@ -239,7 +263,9 @@ fpm_build_and_scp () { # All Arvados Python2 packages depend on Python 2.7. # Make sure we build with that for consistency. set -- "$@" --python-bin python2.7 \ - --python-easyinstall "$EASY_INSTALL2" + --python-easyinstall "$EASY_INSTALL2" \ + --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \ + --depends "$PYTHON2_PACKAGE" ;; python3) # fpm does not actually support a python3 package type. Instead @@ -249,11 +275,15 @@ fpm_build_and_scp () { PACKAGE_TYPE=python set -- "$@" --python-bin python3 \ --python-easyinstall "$EASY_INSTALL3" \ - --python-package-name-prefix python3 --depends "$PYTHON3_PACKAGE" + --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \ + --depends "$PYTHON3_PACKAGE" ;; esac - declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests") + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "-s" "$PACKAGE_TYPE" "-t" "$FORMAT") + if [ python = "$PACKAGE_TYPE" ]; then + COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*) + fi if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then COMMAND_ARR+=('-n' "$PACKAGE_NAME") @@ -286,14 +316,14 @@ fpm_build_and_scp () { fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS } -# verify build results and scp debs, if needed +# verify build results and scp packages, if needed fpm_verify_and_scp () { FPM_EXIT_CODE=$1 shift FPM_RESULTS=$@ FPM_PACKAGE_NAME='' - if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then + if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]} fi @@ -338,8 +368,8 @@ if [[ "$DEBUG" != 0 ]]; then echo "umask is" `umask` fi -if [[ ! -d "$WORKSPACE/debs" ]]; then - mkdir -p $WORKSPACE/debs +if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then + mkdir -p $WORKSPACE/packages/$TARGET fi # Perl packages @@ -366,7 +396,7 @@ perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ make install INSTALLDIRS=perl >"$PERL_OUT" && \ fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ - mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/" + mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" # Ruby gems if [[ "$DEBUG" != 0 ]]; then @@ -489,7 +519,7 @@ git log --format=format:%H -n1 . > git-commit.version # Build arvados src deb package cd "$WORKSPACE" PKG_VERSION=$(version_from_git) -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" # clean up, check out master and step away from detached-head state @@ -509,7 +539,7 @@ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/keepstore" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" # Get GO SDK version @@ -529,7 +559,7 @@ else fi go get "git.curoverse.com/arvados.git/services/keepproxy" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" # datamanager @@ -544,7 +574,7 @@ else fi go get "git.curoverse.com/arvados.git/services/datamanager" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." # arv-git-httpd @@ -559,14 +589,14 @@ else fi go get "git.curoverse.com/arvados.git/services/arv-git-httpd" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." # crunchstat cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/crunchstat" -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" # The Python SDK @@ -576,32 +606,34 @@ fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # prefix from only one of the dependencies of a package... Maybe I could # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends="$PYTHON2_PACKAGE" +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends="$PYTHON2_PACKAGE" +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager -cd $WORKSPACE/debs -fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends="$PYTHON2_PACKAGE" +cd $WORKSPACE/packages/$TARGET +fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies for deppkg in "${PYTHON_BACKPORTS[@]}"; do - fpm_build_and_scp "$deppkg" + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") + fpm_build_and_scp "$deppkg" "$outname" done # Python 3 dependencies for deppkg in "${PYTHON3_BACKPORTS[@]}"; do + outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3 + fpm_build_and_scp "$deppkg" "$outname" "" python3 done # Build the API server package @@ -623,7 +655,7 @@ fi /usr/bin/git rev-parse HEAD > git-commit.version -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET # Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now, # we do that in the upgrade script. @@ -697,7 +729,7 @@ if [[ "$?" != "0" ]]; then EXITCODE=1 fi -cd $WORKSPACE/debs +cd $WORKSPACE/packages/$TARGET # This is the complete package with vendor/bundle included. # It's big, so we do not build it by default. @@ -735,7 +767,7 @@ fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS # Finally, publish the packages, if necessary if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then ssh -p2222 $SCPUSER@$SCPHOST -t bash - <