From 5cf15331acb6250da715ee9c41a2a0db7986917c Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Wed, 22 Jul 2015 16:38:23 -0400 Subject: [PATCH] 6497: Separate package build and uploader scripts. This lets us avoid redundant uploads as we build for multiple targets, and lets us better customize the upload process per target. run-build-packages had code to avoid doing an upload when there was no new package to build. run_upload_packages uses a timestamp file to achieve a similar result. Other changes to run-build-packages to support this: * Better detection of when Python and Ruby packages have already been built. * Refactor Gem building into a function to fix various bugs (wrong version checked for existing arvados-cli; distro packages not installed in the proper directory). * Refactor git log querying to reduce code duplication. --- jenkins/run-build-packages.sh | 249 ++++++++++--------------------- jenkins/run_upload_packages.py | 259 +++++++++++++++++++++++++++++++++ 2 files changed, 337 insertions(+), 171 deletions(-) create mode 100755 jenkins/run_upload_packages.py diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index 42b42da..fc12cb4 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -2,19 +2,13 @@ read -rd "\000" helpmessage <&2 exit 1 ;; - --scp-user) - SCPUSER="$2"; shift - ;; - --scp-host) - SCPHOST="$2"; shift - ;; --target) TARGET="$2"; shift ;; --debug) DEBUG=1 ;; - --upload) - UPLOAD=1 - ;; --build-bundle-packages) BUILD_BUNDLE_PACKAGES=1 ;; @@ -77,15 +59,6 @@ while [ $# -gt 0 ]; do shift done -# Sanity checks -if [[ "$UPLOAD" != '0' && ("$SCPUSER" == '' || "$SCPHOST" == '') ]]; then - echo >&2 "$helpmessage" - echo >&2 - echo >&2 "Error: please specify --scp-user and --scp-host if --upload is set" - echo >&2 - exit 1 -fi - declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS PYTHON2_VERSION=2.7 @@ -94,9 +67,6 @@ PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=s case "$TARGET" in debian7) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -110,9 +80,6 @@ case "$TARGET" in ;; debian8) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/jessie && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -126,9 +93,6 @@ case "$TARGET" in ;; ubuntu1204) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -142,9 +106,6 @@ case "$TARGET" in ;; centos6) FORMAT=rpm - FPM_OUTDIR=rpm - REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' - PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') @@ -216,45 +177,66 @@ if [[ "$DEBUG" != 0 ]]; then echo "Workspace is $WORKSPACE" fi +format_last_commit_here() { + local format=$1; shift + TZ=UTC git log -n1 --first-parent "--format=format:$format" . +} + version_from_git() { # Generates a version number from the git log for the current working # directory, and writes it to stdout. local git_ts git_hash - declare $(TZ=UTC git log -n1 --first-parent \ - --format=format:"git_ts=%ct git_hash=%h" .) + declare $(format_last_commit_here "git_ts=%ct git_hash=%h") echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash" } +nohash_version_from_git() { + version_from_git | cut -d. -f1-3 +} + timestamp_from_git() { - # Generates a version number from the git log for the current working - # directory, and writes it to stdout. - local git_ts git_hash - declare $(TZ=UTC git log -n1 --first-parent \ - --format=format:"git_ts=%ct git_hash=%h" .) - echo "$git_ts" + format_last_commit_here "%ct" } handle_python_package () { # This function assumes the current working directory is the python package directory - if [[ "$UPLOAD" != 0 ]]; then - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - if [[ "$DEBUG" != 0 ]]; then - python setup.py sdist upload - else - python setup.py -q sdist upload - fi + if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then + # This package doesn't need rebuilding. + return + fi + # Make sure only to use sdist - that's the only format pip can deal with (sigh) + if [[ "$DEBUG" != 0 ]]; then + python setup.py sdist else - # Make sure only to use sdist - that's the only format pip can deal with (sigh) + python setup.py -q sdist + fi +} + +handle_ruby_gem() { + local gem_name=$1; shift + local gem_version=$(nohash_version_from_git) + + if [ -e "${gem_name}-${gem_version}.gem" ]; then + # This gem doesn't need rebuilding. + return + fi + + find -maxdepth 1 -name "${gem_name}-*.gem" -delete + if [[ "$DEBUG" != 0 ]]; then - python setup.py sdist + gem build "$gem_name.gemspec" else - python setup.py -q sdist + # -q appears to be broken in gem version 2.2.2 + gem build "$gem_name.gemspec" -q >/dev/null 2>&1 fi - fi + + fpm_build "$gem_name"-*.gem "" "Curoverse, Inc." gem "" \ + --prefix "$FPM_GEM_PREFIX" + mv -t "$WORKSPACE/packages/$TARGET/" "$gem_name"*."$FORMAT" } # Build packages for everything -fpm_build_and_scp () { +fpm_build () { # The package source. Depending on the source type, this can be a # path, or the name of the package in an upstream repository (e.g., # pip). @@ -329,11 +311,11 @@ fpm_build_and_scp () { FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS } -# verify build results and scp packages, if needed -fpm_verify_and_scp () { +# verify build results +fpm_verify () { FPM_EXIT_CODE=$1 shift FPM_RESULTS=$@ @@ -349,19 +331,10 @@ fpm_verify_and_scp () { echo echo $FPM_RESULTS echo - else - if [[ ! $FPM_RESULTS =~ "File already exists" ]]; then - if [[ "$FPM_EXIT_CODE" != "0" ]]; then - echo "Error building package for $1:\n $FPM_RESULTS" - else - if [[ "$UPLOAD" != 0 ]]; then - scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/ - CALL_FREIGHT=1 - fi - fi - else - echo "Package $FPM_PACKAGE_NAME exists, not rebuilding" - fi + elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then + echo "Package $FPM_PACKAGE_NAME exists, not rebuilding" + elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then + echo "Error building package for $1:\n $FPM_RESULTS" fi } @@ -410,7 +383,7 @@ rm -rf install perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ make install INSTALLDIRS=perl >"$PERL_OUT" && \ - fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ + fpm_build install/lib/=/usr/share libarvados-perl \ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" @@ -427,64 +400,11 @@ else FPM_GEM_PREFIX=$(gem environment gemdir) fi -cd "$WORKSPACE" -cd sdk/ruby - -ARVADOS_GEM_EPOCH=`git log -n1 --first-parent --format=%ct` -ARVADOS_GEM_DATE=`date --utc --date="@${ARVADOS_GEM_EPOCH}" +%Y%m%d%H%M%S` -ARVADOS_GEM_VERSION="0.1.${ARVADOS_GEM_DATE}" - -# see if this gem needs building/uploading -gem search arvados -r -a |grep -q $ARVADOS_GEM_VERSION - -if [[ "$?" != "0" ]]; then - # clean up old packages - find -maxdepth 1 \( -name 'arvados-*.gem' -or -name 'rubygem-arvados_*.deb' -or -name 'rubygem-arvados_*.rpm' \) \ - -delete - - if [[ "$DEBUG" != 0 ]]; then - gem build arvados.gemspec - else - # -q appears to be broken in gem version 2.2.2 - gem build arvados.gemspec -q >/dev/null 2>&1 - fi - - if [[ "$UPLOAD" != 0 ]]; then - # publish new gem - gem push arvados-*gem - fi - - fpm_build_and_scp arvados-*.gem "" "Curoverse, Inc." gem "" \ - --prefix "$FPM_GEM_PREFIX" -fi - -# Build arvados-cli GEM -cd "$WORKSPACE" -cd sdk/cli - -ARVADOS_CLI_GEM_EPOCH=`git log -n1 --first-parent --format=%ct` -ARVADOS_CLI_GEM_DATE=`date --utc --date="@${ARVADOS_CLI_GEM_EPOCH}" +%Y%m%d%H%M%S` -ARVADOS_CLI_GEM_VERSION="0.1.${ARVADOS_CLI_GEM_DATE}" - -# see if this gem needs building/uploading -gem search arvados-cli -r -a |grep -q $ARVADOS_GEM_VERSION +cd "$WORKSPACE/sdk/ruby" +handle_ruby_gem arvados -if [[ "$?" != "0" ]]; then - # clean up old gems - rm -f arvados-cli*gem - - if [[ "$DEBUG" != 0 ]]; then - gem build arvados-cli.gemspec - else - # -q appears to be broken in gem version 2.2.2 - gem build arvados-cli.gemspec -q >/dev/null - fi - - if [[ "$UPLOAD" != 0 ]]; then - # publish new gem - gem push arvados-cli*gem - fi -fi +cd "$WORKSPACE/sdk/cli" +handle_ruby_gem arvados-cli # Python packages if [[ "$DEBUG" != 0 ]]; then @@ -522,21 +442,22 @@ if [[ "$DEBUG" != 0 ]]; then git checkout master git pull # go into detached-head state - git checkout `git log --format=format:%h -n1 --first-parent .` + MASTER_COMMIT_HASH=$(format_last_commit_here "%H") + git checkout "$MASTER_COMMIT_HASH" else git checkout -q master git pull -q # go into detached-head state - git checkout -q `git log --format=format:%h -n1 --first-parent .` + MASTER_COMMIT_HASH=$(format_last_commit_here "%H") + git checkout -q "$MASTER_COMMIT_HASH" fi - -git log --format=format:%H -n1 --first-parent . > git-commit.version +echo "$MASTER_COMMIT_HASH" >git-commit.version # Build arvados src deb package cd "$WORKSPACE" PKG_VERSION=$(version_from_git) cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" +fpm_build $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" # clean up, check out master and step away from detached-head state cd "$WORKSPACE/src-build-dir" @@ -556,7 +477,7 @@ cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/keepstore" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" +fpm_build $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" # Get GO SDK version cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go" @@ -576,7 +497,7 @@ fi go get "git.curoverse.com/arvados.git/services/keepproxy" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" +fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" # datamanager cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager" @@ -591,7 +512,7 @@ fi go get "git.curoverse.com/arvados.git/services/datamanager" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." +fpm_build $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." # arv-git-httpd cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd" @@ -606,14 +527,14 @@ fi go get "git.curoverse.com/arvados.git/services/arv-git-httpd" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." +fpm_build $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." # crunchstat cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/crunchstat" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" +fpm_build $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" # The Python SDK # Please resist the temptation to add --no-python-fix-name to the fpm call here @@ -623,33 +544,33 @@ fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git +fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" +fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" +fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" +fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies for deppkg in "${PYTHON_BACKPORTS[@]}"; do outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") - fpm_build_and_scp "$deppkg" "$outname" + fpm_build "$deppkg" "$outname" done # Python 3 dependencies for deppkg in "${PYTHON3_BACKPORTS[@]}"; do outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build_and_scp "$deppkg" "$outname" "" python3 + fpm_build "$deppkg" "$outname" "" python3 done # libpam-arvados @@ -658,10 +579,10 @@ PKG_VERSION=$(version_from_git) cd $WORKSPACE/packages/$TARGET if [[ "$FORMAT" == "deb" ]]; then - fpm_build_and_scp $WORKSPACE/sdk/pam/debian/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/debian/arvados_pam=/etc/default/arvados_pam + fpm_build $WORKSPACE/sdk/pam/debian/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/debian/arvados_pam=/etc/default/arvados_pam #else # FIXME enable and test once we have the centos pam.d file - #fpm_build_and_scp $WORKSPACE/sdk/pam/centos/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/centos/arvados_pam=/etc/default/arvados_pam + #fpm_build $WORKSPACE/sdk/pam/centos/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/centos/arvados_pam=/etc/default/arvados_pam fi # Build the API server package @@ -704,7 +625,7 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. @@ -718,7 +639,7 @@ fi FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify $FPM_EXIT_CODE $FPM_RESULTS # API server package build done @@ -774,7 +695,7 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. @@ -789,23 +710,9 @@ fi FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify $FPM_EXIT_CODE $FPM_RESULTS # Workbench package build done - -# Finally, publish the packages, if necessary -if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then - ssh -p2222 $SCPUSER@$SCPHOST -t bash - <= since_timestamp: + yield path + + def upload_file(self, path): + raise NotImplementedError("PackageSuite.upload_file") + + def upload_files(self, paths): + for path in paths: + self.logger.info("Uploading %s", path) + self.upload_file(path) + + def post_uploads(self, paths): + pass + + def update_packages(self, since_timestamp): + upload_paths = list(self.files_to_upload(since_timestamp)) + if upload_paths: + self.upload_files(upload_paths) + self.post_uploads(upload_paths) + + +class PythonPackageSuite(PackageSuite): + LOGGER_PART = 'python' + + def __init__(self, glob_root, rel_globs): + super().__init__(glob_root, rel_globs) + self.seen_packages = set() + + def upload_file(self, path): + src_dir = os.path.dirname(os.path.dirname(path)) + if src_dir in self.seen_packages: + return + self.seen_packages.add(src_dir) + # NOTE: If we ever start uploading Python 3 packages, we'll need to + # figure out some way to adapt cmd to match. It might be easiest + # to give all our setup.py files the executable bit, and run that + # directly. + # We also must run `sdist` before `upload`: `upload` uploads any + # distributions previously generated in the command. It doesn't + # know how to upload distributions already on disk. We write the + # result to a dedicated directory to avoid interfering with our + # timestamp tracking. + cmd = ['python2.7', 'setup.py'] + if not self.logger.isEnabledFor(logging.INFO): + cmd.append('--quiet') + cmd.extend(['sdist', '--dist-dir', '.upload_dist', 'upload']) + subprocess.check_call(cmd, cwd=src_dir) + shutil.rmtree(os.path.join(src_dir, '.upload_dist')) + + +class GemPackageSuite(PackageSuite): + LOGGER_PART = 'gems' + + def upload_file(self, path): + cmd = ['gem', 'push', path] + push_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + repushed = any(line == b'Repushing of gem versions is not allowed.\n' + for line in push_proc.stdout) + # Read any remaining stdout before closing. + for line in push_proc.stdout: + pass + push_proc.stdout.close() + if (push_proc.wait() != 0) and not repushed: + raise subprocess.CalledProcessError(push_proc.returncode, cmd) + + +class DistroPackageSuite(PackageSuite): + NEED_SSH = True + REMOTE_DEST_DIR = 'tmp' + + def __init__(self, glob_root, rel_globs, target, ssh_host, ssh_opts): + super().__init__(glob_root, rel_globs) + self.target = target + self.ssh_host = ssh_host + self.ssh_opts = ['-o' + opt for opt in ssh_opts] + if not self.logger.isEnabledFor(logging.INFO): + self.ssh_opts.append('-q') + + def _build_cmd(self, base_cmd, *args): + cmd = [base_cmd] + cmd.extend(self.ssh_opts) + cmd.extend(args) + return cmd + + def _paths_basenames(self, paths): + return (os.path.basename(path) for path in paths) + + def _run_script(self, script, *args): + # SSH will use a shell to run our bash command, so we have to + # quote our arguments. + # self.__class__.__name__ provides $0 for the script, which makes a + # nicer message if there's an error. + subprocess.check_call(self._build_cmd( + 'ssh', self.ssh_host, 'bash', '-ec', pipes.quote(script), + self.__class__.__name__, *(pipes.quote(s) for s in args))) + + def upload_files(self, paths): + cmd = self._build_cmd('scp', *paths) + cmd.append('{self.ssh_host}:{self.REMOTE_DEST_DIR}'.format(self=self)) + subprocess.check_call(cmd) + + +class DebianPackageSuite(DistroPackageSuite): + FREIGHT_SCRIPT = """ +cd "$1"; shift +TARGET=$1; shift +freight add "$@" "apt/$TARGET" +freight cache +rm "$@" +""" + + def post_uploads(self, paths): + self._run_script(self.FREIGHT_SCRIPT, self.REMOTE_DEST_DIR, + self.target, *self._paths_basenames(paths)) + + +class RedHatPackageSuite(DistroPackageSuite): + CREATEREPO_SCRIPT = """ +cd "$1"; shift +REPODIR=$1; shift +rpmsign --addsign "$@"