X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/86cc2e4b923ab24ef9a6460efa23904a9f8ed742..fe19ce4b94f7ae3277e9016bc7886179f46ec472:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index 3493f7c..cd83513 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -2,19 +2,13 @@ read -rd "\000" helpmessage <&2 exit 1 ;; - --scp-user) - SCPUSER="$2"; shift - ;; - --scp-host) - SCPHOST="$2"; shift - ;; --target) TARGET="$2"; shift ;; --debug) DEBUG=1 ;; - --upload) - UPLOAD=1 - ;; --build-bundle-packages) BUILD_BUNDLE_PACKAGES=1 ;; @@ -77,15 +59,19 @@ while [ $# -gt 0 ]; do shift done -# Sanity checks -if [[ "$UPLOAD" != '0' && ("$SCPUSER" == '' || "$SCPHOST" == '') ]]; then - echo >&2 "$helpmessage" - echo >&2 - echo >&2 "Error: please specify --scp-user and --scp-host if --upload is set" - echo >&2 - exit 1 +STDOUT_IF_DEBUG=/dev/null +STDERR_IF_DEBUG=/dev/null +DASHQ_UNLESS_DEBUG=-q +if [[ "$DEBUG" != 0 ]]; then + STDOUT_IF_DEBUG=/dev/stdout + STDERR_IF_DEBUG=/dev/stderr + DASHQ_UNLESS_DEBUG= fi +debug_echo () { + echo "$@" >"$STDOUT_IF_DEBUG" +} + declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS PYTHON2_VERSION=2.7 @@ -94,9 +80,6 @@ PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=s case "$TARGET" in debian7) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -110,9 +93,6 @@ case "$TARGET" in ;; debian8) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -126,9 +106,6 @@ case "$TARGET" in ;; ubuntu1204) FORMAT=deb - FPM_OUTDIR=tmp - REPO_UPDATE_CMD='freight add *deb apt/precise && freight cache && rm -f *deb' - PYTHON2_PACKAGE=python$PYTHON2_VERSION PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION @@ -140,11 +117,19 @@ case "$TARGET" in ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; + ubuntu1404) + FORMAT=deb + PYTHON2_PACKAGE=python$PYTHON2_VERSION + PYTHON2_PKG_PREFIX=python + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_PKG_PREFIX=python3 + PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse ciso8601 \ + google-api-python-client six uritemplate oauth2client httplib2 \ + rsa apache-libcloud pycurl backports.ssl_match_hostname) + PYTHON3_BACKPORTS=(docker-py requests websocket-client) + ;; centos6) FORMAT=rpm - FPM_OUTDIR=rpm - REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/' - PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') @@ -211,50 +196,55 @@ if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then exit 1 # fail fi -if [[ "$DEBUG" != 0 ]]; then - echo "$0 is running from $RUN_BUILD_PACKAGES_PATH" - echo "Workspace is $WORKSPACE" -fi +debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH" +debug_echo "Workspace is $WORKSPACE" + +format_last_commit_here() { + local format=$1; shift + TZ=UTC git log -n1 --first-parent "--format=format:$format" . +} version_from_git() { # Generates a version number from the git log for the current working # directory, and writes it to stdout. local git_ts git_hash - declare $(TZ=UTC git log -n1 --first-parent \ - --format=format:"git_ts=%ct git_hash=%h" .) + declare $(format_last_commit_here "git_ts=%ct git_hash=%h") echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash" } +nohash_version_from_git() { + version_from_git | cut -d. -f1-3 +} + timestamp_from_git() { - # Generates a version number from the git log for the current working - # directory, and writes it to stdout. - local git_ts git_hash - declare $(TZ=UTC git log -n1 --first-parent \ - --format=format:"git_ts=%ct git_hash=%h" .) - echo "$git_ts" + format_last_commit_here "%ct" } handle_python_package () { # This function assumes the current working directory is the python package directory - if [[ "$UPLOAD" != 0 ]]; then - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - if [[ "$DEBUG" != 0 ]]; then - python setup.py sdist upload - else - python setup.py -q sdist upload - fi - else - # Make sure only to use sdist - that's the only format pip can deal with (sigh) - if [[ "$DEBUG" != 0 ]]; then - python setup.py sdist - else - python setup.py -q sdist - fi + if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then + # This package doesn't need rebuilding. + return fi + # Make sure only to use sdist - that's the only format pip can deal with (sigh) + python setup.py $DASHQ_UNLESS_DEBUG sdist +} + +handle_ruby_gem() { + local gem_name=$1; shift + local gem_version=$(nohash_version_from_git) + local gem_src_dir="$(pwd)" + + if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then + find -maxdepth 1 -name "${gem_name}-*.gem" -delete + + # -q appears to be broken in gem version 2.2.2 + $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG" + fi } # Build packages for everything -fpm_build_and_scp () { +fpm_build () { # The package source. Depending on the source type, this can be a # path, or the name of the package in an upstream repository (e.g., # pip). @@ -318,22 +308,37 @@ fpm_build_and_scp () { COMMAND_ARR+=("$i") done + # Append --depends X and other arguments specified by fpm-info.sh in + # the package source dir. These are added last so they can override + # the arguments added by this script. + declare -a fpm_args=() + declare -a fpm_depends=() + if [[ -d "$PACKAGE" ]]; then + FPM_INFO="$PACKAGE/fpm-info.sh" + else + FPM_INFO="${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE}/fpm-info.sh" + fi + if [[ -e "$FPM_INFO" ]]; then + debug_echo "Loading fpm overrides from $FPM_INFO" + source "$FPM_INFO" + fi + for i in "${fpm_depends[@]}"; do + COMMAND_ARR+=('--depends' "$i") + done + COMMAND_ARR+=("${fpm_args[@]}") + COMMAND_ARR+=("$PACKAGE") - if [[ "$DEBUG" != 0 ]]; then - echo - echo "${COMMAND_ARR[@]}" - echo - fi + debug_echo -e "\n${COMMAND_ARR[@]}\n" FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS } -# verify build results and scp packages, if needed -fpm_verify_and_scp () { +# verify build results +fpm_verify () { FPM_EXIT_CODE=$1 shift FPM_RESULTS=$@ @@ -349,24 +354,18 @@ fpm_verify_and_scp () { echo echo $FPM_RESULTS echo - else - if [[ ! $FPM_RESULTS =~ "File already exists" ]]; then - if [[ "$FPM_EXIT_CODE" != "0" ]]; then - echo "Error building package for $1:\n $FPM_RESULTS" - else - if [[ "$UPLOAD" != 0 ]]; then - scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/ - CALL_FREIGHT=1 - fi - fi - else - echo "Package $FPM_PACKAGE_NAME exists, not rebuilding" - fi + elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then + echo "Package $FPM_PACKAGE_NAME exists, not rebuilding" + elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then + echo "Error building package for $1:\n $FPM_RESULTS" fi } if [[ -f /etc/profile.d/rvm.sh ]]; then - source /etc/profile.d/rvm.sh + source /etc/profile.d/rvm.sh + GEM="rvm-exec default gem" +else + GEM=gem fi # Make all files world-readable -- jenkins runs with umask 027, and has checked @@ -380,171 +379,81 @@ find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {} # gems and packages umask 0022 -if [[ "$DEBUG" != 0 ]]; then - echo "umask is" `umask` -fi +debug_echo "umask is" `umask` if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then mkdir -p $WORKSPACE/packages/$TARGET fi # Perl packages -if [[ "$DEBUG" != 0 ]]; then - echo -e "\nPerl packages\n" -fi - -if [[ "$DEBUG" != 0 ]]; then - PERL_OUT=/dev/stdout -else - PERL_OUT=/dev/null -fi +debug_echo -e "\nPerl packages\n" cd "$WORKSPACE/sdk/perl" if [[ -e Makefile ]]; then - make realclean >"$PERL_OUT" + make realclean >"$STDOUT_IF_DEBUG" fi find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ -delete rm -rf install -perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \ - make install INSTALLDIRS=perl >"$PERL_OUT" && \ - fpm_build_and_scp install/lib/=/usr/share libarvados-perl \ +perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ + make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ + fpm_build install/lib/=/usr/share libarvados-perl \ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" # Ruby gems -if [[ "$DEBUG" != 0 ]]; then - echo - echo "Ruby gems" - echo -fi - -if type rvm-exec >/dev/null 2>&1; then - FPM_GEM_PREFIX=$(rvm-exec system gem environment gemdir) -else - FPM_GEM_PREFIX=$(gem environment gemdir) -fi +debug_echo -e "\nRuby gems\n" -cd "$WORKSPACE" -cd sdk/ruby +FPM_GEM_PREFIX=$($GEM environment gemdir) -ARVADOS_GEM_EPOCH=`git log -n1 --first-parent --format=%ct` -ARVADOS_GEM_DATE=`date --utc --date="@${ARVADOS_GEM_EPOCH}" +%Y%m%d%H%M%S` -ARVADOS_GEM_VERSION="0.1.${ARVADOS_GEM_DATE}" +cd "$WORKSPACE/sdk/ruby" +handle_ruby_gem arvados -# see if this gem needs building/uploading -gem search arvados -r -a |grep -q $ARVADOS_GEM_VERSION +cd "$WORKSPACE/sdk/cli" +handle_ruby_gem arvados-cli -if [[ "$?" != "0" ]]; then - # clean up old packages - find -maxdepth 1 \( -name 'arvados-*.gem' -or -name 'rubygem-arvados_*.deb' -or -name 'rubygem-arvados_*.rpm' \) \ - -delete - - if [[ "$DEBUG" != 0 ]]; then - gem build arvados.gemspec - else - # -q appears to be broken in gem version 2.2.2 - gem build arvados.gemspec -q >/dev/null 2>&1 - fi - - if [[ "$UPLOAD" != 0 ]]; then - # publish new gem - gem push arvados-*gem - fi - - fpm_build_and_scp arvados-*.gem "" "Curoverse, Inc." gem "" \ - --prefix "$FPM_GEM_PREFIX" -fi - -# Build arvados-cli GEM -cd "$WORKSPACE" -cd sdk/cli - -ARVADOS_CLI_GEM_EPOCH=`git log -n1 --first-parent --format=%ct` -ARVADOS_CLI_GEM_DATE=`date --utc --date="@${ARVADOS_CLI_GEM_EPOCH}" +%Y%m%d%H%M%S` -ARVADOS_CLI_GEM_VERSION="0.1.${ARVADOS_CLI_GEM_DATE}" - -# see if this gem needs building/uploading -gem search arvados-cli -r -a |grep -q $ARVADOS_GEM_VERSION - -if [[ "$?" != "0" ]]; then - # clean up old gems - rm -f arvados-cli*gem - - if [[ "$DEBUG" != 0 ]]; then - gem build arvados-cli.gemspec - else - # -q appears to be broken in gem version 2.2.2 - gem build arvados-cli.gemspec -q >/dev/null - fi - - if [[ "$UPLOAD" != 0 ]]; then - # publish new gem - gem push arvados-cli*gem - fi -fi +cd "$WORKSPACE/services/login-sync" +handle_ruby_gem arvados-login-sync # Python packages -if [[ "$DEBUG" != 0 ]]; then - echo - echo "Python packages" - echo -fi +debug_echo -e "\nPython packages\n" -cd "$WORKSPACE" +cd "$WORKSPACE/sdk/pam" +handle_python_package -cd sdk/python +cd "$WORKSPACE/sdk/python" handle_python_package -cd ../../services/fuse +cd "$WORKSPACE/services/fuse" handle_python_package -cd ../../services/nodemanager +cd "$WORKSPACE/services/nodemanager" handle_python_package -# Arvados-src -# We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package -if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then - mkdir "$WORKSPACE/src-build-dir" - cd "$WORKSPACE" - if [[ "$DEBUG" != 0 ]]; then - git clone https://github.com/curoverse/arvados.git src-build-dir - else - git clone -q https://github.com/curoverse/arvados.git src-build-dir - fi -fi +# arvados-src +( + set -e -cd "$WORKSPACE/src-build-dir" -# just in case, check out master -if [[ "$DEBUG" != 0 ]]; then - git checkout master - git pull - # go into detached-head state - git checkout `git log --format=format:%h -n1 --first-parent .` -else - git checkout -q master - git pull -q - # go into detached-head state - git checkout -q `git log --format=format:%h -n1 --first-parent .` -fi + cd "$WORKSPACE" + COMMIT_HASH=$(format_last_commit_here "%H") -git log --format=format:%H -n1 --first-parent . > git-commit.version + SRC_BUILD_DIR=$(mktemp -d) + git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" + cd "$SRC_BUILD_DIR" -# Build arvados src deb package -cd "$WORKSPACE" -PKG_VERSION=$(version_from_git) -cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" + # go into detached-head state + git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" + echo "$COMMIT_HASH" >git-commit.version -# clean up, check out master and step away from detached-head state -cd "$WORKSPACE/src-build-dir" -if [[ "$DEBUG" != 0 ]]; then - git checkout master -else - git checkout -q master -fi + cd "$SRC_BUILD_DIR" + PKG_VERSION=$(version_from_git) + cd $WORKSPACE/packages/$TARGET + fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" + + rm -r "$SRC_BUILD_DIR" +) # Keep export GOPATH=$(mktemp -d) @@ -556,7 +465,7 @@ cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/keepstore" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" +fpm_build $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" # Get GO SDK version cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go" @@ -576,7 +485,7 @@ fi go get "git.curoverse.com/arvados.git/services/keepproxy" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" +fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" # datamanager cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager" @@ -591,7 +500,7 @@ fi go get "git.curoverse.com/arvados.git/services/datamanager" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." +fpm_build $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." # arv-git-httpd cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd" @@ -606,14 +515,14 @@ fi go get "git.curoverse.com/arvados.git/services/arv-git-httpd" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." +fpm_build $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." # crunchstat cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" PKG_VERSION=$(version_from_git) go get "git.curoverse.com/arvados.git/services/crunchstat" cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" +fpm_build $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" # The Python SDK # Please resist the temptation to add --no-python-fix-name to the fpm call here @@ -623,47 +532,46 @@ fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curover # whip up a patch and send it upstream, but that will be for another day. Ward, # 2014-05-15 cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" +rm -rf "$WORKSPACE/sdk/python/build" +fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git + +# The PAM module +if [[ $TARGET =~ debian|ubuntu ]]; then + cd $WORKSPACE/packages/$TARGET + rm -rf "$WORKSPACE/sdk/pam/build" + fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python +fi # The FUSE driver # Please see comment about --no-python-fix-name above; we stay consistent and do # not omit the python- prefix first. cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" +rm -rf "$WORKSPACE/services/fuse/build" +fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" # The node manager cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" +rm -rf "$WORKSPACE/services/nodemanager/build" +fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" # The Docker image cleaner cd $WORKSPACE/packages/$TARGET -fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" +rm -rf "$WORKSPACE/services/dockercleaner/build" +fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" # A few dependencies for deppkg in "${PYTHON_BACKPORTS[@]}"; do outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") - fpm_build_and_scp "$deppkg" "$outname" + fpm_build "$deppkg" "$outname" done # Python 3 dependencies for deppkg in "${PYTHON3_BACKPORTS[@]}"; do outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build_and_scp "$deppkg" "$outname" "" python3 + fpm_build "$deppkg" "$outname" "" python3 done -# libpam-arvados -cd $WORKSPACE/sdk/pam -PKG_VERSION=$(version_from_git) -cd $WORKSPACE/packages/$TARGET - -if [[ "$FORMAT" == "deb" ]]; then - fpm_build_and_scp $WORKSPACE/sdk/pam/debian/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/debian/arvados_pam=/etc/default/arvados_pam -#else - # FIXME enable and test once we have the centos pam.d file - #fpm_build_and_scp $WORKSPACE/sdk/pam/centos/shellinabox=/etc/pam.d/shellinabox libpam-arvados 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=Apache License, Version 2.0" "--description=PAM module for Arvados" "--config-files=/etc/default" "-d libpam-python" $WORKSPACE/sdk/pam/arvados_pam.py=/usr/bin/arvados_pam.py $WORKSPACE/sdk/pam/centos/arvados_pam=/etc/default/arvados_pam -fi - # Build the API server package cd "$WORKSPACE/services/api" @@ -675,11 +583,8 @@ if [[ ! -d "$WORKSPACE/services/api/tmp" ]]; then mkdir $WORKSPACE/services/api/tmp fi -BUNDLE_OUTPUT=`bundle install --path vendor/bundle` -if [[ "$DEBUG" != 0 ]]; then - echo $BUNDLE_OUTPUT -fi +bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG" /usr/bin/git rev-parse HEAD > git-commit.version @@ -696,29 +601,21 @@ cd $WORKSPACE/packages/$TARGET if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") - if [[ "$DEBUG" != 0 ]]; then - echo - echo "${COMMAND_ARR[@]}" - echo - fi + debug_echo -e "\n${COMMAND_ARR[@]}\n" FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") -if [[ "$DEBUG" != 0 ]]; then - echo - echo "${COMMAND_ARR[@]}" - echo -fi +debug_echo -e "\n${COMMAND_ARR[@]}\n" FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify $FPM_EXIT_CODE $FPM_RESULTS # API server package build done @@ -733,11 +630,7 @@ if [[ ! -d "$WORKSPACE/apps/workbench/tmp" ]]; then mkdir $WORKSPACE/apps/workbench/tmp fi -BUNDLE_OUTPUT=`bundle install --path vendor/bundle` - -if [[ "$DEBUG" != 0 ]]; then - echo $BUNDLE_OUTPUT -fi +bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG" /usr/bin/git rev-parse HEAD > git-commit.version @@ -766,46 +659,24 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") - if [[ "$DEBUG" != 0 ]]; then - echo - echo "${COMMAND_ARR[@]}" - echo - fi + debug_echo -e "\n${COMMAND_ARR[@]}\n" FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? - fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS + fpm_verify $FPM_EXIT_CODE $FPM_RESULTS fi # Build the 'bare' package without vendor/bundle. declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") -if [[ "$DEBUG" != 0 ]]; then - echo - echo "${COMMAND_ARR[@]}" - echo -fi +debug_echo -e "\n${COMMAND_ARR[@]}\n" FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? -fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS +fpm_verify $FPM_EXIT_CODE $FPM_RESULTS # Workbench package build done - -# Finally, publish the packages, if necessary -if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then - ssh -p2222 $SCPUSER@$SCPHOST -t bash - <