jenkins/run-build-packages.sh

   1 #!/bin/bash
   2
   3
   4 read -rd "\000" helpmessage <<EOF
   5 $(basename $0): Build Arvados packages and (optionally) upload them.
   6
   7 Syntax:
   8         WORKSPACE=/path/to/arvados $(basename $0) [options]
   9
  10 Options:
  11
  12 --upload
  13     Upload packages (default: false)
  14 --scp-user USERNAME
  15     scp user for repository server (only required when --upload is specified)
  16 --scp-host HOSTNAME
  17     scp host for repository server (only required when --upload is specified)
  18 --build-bundle-packages  (default: false)
  19     Build api server and workbench packages with vendor/bundle included
  20 --debug
  21     Output debug information (default: false)
  22 --target
  23     Distribution to build packages for (default: debian7)
  24
  25 WORKSPACE=path         Path to the Arvados source tree to build packages from
  26
  27 EOF
  28
  29 EXITCODE=0
  30 CALL_FREIGHT=0
  31
  32 DEBUG=0
  33 UPLOAD=0
  34 BUILD_BUNDLE_PACKAGES=0
  35 TARGET=debian7
  36
  37 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  38     help,upload,scp-user:,scp-host:,build-bundle-packages,debug,target: \
  39     -- "" "$@")
  40 if [ $? -ne 0 ]; then
  41     exit 1
  42 fi
  43
  44 eval set -- "$PARSEDOPTS"
  45 while [ $# -gt 0 ]; do
  46     case "$1" in
  47         --help)
  48             echo >&2 "$helpmessage"
  49             echo >&2
  50             exit 1
  51             ;;
  52         --scp-user)
  53             SCPUSER="$2"; shift
  54             ;;
  55         --scp-host)
  56             SCPHOST="$2"; shift
  57             ;;
  58         --target)
  59             TARGET="$2"; shift
  60             ;;
  61         --debug)
  62             DEBUG=1
  63             ;;
  64         --upload)
  65             UPLOAD=1
  66             ;;
  67         --build-bundle-packages)
  68             BUILD_BUNDLE_PACKAGES=1
  69             ;;
  70         --)
  71             if [ $# -gt 1 ]; then
  72                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  73                 exit 1
  74             fi
  75             ;;
  76     esac
  77     shift
  78 done
  79
  80 # Sanity checks
  81 if [[ "$UPLOAD" != '0' && ("$SCPUSER" == '' || "$SCPHOST" == '') ]]; then
  82   echo >&2 "$helpmessage"
  83   echo >&2
  84   echo >&2 "Error: please specify --scp-user and --scp-host if --upload is set"
  85   echo >&2
  86   exit 1
  87 fi
  88
  89 declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
  90
  91 PYTHON2_VERSION=2.7
  92 PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
  93
  94 case "$TARGET" in
  95     debian7)
  96         FORMAT=deb
  97         FPM_OUTDIR=tmp
  98         REPO_UPDATE_CMD='freight add *deb apt/wheezy && freight cache && rm -f *deb'
  99
 100         PYTHON2_PACKAGE=python$PYTHON2_VERSION
 101         PYTHON3_PACKAGE=python$PYTHON3_VERSION
 102         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
 103             oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
 104             virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
 105             ciso8601 pycrypto backports.ssl_match_hostname pycurl)
 106         PYTHON3_BACKPORTS=(docker-py six requests)
 107         ;;
 108     centos6)
 109         FORMAT=rpm
 110         FPM_OUTDIR=rpm
 111         REPO_UPDATE_CMD='mv *rpm /var/www/rpm.arvados.org/CentOS/6/os/x86_64/ && createrepo /var/www/rpm.arvados.org/CentOS/6/os/x86_64/'
 112
 113         PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
 114         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
 115         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
 116             oauth2client pyasn1 pyasn1-modules rsa uritemplate httplib2 ws4py \
 117             virtualenv pykka apache-libcloud requests six pyexecjs jsonschema \
 118             ciso8601 pycrypto backports.ssl_match_hostname pycurl)
 119         PYTHON3_BACKPORTS=(docker-py six requests)
 120         ;;
 121     *)
 122         echo -e "$0: Unknown target '$TARGET'.\n" >&2
 123         exit 1
 124         ;;
 125 esac
 126
 127
 128 if ! [[ -n "$WORKSPACE" ]]; then
 129   echo >&2 "$helpmessage"
 130   echo >&2
 131   echo >&2 "Error: WORKSPACE environment variable not set"
 132   echo >&2
 133   exit 1
 134 fi
 135
 136 # Test for fpm
 137 fpm --version >/dev/null 2>&1
 138
 139 if [[ "$?" != 0 ]]; then
 140   echo >&2 "$helpmessage"
 141   echo >&2
 142   echo >&2 "Error: fpm not found"
 143   echo >&2
 144   exit 1
 145 fi
 146
 147 find_easy_install() {
 148     for version_suffix in "$@"; do
 149         if "easy_install$version_suffix" --version >/dev/null 2>&1; then
 150             echo "easy_install$version_suffix"
 151             return 0
 152         fi
 153     done
 154     cat >&2 <<EOF
 155 $helpmessage
 156
 157 Error: easy_install$1 (from Python setuptools module) not found
 158
 159 EOF
 160     exit 1
 161 }
 162
 163 EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
 164 EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
 165
 166 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 167 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 168 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 169   # error; for some reason, the path is not accessible
 170   # to the script (e.g. permissions re-evaled after suid)
 171   exit 1  # fail
 172 fi
 173
 174 if [[ "$DEBUG" != 0 ]]; then
 175   echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 176   echo "Workspace is $WORKSPACE"
 177 fi
 178
 179 version_from_git() {
 180   # Generates a version number from the git log for the current working
 181   # directory, and writes it to stdout.
 182   local git_ts git_hash
 183   declare $(TZ=UTC git log -n1 --first-parent --max-count=1 \
 184       --format=format:"git_ts=%ct git_hash=%h" .)
 185   echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
 186 }
 187
 188 timestamp_from_git() {
 189   # Generates a version number from the git log for the current working
 190   # directory, and writes it to stdout.
 191   local git_ts git_hash
 192   declare $(TZ=UTC git log -n1 --first-parent --max-count=1 \
 193       --format=format:"git_ts=%ct git_hash=%h" .)
 194   echo "$git_ts"
 195 }
 196
 197 handle_python_package () {
 198   # This function assumes the current working directory is the python package directory
 199   if [[ "$UPLOAD" != 0 ]]; then
 200     # Make sure only to use sdist - that's the only format pip can deal with (sigh)
 201     if [[ "$DEBUG" != 0 ]]; then
 202       python setup.py sdist upload
 203     else
 204       python setup.py -q sdist upload
 205     fi
 206   else
 207     # Make sure only to use sdist - that's the only format pip can deal with (sigh)
 208     if [[ "$DEBUG" != 0 ]]; then
 209       python setup.py sdist
 210     else
 211       python setup.py -q sdist
 212     fi
 213   fi
 214 }
 215
 216 # Build debs for everything
 217 fpm_build_and_scp () {
 218   # The package source.  Depending on the source type, this can be a
 219   # path, or the name of the package in an upstream repository (e.g.,
 220   # pip).
 221   PACKAGE=$1
 222   shift
 223   # The name of the package to build.  Defaults to $PACKAGE.
 224   PACKAGE_NAME=${1:-$PACKAGE}
 225   shift
 226   # Optional: the vendor of the package.  Should be "Curoverse, Inc." for
 227   # packages of our own software.  Passed to fpm --vendor.
 228   VENDOR=$1
 229   shift
 230   # The type of source package.  Passed to fpm -s.  Default "python".
 231   PACKAGE_TYPE=${1:-python}
 232   shift
 233   # Optional: the package version number.  Passed to fpm -v.
 234   VERSION=$1
 235   shift
 236
 237   case "$PACKAGE_TYPE" in
 238       python)
 239           # All Arvados Python2 packages depend on Python 2.7.
 240           # Make sure we build with that for consistency.
 241           set -- "$@" --python-bin python2.7 \
 242               --python-easyinstall "$EASY_INSTALL2"
 243           ;;
 244       python3)
 245           # fpm does not actually support a python3 package type.  Instead
 246           # we recognize it as a convenience shortcut to add several
 247           # necessary arguments to fpm's command line later, after we're
 248           # done handling positional arguments.
 249           PACKAGE_TYPE=python
 250           set -- "$@" --python-bin python3 \
 251               --python-easyinstall "$EASY_INSTALL3" \
 252               --python-package-name-prefix python3 --depends "$PYTHON3_PACKAGE"
 253           ;;
 254   esac
 255
 256   declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT" "-x" "usr/local/lib/python2.7/dist-packages/tests")
 257
 258   if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
 259     COMMAND_ARR+=('-n' "$PACKAGE_NAME")
 260   fi
 261
 262   if [[ "$VENDOR" != "" ]]; then
 263     COMMAND_ARR+=('--vendor' "$VENDOR")
 264   fi
 265
 266   if [[ "$VERSION" != "" ]]; then
 267     COMMAND_ARR+=('-v' "$VERSION")
 268   fi
 269
 270   # Append remaining function arguments directly to fpm's command line.
 271   for i; do
 272     COMMAND_ARR+=("$i")
 273   done
 274
 275   COMMAND_ARR+=("$PACKAGE")
 276
 277   if [[ "$DEBUG" != 0 ]]; then
 278     echo
 279     echo "${COMMAND_ARR[@]}"
 280     echo
 281   fi
 282
 283   FPM_RESULTS=$("${COMMAND_ARR[@]}")
 284   FPM_EXIT_CODE=$?
 285
 286   fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
 287 }
 288
 289 # verify build results and scp debs, if needed
 290 fpm_verify_and_scp () {
 291   FPM_EXIT_CODE=$1
 292   shift
 293   FPM_RESULTS=$@
 294
 295   FPM_PACKAGE_NAME=''
 296   if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.)(deb|rpm) ]]; then
 297     FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
 298   fi
 299
 300   if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
 301     EXITCODE=1
 302     echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
 303     echo
 304     echo $FPM_RESULTS
 305     echo
 306   else
 307     if [[ ! $FPM_RESULTS =~ "File already exists" ]]; then
 308       if [[ "$FPM_EXIT_CODE" != "0" ]]; then
 309         echo "Error building package for $1:\n $FPM_RESULTS"
 310       else
 311         if [[ "$UPLOAD" != 0 ]]; then
 312           scp -P2222 $FPM_PACKAGE_NAME $SCPUSER@$SCPHOST:$FPM_OUTDIR/
 313           CALL_FREIGHT=1
 314         fi
 315       fi
 316     else
 317       echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
 318     fi
 319   fi
 320 }
 321
 322 if [[ -f /etc/profile.d/rvm.sh ]]; then
 323   source /etc/profile.d/rvm.sh
 324 fi
 325
 326 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 327 # out our git tree here
 328 chmod o+r "$WORKSPACE" -R
 329
 330 # More cleanup - make sure all executables that we'll package are 755
 331 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 332
 333 # Now fix our umask to something better suited to building and publishing
 334 # gems and packages
 335 umask 0022
 336
 337 if [[ "$DEBUG" != 0 ]]; then
 338   echo "umask is" `umask`
 339 fi
 340
 341 if [[ ! -d "$WORKSPACE/debs" ]]; then
 342   mkdir -p $WORKSPACE/debs
 343 fi
 344
 345 # Perl packages
 346 if [[ "$DEBUG" != 0 ]]; then
 347   echo -e "\nPerl packages\n"
 348 fi
 349
 350 if [[ "$DEBUG" != 0 ]]; then
 351   PERL_OUT=/dev/stdout
 352 else
 353   PERL_OUT=/dev/null
 354 fi
 355
 356 cd "$WORKSPACE/sdk/perl"
 357
 358 if [[ -e Makefile ]]; then
 359   make realclean >"$PERL_OUT"
 360 fi
 361 find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
 362     -delete
 363 rm -rf install
 364
 365 perl Makefile.PL INSTALL_BASE=install >"$PERL_OUT" && \
 366     make install INSTALLDIRS=perl >"$PERL_OUT" && \
 367     fpm_build_and_scp install/lib/=/usr/share libarvados-perl \
 368     "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \
 369     mv libarvados-perl*.$FORMAT "$WORKSPACE/debs/"
 370
 371 # Ruby gems
 372 if [[ "$DEBUG" != 0 ]]; then
 373   echo
 374   echo "Ruby gems"
 375   echo
 376 fi
 377
 378 if type rvm-exec >/dev/null 2>&1; then
 379   FPM_GEM_PREFIX=$(rvm-exec system gem environment gemdir)
 380 else
 381   FPM_GEM_PREFIX=$(gem environment gemdir)
 382 fi
 383
 384 cd "$WORKSPACE"
 385 cd sdk/ruby
 386
 387 ARVADOS_GEM_EPOCH=`git log -n1 --first-parent --format=%ct`
 388 ARVADOS_GEM_DATE=`date --utc --date="@${ARVADOS_GEM_EPOCH}" +%Y%m%d%H%M%S`
 389 ARVADOS_GEM_VERSION="0.1.${ARVADOS_GEM_DATE}"
 390
 391 # see if this gem needs building/uploading
 392 gem search arvados -r -a |grep -q $ARVADOS_GEM_VERSION
 393
 394 if [[ "$?" != "0" ]]; then
 395   # clean up old packages
 396   find -maxdepth 1 \( -name 'arvados-*.gem' -or -name 'rubygem-arvados_*.deb' -or -name 'rubygem-arvados_*.rpm' \) \
 397       -delete
 398
 399   if [[ "$DEBUG" != 0 ]]; then
 400     gem build arvados.gemspec
 401   else
 402     # -q appears to be broken in gem version 2.2.2
 403     gem build arvados.gemspec -q >/dev/null 2>&1
 404   fi
 405
 406   if [[ "$UPLOAD" != 0 ]]; then
 407     # publish new gem
 408     gem push arvados-*gem
 409   fi
 410
 411   fpm_build_and_scp arvados-*.gem "" "Curoverse, Inc." gem "" \
 412       --prefix "$FPM_GEM_PREFIX"
 413 fi
 414
 415 # Build arvados-cli GEM
 416 cd "$WORKSPACE"
 417 cd sdk/cli
 418
 419 ARVADOS_CLI_GEM_EPOCH=`git log -n1 --first-parent --format=%ct`
 420 ARVADOS_CLI_GEM_DATE=`date --utc --date="@${ARVADOS_CLI_GEM_EPOCH}" +%Y%m%d%H%M%S`
 421 ARVADOS_CLI_GEM_VERSION="0.1.${ARVADOS_CLI_GEM_DATE}"
 422
 423 # see if this gem needs building/uploading
 424 gem search arvados-cli -r -a |grep -q $ARVADOS_GEM_VERSION
 425
 426 if [[ "$?" != "0" ]]; then
 427   # clean up old gems
 428   rm -f arvados-cli*gem
 429
 430   if [[ "$DEBUG" != 0 ]]; then
 431     gem build arvados-cli.gemspec
 432   else
 433     # -q appears to be broken in gem version 2.2.2
 434     gem build arvados-cli.gemspec -q >/dev/null
 435   fi
 436
 437   if [[ "$UPLOAD" != 0 ]]; then
 438     # publish new gem
 439     gem push arvados-cli*gem
 440   fi
 441 fi
 442
 443 # Python packages
 444 if [[ "$DEBUG" != 0 ]]; then
 445   echo
 446   echo "Python packages"
 447   echo
 448 fi
 449
 450 cd "$WORKSPACE"
 451
 452 cd sdk/python
 453 handle_python_package
 454
 455 cd ../../services/fuse
 456 handle_python_package
 457
 458 cd ../../services/nodemanager
 459 handle_python_package
 460
 461 # Arvados-src
 462 # We use $WORKSPACE/src-build-dir as the clean directory from which to build the src package
 463 if [[ ! -d "$WORKSPACE/src-build-dir" ]]; then
 464   mkdir "$WORKSPACE/src-build-dir"
 465   cd "$WORKSPACE"
 466   if [[ "$DEBUG" != 0 ]]; then
 467     git clone https://github.com/curoverse/arvados.git src-build-dir
 468   else
 469     git clone -q https://github.com/curoverse/arvados.git src-build-dir
 470   fi
 471 fi
 472
 473 cd "$WORKSPACE/src-build-dir"
 474 # just in case, check out master
 475 if [[ "$DEBUG" != 0 ]]; then
 476   git checkout master
 477   git pull
 478   # go into detached-head state
 479   git checkout `git log --format=format:%h -n1 .`
 480 else
 481   git checkout -q master
 482   git pull -q
 483   # go into detached-head state
 484   git checkout -q `git log --format=format:%h -n1 .`
 485 fi
 486
 487 git log --format=format:%H -n1 . > git-commit.version
 488
 489 # Build arvados src deb package
 490 cd "$WORKSPACE"
 491 PKG_VERSION=$(version_from_git)
 492 cd $WORKSPACE/debs
 493 fpm_build_and_scp $WORKSPACE/src-build-dir/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
 494
 495 # clean up, check out master and step away from detached-head state
 496 cd "$WORKSPACE/src-build-dir"
 497 if [[ "$DEBUG" != 0 ]]; then
 498   git checkout master
 499 else
 500   git checkout -q master
 501 fi
 502
 503 # Keep
 504 export GOPATH=$(mktemp -d)
 505 mkdir -p "$GOPATH/src/git.curoverse.com"
 506 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
 507
 508 # keepstore
 509 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore"
 510 PKG_VERSION=$(version_from_git)
 511 go get "git.curoverse.com/arvados.git/services/keepstore"
 512 cd $WORKSPACE/debs
 513 fpm_build_and_scp $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN"
 514
 515 # Get GO SDK version
 516 cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
 517 GO_SDK_VERSION=$(version_from_git)
 518 GO_SDK_TIMESTAMP=$(timestamp_from_git)
 519
 520 # keepproxy
 521 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepproxy"
 522 KEEPPROXY_VERSION=$(version_from_git)
 523 KEEPPROXY_TIMESTAMP=$(timestamp_from_git)
 524
 525 if [[ "$GO_SDK_TIMESTAMP" -gt "$KEEPPROXY_TIMESTAMP" ]]; then
 526   PKG_VERSION=$GO_SDK_VERSION
 527 else
 528   PKG_VERSION=$KEEPPROXY_VERSION
 529 fi
 530
 531 go get "git.curoverse.com/arvados.git/services/keepproxy"
 532 cd $WORKSPACE/debs
 533 fpm_build_and_scp $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
 534
 535 # datamanager
 536 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager"
 537 DATAMANAGER_VERSION=$(version_from_git)
 538 DATAMANAGER_TIMESTAMP=$(timestamp_from_git)
 539
 540 if [[ "$GO_SDK_TIMESTAMP" -gt "$DATAMANAGER_TIMESTAMP" ]]; then
 541   PKG_VERSION=$GO_SDK_VERSION
 542 else
 543   PKG_VERSION=$DATAMANAGER_VERSION
 544 fi
 545
 546 go get "git.curoverse.com/arvados.git/services/datamanager"
 547 cd $WORKSPACE/debs
 548 fpm_build_and_scp $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
 549
 550 # arv-git-httpd
 551 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd"
 552 ARVGITHTTPD_VERSION=$(version_from_git)
 553 ARVGITHTTPD_TIMESTAMP=$(timestamp_from_git)
 554
 555 if [[ "$GO_SDK_TIMESTAMP" -gt "$ARVGITHTTPD_TIMESTAMP" ]]; then
 556   PKG_VERSION=$GO_SDK_VERSION
 557 else
 558   PKG_VERSION=$ARVGITHTTPD_VERSION
 559 fi
 560
 561 go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
 562 cd $WORKSPACE/debs
 563 fpm_build_and_scp $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
 564
 565 # crunchstat
 566 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
 567 PKG_VERSION=$(version_from_git)
 568 go get "git.curoverse.com/arvados.git/services/crunchstat"
 569 cd $WORKSPACE/debs
 570 fpm_build_and_scp $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs"
 571
 572 # The Python SDK
 573 # Please resist the temptation to add --no-python-fix-name to the fpm call here
 574 # (which would remove the python- prefix from the package name), because this
 575 # package is a dependency of arvados-fuse, and fpm can not omit the python-
 576 # prefix from only one of the dependencies of a package...  Maybe I could
 577 # whip up a patch and send it upstream, but that will be for another day. Ward,
 578 # 2014-05-15
 579 cd $WORKSPACE/debs
 580 fpm_build_and_scp $WORKSPACE/sdk/python python-arvados-python-client 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends="$PYTHON2_PACKAGE"
 581
 582 # The FUSE driver
 583 # Please see comment about --no-python-fix-name above; we stay consistent and do
 584 # not omit the python- prefix first.
 585 cd $WORKSPACE/debs
 586 fpm_build_and_scp $WORKSPACE/services/fuse python-arvados-fuse 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends="$PYTHON2_PACKAGE"
 587
 588 # The node manager
 589 cd $WORKSPACE/debs
 590 fpm_build_and_scp $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends="$PYTHON2_PACKAGE"
 591
 592 # The Docker image cleaner
 593 cd $WORKSPACE/debs
 594 fpm_build_and_scp $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
 595
 596 # A few dependencies
 597 for deppkg in "${PYTHON_BACKPORTS[@]}"; do
 598     fpm_build_and_scp "$deppkg"
 599 done
 600
 601 # Python 3 dependencies
 602 for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
 603     # The empty string is the vendor argument: these aren't Curoverse software.
 604     fpm_build_and_scp "$deppkg" "python3-$deppkg" "" python3
 605 done
 606
 607 # Build the API server package
 608
 609 cd "$WORKSPACE/services/api"
 610
 611 API_VERSION=$(version_from_git)
 612 PACKAGE_NAME=arvados-api-server
 613
 614 if [[ ! -d "$WORKSPACE/services/api/tmp" ]]; then
 615   mkdir $WORKSPACE/services/api/tmp
 616 fi
 617
 618 BUNDLE_OUTPUT=`bundle install --path vendor/bundle`
 619
 620 if [[ "$DEBUG" != 0 ]]; then
 621   echo $BUNDLE_OUTPUT
 622 fi
 623
 624 /usr/bin/git rev-parse HEAD > git-commit.version
 625
 626 cd $WORKSPACE/debs
 627
 628 # Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now,
 629 # we do that in the upgrade script.
 630 # TODO: add bogus database.yml file so we can precompile the assets and put them in the
 631 # package. Then remove that database.yml file again. It has to be a valid file though.
 632 #RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile
 633
 634 # This is the complete package with vendor/bundle included.
 635 # It's big, so we do not build it by default.
 636 if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
 637   declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
 638
 639   if [[ "$DEBUG" != 0 ]]; then
 640     echo
 641     echo "${COMMAND_ARR[@]}"
 642     echo
 643   fi
 644
 645   FPM_RESULTS=$("${COMMAND_ARR[@]}")
 646   FPM_EXIT_CODE=$?
 647   fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
 648 fi
 649
 650 # Build the 'bare' package without vendor/bundle.
 651 declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
 652
 653 if [[ "$DEBUG" != 0 ]]; then
 654   echo
 655   echo "${COMMAND_ARR[@]}"
 656   echo
 657 fi
 658
 659 FPM_RESULTS=$("${COMMAND_ARR[@]}")
 660 FPM_EXIT_CODE=$?
 661 fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
 662
 663 # API server package build done
 664
 665 # Build the workbench server package
 666
 667 cd "$WORKSPACE/apps/workbench"
 668
 669 WORKBENCH_VERSION=$(version_from_git)
 670 PACKAGE_NAME=arvados-workbench
 671
 672 if [[ ! -d "$WORKSPACE/apps/workbench/tmp" ]]; then
 673   mkdir $WORKSPACE/apps/workbench/tmp
 674 fi
 675
 676 BUNDLE_OUTPUT=`bundle install --path vendor/bundle`
 677
 678 if [[ "$DEBUG" != 0 ]]; then
 679   echo $BUNDLE_OUTPUT
 680 fi
 681
 682 /usr/bin/git rev-parse HEAD > git-commit.version
 683
 684 # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
 685 # and we want that in the package, so it's easier to not exclude the tmp directory
 686 # from the package - empty it instead.
 687 rm -rf $WORKSPACE/apps/workbench/tmp/*
 688
 689 # Set up application.yml so that asset precompilation works
 690 \cp config/application.yml.example config/application.yml -f
 691 sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
 692
 693 RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
 694
 695 if [[ "$?" != "0" ]]; then
 696   echo "ERROR: Asset precompilation failed"
 697   EXITCODE=1
 698 fi
 699
 700 cd $WORKSPACE/debs
 701
 702 # This is the complete package with vendor/bundle included.
 703 # It's big, so we do not build it by default.
 704 if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
 705
 706   declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
 707
 708   if [[ "$DEBUG" != 0 ]]; then
 709     echo
 710     echo "${COMMAND_ARR[@]}"
 711     echo
 712   fi
 713
 714   FPM_RESULTS=$("${COMMAND_ARR[@]}")
 715   FPM_EXIT_CODE=$?
 716   fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
 717 fi
 718
 719 # Build the 'bare' package without vendor/bundle.
 720
 721 declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
 722
 723 if [[ "$DEBUG" != 0 ]]; then
 724   echo
 725   echo "${COMMAND_ARR[@]}"
 726   echo
 727 fi
 728
 729 FPM_RESULTS=$("${COMMAND_ARR[@]}")
 730 FPM_EXIT_CODE=$?
 731 fpm_verify_and_scp $FPM_EXIT_CODE $FPM_RESULTS
 732
 733 # Workbench package build done
 734
 735 # Finally, publish the packages, if necessary
 736 if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then
 737   ssh -p2222 $SCPUSER@$SCPHOST -t bash - <<EOF
 738 if [ -n "\$(find -name "$FPM_OUTDIR/*.$FORMAT" -print -quit)" ]; then
 739     cd "$FPM_OUTDIR" && $REPO_UPDATE_CMD
 740 fi
 741 EOF
 742 else
 743   if [[ "$UPLOAD" != 0 ]]; then
 744     echo "No new packages generated. No freight run necessary."
 745   fi
 746 fi
 747
 748 # clean up temporary GOPATH
 749 rm -rf "$GOPATH"
 750
 751 exit $EXITCODE