From 6c26aa2c194f60b2823da49166741203524a0b48 Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Wed, 3 Jun 2015 16:27:48 -0400 Subject: [PATCH] Build API server and Workbench debian packages. Refs #6096 --- .../arvados-api-server-upgrade.sh | 65 ++++++++ jenkins/arvados-api-server-extras/postinst.sh | 12 ++ .../arvados-workbench-upgrade.sh | 57 +++++++ jenkins/arvados-workbench-extras/postinst.sh | 12 ++ jenkins/run-build-packages.sh | 154 +++++++++++++++++- 5 files changed, 294 insertions(+), 6 deletions(-) create mode 100755 jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh create mode 100644 jenkins/arvados-api-server-extras/postinst.sh create mode 100755 jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh create mode 100644 jenkins/arvados-workbench-extras/postinst.sh diff --git a/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh b/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh new file mode 100755 index 0000000..cf32fdb --- /dev/null +++ b/jenkins/arvados-api-server-extras/arvados-api-server-upgrade.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +RELEASE_PATH=/var/www/arvados-api/current +SHARED_PATH=/var/www/arvados-api/shared +CONFIG_PATH=/etc/arvados/api/ + +echo "Assumption: nginx is configured to serve `hostname` from /var/www/`hostname`/current" +echo "Assumption: /var/www/`hostname` is symlinked to /var/www/arvados-api" +echo "Assumption: configuration files are in /etc/arvados/api/" +echo "Assumption: nginx and passenger run as the www-data user" +echo + +echo "Copying files from $CONFIG_PATH" +cp -f $CONFIG_PATH/database.yml $RELEASE_PATH/config/database.yml +cp -f $RELEASE_PATH/config/environments/production.rb.example $RELEASE_PATH/config/environments/production.rb +cp -f $CONFIG_PATH/application.yml $RELEASE_PATH/config/application.yml +cp -f $CONFIG_PATH/omniauth.rb $RELEASE_PATH/config/initializers/omniauth.rb +echo "Done." + +# Before we do anything else, make sure some directories and files are in place +if [[ ! -e $SHARED_PATH/log ]]; then mkdir -p $SHARED_PATH/log; fi +if [[ ! -e $RELEASE_PATH/tmp ]]; then mkdir -p $RELEASE_PATH/tmp; fi +if [[ ! -e $RELEASE_PATH/log ]]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi +if [[ ! -e $SHARED_PATH/log/production.log ]]; then touch $SHARED_PATH/log/production.log; fi + +echo "Running bundle install" +(cd $RELEASE_PATH && RAILS_ENV=production bundle install --path $SHARED_PATH/vendor_bundle) +echo "Done." + +echo "Precompiling assets" +# precompile assets; thankfully this does not take long +(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake assets:precompile) +echo "Done." + +echo "Ensuring directory and file permissions" +# Ensure correct ownership of a few files +chown www-data:www-data $RELEASE_PATH/config/environment.rb +chown www-data:www-data $RELEASE_PATH/config.ru +chown www-data:www-data $RELEASE_PATH/config/database.yml +chown www-data:www-data $RELEASE_PATH/Gemfile.lock +chown -R www-data:www-data $RELEASE_PATH/tmp +chown -R www-data:www-data $SHARED_PATH/log +chown www-data:www-data $RELEASE_PATH/db/structure.sql +chmod 644 $SHARED_PATH/log/* +echo "Done." + +echo "Running sanity check" +(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake config:check) +echo "Done." + +SANITY_CHECK_EXIT_CODE=$? + +if [[ "$SANITY_CHECK_EXIT_CODE" != "0" ]]; then + echo "Sanity check failed, aborting. Please roll back to the previous version of the package." + echo "The database has not been migrated yet, so reinstalling the previous version is safe." + exit $SANITY_CHECK_EXIT_CODE +fi + +echo "Starting db:migrate" +(cd $RELEASE_PATH && bundle exec rake RAILS_ENV=production db:migrate) +echo "Done." + +echo "Restarting nginx" +service nginx restart +echo "Done." diff --git a/jenkins/arvados-api-server-extras/postinst.sh b/jenkins/arvados-api-server-extras/postinst.sh new file mode 100644 index 0000000..1aeffe9 --- /dev/null +++ b/jenkins/arvados-api-server-extras/postinst.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd /var/www/arvados-api + +chown -R www-data:www-data tmp >/dev/null 2>&1 +chown -R www-data:www-data log >/dev/null 2>&1 +chown www-data:www-data db/structure.sql >/dev/null 2>&1 +chmod 644 log/* >/dev/null 2>&1 + +# Errors above are not serious +exit 0 + diff --git a/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh b/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh new file mode 100755 index 0000000..33e05d3 --- /dev/null +++ b/jenkins/arvados-workbench-extras/arvados-workbench-upgrade.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +RELEASE_PATH=/var/www/arvados-workbench/current +SHARED_PATH=/var/www/arvados-workbench/shared +CONFIG_PATH=/etc/arvados/workbench/ + +echo "Assumption: nginx is configured to serve workbench.`hostname` from /var/www/workbench.`hostname`/current" +echo "Assumption: /var/www/`hostname` is symlinked to /var/www/arvados-workbench" +echo "Assumption: configuration files are in /etc/arvados/workbench/" +echo "Assumption: nginx and passenger run as the www-data user" +echo + +echo "Copying files from $CONFIG_PATH" +cp -f $CONFIG_PATH/application.yml $RELEASE_PATH/config/application.yml +cp -f $RELEASE_PATH/config/environments/production.rb.example $RELEASE_PATH/config/environments/production.rb +echo "Done." + +# Before we do anything else, make sure some directories and files are in place +if [[ ! -e $SHARED_PATH/log ]]; then mkdir -p $SHARED_PATH/log; fi +if [[ ! -e $RELEASE_PATH/tmp ]]; then mkdir -p $RELEASE_PATH/tmp; fi +if [[ ! -e $RELEASE_PATH/log ]]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi +if [[ ! -e $SHARED_PATH/log/production.log ]]; then touch $SHARED_PATH/log/production.log; fi + +echo "Running bundle install" +(cd $RELEASE_PATH && RAILS_ENV=production bundle install --path $SHARED_PATH/vendor_bundle) +echo "Done." + +# We do not need to precompile assets, they are already part of the package. + +echo "Ensuring directory and file permissions" +chown www-data:www-data $RELEASE_PATH/config/environment.rb +chown www-data:www-data $RELEASE_PATH/config.ru +chown www-data:www-data $RELEASE_PATH/config/database.yml +chown www-data:www-data $RELEASE_PATH/Gemfile.lock +chown -R www-data:www-data $RELEASE_PATH/tmp +chown -R www-data:www-data $SHARED_PATH/log +chown www-data:www-data $RELEASE_PATH/db/schema.rb +chmod 644 $SHARED_PATH/log/* +echo "Done." + +echo "Running sanity check" +(cd $RELEASE_PATH; RAILS_ENV=production bundle exec rake config:check) +echo "Done." + +SANITY_CHECK_EXIT_CODE=$? + +if [[ "$SANITY_CHECK_EXIT_CODE" != "0" ]]; then + echo "Sanity check failed, aborting. Please roll back to the previous version of the package." + exit $SANITY_CHECK_EXIT_CODE +fi + +# We do not need to run db:migrate because Workbench is stateless + +echo "Restarting nginx" +service nginx restart +echo "Done." + diff --git a/jenkins/arvados-workbench-extras/postinst.sh b/jenkins/arvados-workbench-extras/postinst.sh new file mode 100644 index 0000000..d0aa9b0 --- /dev/null +++ b/jenkins/arvados-workbench-extras/postinst.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd /var/www/arvados-workbench + +chown -R www-data:www-data tmp >/dev/null 2>&1 +chown -R www-data:www-data log >/dev/null 2>&1 +chown www-data:www-data db/schema.rb >/dev/null 2>&1 +chmod 644 log/* >/dev/null 2>&1 + +# Errors above are not serious +exit 0 + diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index 691e93b..c5ae462 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -9,10 +9,16 @@ Syntax: Options: ---upload Upload packages (default: false) ---scp-user USERNAME Scp user for apt server (only required when --upload is specified) ---apt-server HOSTNAME Apt server hostname (only required when --upload is specified) ---debug Output debug information (default: false) +--upload + Upload packages (default: false) +--scp-user USERNAME + Scp user for apt server (only required when --upload is specified) +--apt-server HOSTNAME + Apt server hostname (only required when --upload is specified) +--build-bundle-packages (default: false) + Build api server and workbench packages with vendor/bundle included +--debug + Output debug information (default: false) WORKSPACE=path Path to the Arvados source tree to build packages from @@ -45,6 +51,9 @@ do --upload) UPLOAD=1 ;; + --build-bundle-packages) + BUILD_BUNDLE_PACKAGES=1 + ;; *) echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help" exit 1 @@ -91,7 +100,16 @@ EOF exit 1 fi +RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`" +RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized +if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + if [[ "$DEBUG" != 0 ]]; then + echo "$0 is running from $RUN_BUILD_PACKAGES_PATH" echo "Workspace is $WORKSPACE" fi @@ -192,6 +210,15 @@ build_and_scp_deb () { FPM_RESULTS=$("${COMMAND_ARR[@]}") FPM_EXIT_CODE=$? + verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS +} + +# verify build results and scp debs, if needed +verify_and_scp_deb () { + FPM_EXIT_CODE=$1 + shift + FPM_RESULTS=$@ + FPM_PACKAGE_NAME='' if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\-.]*\.deb) ]]; then FPM_PACKAGE_NAME=${BASH_REMATCH[1]} @@ -270,7 +297,7 @@ if [[ "$DEBUG" != 0 ]]; then echo fi -if type rvm-exec 2>/dev/null; then +if type rvm-exec >/dev/null 2>&1; then FPM_GEM_PREFIX=$(rvm-exec system gem environment gemdir) else FPM_GEM_PREFIX=$(gem environment gemdir) @@ -286,7 +313,7 @@ if [[ "$DEBUG" != 0 ]]; then gem build arvados.gemspec else # -q appears to be broken in gem version 2.2.2 - gem build arvados.gemspec -q >/dev/null + gem build arvados.gemspec -q >/dev/null 2>&1 fi if [[ "$UPLOAD" != 0 ]]; then @@ -483,12 +510,127 @@ for deppkg in python-gflags pyvcf google-api-python-client oauth2client \ pycrypto backports.ssl_match_hostname; do build_and_scp_deb "$deppkg" done + # Python 3 dependencies for deppkg in docker-py six requests; do # The empty string is the vendor argument: these aren't Curoverse software. build_and_scp_deb "$deppkg" "python3-$deppkg" "" python3 done +# Build the API server package + +cd "$WORKSPACE/services/api" + +API_VERSION=$(version_from_git) +PACKAGE_NAME=arvados-api-server + +if [[ ! -d "$WORKSPACE/services/api/tmp" ]]; then + mkdir $WORKSPACE/services/api/tmp +fi + +BUNDLE_OUTPUT=`bundle install --path vendor/bundle` + +if [[ "$DEBUG" != 0 ]]; then + echo $BUNDLE_OUTPUT +fi + +/usr/bin/git rev-parse HEAD > git-commit.version + +cd $WORKSPACE/debs + +# Annoyingly, we require a database.yml file for rake assets:precompile to work. So for now, +# we do that in the upgrade script. +# TODO: add bogus database.yml file so we can precompile the assets and put them in the +# package. Then remove that database.yml file again. It has to be a valid file though. +#RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile + +# This is the complete package with vendor/bundle included. +# It's big, so we do not build it by default. +if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "$PACKAGE_NAME" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") + + if [[ "$DEBUG" != 0 ]]; then + echo + echo "${COMMAND_ARR[@]}" + echo + fi + + FPM_RESULTS=$("${COMMAND_ARR[@]}") + FPM_EXIT_CODE=$? + verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS +fi + +# Build the 'bare' package without vendor/bundle. +declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "${PACKAGE_NAME}-bare" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") + +if [[ "$DEBUG" != 0 ]]; then + echo + echo "${COMMAND_ARR[@]}" + echo +fi + +FPM_RESULTS=$("${COMMAND_ARR[@]}") +FPM_EXIT_CODE=$? +verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS + +# API server package build done + +# Build the workbench server package + +cd "$WORKSPACE/apps/workbench" + +WORKBENCH_VERSION=$(version_from_git) +PACKAGE_NAME=arvados-workbench + +if [[ ! -d "$WORKSPACE/apps/workbench/tmp" ]]; then + mkdir $WORKSPACE/apps/workbench/tmp +fi + +BUNDLE_OUTPUT=`bundle install --path vendor/bundle` + +if [[ "$DEBUG" != 0 ]]; then + echo $BUNDLE_OUTPUT +fi + +/usr/bin/git rev-parse HEAD > git-commit.version + +RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null + +cd $WORKSPACE/debs + +# This is the complete package with vendor/bundle included. +# It's big, so we do not build it by default. +if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then + + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "$PACKAGE_NAME" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/tmp" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") + + if [[ "$DEBUG" != 0 ]]; then + echo + echo "${COMMAND_ARR[@]}" + echo + fi + + FPM_RESULTS=$("${COMMAND_ARR[@]}") + FPM_EXIT_CODE=$? + verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS +fi + +# Build the 'bare' package without vendor/bundle. + +declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='AGPL v3'" "-s" "dir" "-t" "deb" "-n" "${PACKAGE_NAME}-bare" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/tmp" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") + +if [[ "$DEBUG" != 0 ]]; then + echo + echo "${COMMAND_ARR[@]}" + echo +fi + +FPM_RESULTS=$("${COMMAND_ARR[@]}") +FPM_EXIT_CODE=$? +verify_and_scp_deb $FPM_EXIT_CODE $FPM_RESULTS + +# Workbench package build done + # Finally, publish the packages, if necessary if [[ "$UPLOAD" != 0 && "$CALL_FREIGHT" != 0 ]]; then ssh -p2222 $APTUSER@$APTSERVER -t "cd tmp && ls -laF *deb && freight add *deb apt/wheezy && freight cache && rm -f *deb" -- 2.30.2