3 # A library of functions shared by the various scripts in this directory.
5 # This is the timestamp about when we merged changed to include licenses
6 # with Arvados packages. We use it as a heuristic to add revisions for
8 LICENSE_PACKAGE_TS=20151208015500
11 echo "$@" >"$STDOUT_IF_DEBUG"
15 for version_suffix in "$@"; do
16 if "easy_install$version_suffix" --version >/dev/null 2>&1; then
17 echo "easy_install$version_suffix"
24 Error: easy_install$1 (from Python setuptools module) not found
30 format_last_commit_here() {
31 local format="$1"; shift
32 TZ=UTC git log -n1 --first-parent "--format=format:$format" .
36 # Generates a version number from the git log for the current working
37 # directory, and writes it to stdout.
39 declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
40 echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
43 nohash_version_from_git() {
44 version_from_git | cut -d. -f1-3
47 timestamp_from_git() {
48 format_last_commit_here "%ct"
51 handle_python_package () {
52 # This function assumes the current working directory is the python package directory
53 if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
54 # This package doesn't need rebuilding.
57 # Make sure only to use sdist - that's the only format pip can deal with (sigh)
58 python setup.py $DASHQ_UNLESS_DEBUG sdist
62 local gem_name="$1"; shift
63 local gem_version="$(nohash_version_from_git)"
64 local gem_src_dir="$(pwd)"
66 if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
67 find -maxdepth 1 -name "${gem_name}-*.gem" -delete
69 # -q appears to be broken in gem version 2.2.2
70 $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
74 # Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
76 local src_path="$1"; shift
77 local prog="$1"; shift
78 local description="$1"; shift
79 local license_file="${1:-agpl-3.0.txt}"; shift
81 debug_echo "package_go_binary $src_path as $prog"
83 local basename="${src_path##*/}"
85 mkdir -p "$GOPATH/src/git.curoverse.com"
86 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
88 cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
89 local version="$(version_from_git)"
90 local timestamp="$(timestamp_from_git)"
92 # If the command imports anything from the Arvados SDK, bump the
93 # version number and build a new package whenever the SDK changes.
94 if grep -qr git.curoverse.com/arvados .; then
95 cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
96 if [[ $(timestamp_from_git) -gt "$timestamp" ]]; then
97 version=$(version_from_git)
101 cd $WORKSPACE/packages/$TARGET
102 go get "git.curoverse.com/arvados.git/$src_path"
104 declare -a switches=()
105 systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
106 if [[ -e "${systemd_unit}" ]]; then
107 switches+=("${systemd_unit}=/lib/systemd/system/${prog}.service")
108 switches+=(--after-install "$WORKSPACE/build/go-package-scripts/postinst")
110 switches+=("$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file")
112 fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "${switches[@]}"
115 default_iteration() {
116 local package_name="$1"; shift
117 local package_version="$1"; shift
119 if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
120 [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
126 _build_rails_package_scripts() {
127 local pkgname="$1"; shift
128 local destdir="$1"; shift
129 local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
130 for scriptname in postinst prerm postrm; do
131 cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
132 >"$destdir/$scriptname" || return $?
136 handle_rails_package() {
137 local pkgname="$1"; shift
138 local srcdir="$1"; shift
139 local license_path="$1"; shift
140 local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
141 local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
143 _build_rails_package_scripts "$pkgname" "$scripts_dir"
146 version_from_git >"$version_file"
147 git rev-parse HEAD >git-commit.version
150 if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
151 echo "ERROR: $pkgname package prep failed" >&2
152 rm -rf "$scripts_dir" "$version_file"
156 local railsdir="/var/www/${pkgname%-server}/current"
157 local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
158 "$(cat "$version_file")")
159 local license_arg="$license_path=$railsdir/$(basename "$license_path")"
160 local -a switches=(--iteration=6
161 --after-install "$scripts_dir/postinst"
162 --before-remove "$scripts_dir/prerm"
163 --after-remove "$scripts_dir/postrm")
164 # For some reason fpm excludes need to not start with /.
165 local exclude_root="${railsdir#/}"
166 # .git and packages are for the SSO server, which is built from its
168 local -a exclude_list=(.git packages tmp log coverage Capfile\* \
169 config/deploy\* config/application.yml)
170 # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
171 if [[ "$pkgname" != "arvados-workbench" ]]; then
172 exclude_list+=('config/database.yml')
174 for exclude in ${exclude_list[@]}; do
175 switches+=(-x "$exclude_root/$exclude")
177 fpm_build "${pos_args[@]}" "${switches[@]}" \
178 -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
179 rm -rf "$scripts_dir" "$version_file"
182 # Build packages for everything
184 # The package source. Depending on the source type, this can be a
185 # path, or the name of the package in an upstream repository (e.g.,
189 # The name of the package to build.
192 # Optional: the vendor of the package. Should be "Curoverse, Inc." for
193 # packages of our own software. Passed to fpm --vendor.
196 # The type of source package. Passed to fpm -s. Default "python".
197 PACKAGE_TYPE=${1:-python}
199 # Optional: the package version number. Passed to fpm -v.
203 local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION")"
205 case "$PACKAGE_TYPE" in
207 # All Arvados Python2 packages depend on Python 2.7.
208 # Make sure we build with that for consistency.
209 set -- "$@" --python-bin python2.7 \
210 --python-easyinstall "$EASY_INSTALL2" \
211 --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
212 --prefix "$PYTHON2_PREFIX" \
213 --python-install-lib "$PYTHON2_INSTALL_LIB" \
214 --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
215 --depends "$PYTHON2_PACKAGE"
216 # Fix --iteration for #9242.
217 default_iteration_value=$(($default_iteration_value + 1))
220 # fpm does not actually support a python3 package type. Instead
221 # we recognize it as a convenience shortcut to add several
222 # necessary arguments to fpm's command line later, after we're
223 # done handling positional arguments.
225 set -- "$@" --python-bin python3 \
226 --python-easyinstall "$EASY_INSTALL3" \
227 --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
228 --prefix "$PYTHON3_PREFIX" \
229 --python-install-lib "$PYTHON3_INSTALL_LIB" \
230 --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
231 --depends "$PYTHON3_PACKAGE"
232 # Fix --iteration for #9242.
233 default_iteration_value=$(($default_iteration_value + 1))
237 declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
238 if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
239 # Dependencies are built from setup.py. Since setup.py will never
240 # refer to Debian package iterations, it doesn't make sense to
241 # enforce those in the .deb dependencies.
242 COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
245 if [[ "${DEBUG:-0}" != "0" ]]; then
246 COMMAND_ARR+=('--verbose' '--log' 'info')
249 if [[ -n "$PACKAGE_NAME" ]]; then
250 COMMAND_ARR+=('-n' "$PACKAGE_NAME")
253 if [[ "$VENDOR" != "" ]]; then
254 COMMAND_ARR+=('--vendor' "$VENDOR")
257 if [[ "$VERSION" != "" ]]; then
258 COMMAND_ARR+=('-v' "$VERSION")
260 # We can always add an --iteration here. If another one is specified in $@,
261 # that will take precedence, as desired.
262 COMMAND_ARR+=(--iteration "$default_iteration_value")
264 # Append --depends X and other arguments specified by fpm-info.sh in
265 # the package source dir. These are added last so they can override
266 # the arguments added by this script.
267 declare -a fpm_args=()
268 declare -a build_depends=()
269 declare -a fpm_depends=()
270 declare -a fpm_exclude=()
271 declare -a fpm_dirs=(
272 # source dir part of 'dir' package ("/source=/dest" => "/source"):
274 # backports ("llfuse==0.41.1" => "backports/python-llfuse")
275 "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
276 if [[ -n "$PACKAGE_NAME" ]]; then
277 fpm_dirs+=("${WORKSPACE}/backports/${PACKAGE_NAME}")
279 for pkgdir in "${fpm_dirs[@]}"; do
280 fpminfo="$pkgdir/fpm-info.sh"
281 if [[ -e "$fpminfo" ]]; then
282 debug_echo "Loading fpm overrides from $fpminfo"
287 for pkg in "${build_depends[@]}"; do
288 if [[ $TARGET =~ debian|ubuntu ]]; then
289 pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
290 if [[ -e $pkg_deb ]]; then
291 echo "Installing build_dep $pkg from $pkg_deb"
294 echo "Attemping to install build_dep $pkg using apt-get"
295 apt-get install -y "$pkg"
297 apt-get -y -f install
299 pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
300 if [[ -e $pkg_rpm ]]; then
301 echo "Installing build_dep $pkg from $pkg_rpm"
304 echo "Attemping to install build_dep $pkg"
309 for i in "${fpm_depends[@]}"; do
310 COMMAND_ARR+=('--depends' "$i")
312 for i in "${fpm_exclude[@]}"; do
313 COMMAND_ARR+=('--exclude' "$i")
316 # Append remaining function arguments directly to fpm's command line.
321 COMMAND_ARR+=("${fpm_args[@]}")
323 COMMAND_ARR+=("$PACKAGE")
325 debug_echo -e "\n${COMMAND_ARR[@]}\n"
327 FPM_RESULTS=$("${COMMAND_ARR[@]}")
330 fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
333 # verify build results
340 if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
341 FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
344 if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
346 echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
350 elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
351 echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
352 elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
353 echo "Error building package for $1:\n $FPM_RESULTS"
359 if [[ "$FORMAT" == "deb" ]]; then
360 $SUDO apt-get install $PACKAGES --yes
361 elif [[ "$FORMAT" == "rpm" ]]; then
362 $SUDO yum -q -y install $PACKAGES
367 txt="********** $1 **********"
368 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
372 if [[ "$1" != "0" ]]; then
373 title "!!!!!! $2 FAILED !!!!!!"
374 failures+=("$2 (`timer`)")
376 successes+=("$2 (`timer`)")
385 echo -n "$(($SECONDS - $t0))s"
389 for x in "${successes[@]}"
394 if [[ ${#failures[@]} == 0 ]]
396 echo "All test suites passed."
398 echo "Failures (${#failures[@]}):"
399 for x in "${failures[@]}"