2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 # A library of functions shared by the various scripts in this directory.
8 # This is the timestamp about when we merged changed to include licenses
9 # with Arvados packages. We use it as a heuristic to add revisions for
11 LICENSE_PACKAGE_TS=20151208015500
13 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
14 RAILS_PACKAGE_ITERATION=8
16 RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
20 echo "$@" >"$STDOUT_IF_DEBUG"
24 for version_suffix in "$@"; do
25 if "easy_install$version_suffix" --version >/dev/null 2>&1; then
26 echo "easy_install$version_suffix"
33 Error: easy_install$1 (from Python setuptools module) not found
39 format_last_commit_here() {
40 local format="$1"; shift
41 TZ=UTC git log -n1 --first-parent "--format=format:$format" .
45 # Output the version being built, or if we're building a
46 # dev/prerelease, output a version number based on the git log for
47 # the current working directory.
48 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
49 echo "$ARVADOS_BUILDING_VERSION"
53 local git_ts git_hash prefix
54 if [[ -n "$1" ]] ; then
60 declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
61 ARVADOS_BUILDING_VERSION="${prefix}.$(git describe --abbrev=0).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
62 echo "$ARVADOS_BUILDING_VERSION"
65 nohash_version_from_git() {
66 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
67 echo "$ARVADOS_BUILDING_VERSION"
70 version_from_git $1 | cut -d. -f1-4
73 timestamp_from_git() {
74 format_last_commit_here "%ct"
77 handle_python_package () {
78 # This function assumes the current working directory is the python package directory
79 if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
80 # This package doesn't need rebuilding.
83 # Make sure only to use sdist - that's the only format pip can deal with (sigh)
84 python setup.py $DASHQ_UNLESS_DEBUG sdist
88 local gem_name="$1"; shift
89 local gem_version="$(nohash_version_from_git)"
90 local gem_src_dir="$(pwd)"
92 if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
96 if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
97 find -maxdepth 1 -name "${gem_name}-*.gem" -delete
99 # -q appears to be broken in gem version 2.2.2
100 $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
104 # Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
105 package_go_binary() {
106 local src_path="$1"; shift
107 local prog="$1"; shift
108 local description="$1"; shift
109 local license_file="${1:-agpl-3.0.txt}"; shift
111 if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then
115 debug_echo "package_go_binary $src_path as $prog"
117 local basename="${src_path##*/}"
119 mkdir -p "$GOPATH/src/git.curoverse.com"
120 ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
121 (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
123 cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
124 local version="$(version_from_git)"
125 local timestamp="$(timestamp_from_git)"
127 # Update the version number and build a new package if the vendor
128 # bundle has changed, or the command imports anything from the
129 # Arvados SDK and the SDK has changed.
130 declare -a checkdirs=(vendor)
131 if grep -qr git.curoverse.com/arvados .; then
133 if [[ "$prog" -eq "crunch-dispatch-slurm" ]]; then
134 checkdirs+=(lib/dispatchcloud)
137 for dir in ${checkdirs[@]}; do
138 cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
139 ts="$(timestamp_from_git)"
140 if [[ "$ts" -gt "$timestamp" ]]; then
141 version=$(version_from_git)
146 cd $WORKSPACE/packages/$TARGET
147 test_package_presence $prog $version go
149 if [[ "$?" != "0" ]]; then
153 go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
156 systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
157 if [[ -e "${systemd_unit}" ]]; then
159 --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
160 --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
161 "${systemd_unit}=/lib/systemd/system/${prog}.service")
163 switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
165 fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" 'Curoverse, Inc.' dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
168 default_iteration() {
169 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
170 echo "$ARVADOS_BUILDING_ITERATION"
173 local package_name="$1"; shift
174 local package_version="$1"; shift
175 local package_type="$1"; shift
177 if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
178 [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
181 if [[ $package_type =~ ^python ]]; then
182 # Fix --iteration for #9242.
188 _build_rails_package_scripts() {
189 local pkgname="$1"; shift
190 local destdir="$1"; shift
191 local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
192 for scriptname in postinst prerm postrm; do
193 cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
194 >"$destdir/$scriptname" || return $?
198 test_rails_package_presence() {
199 local pkgname="$1"; shift
200 local srcdir="$1"; shift
202 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
210 local version="$(version_from_git)"
214 test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
217 test_package_presence() {
218 local pkgname="$1"; shift
219 local version="$1"; shift
220 local pkgtype="$1"; shift
221 local iteration="$1"; shift
222 local arch="$1"; shift
224 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
228 if [[ "$iteration" == "" ]]; then
229 iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
232 if [[ "$arch" == "" ]]; then
233 rpm_architecture="x86_64"
234 deb_architecture="amd64"
236 if [[ "$pkgtype" =~ ^(python|python3)$ ]]; then
237 rpm_architecture="noarch"
238 deb_architecture="all"
241 if [[ "$pkgtype" =~ ^(src)$ ]]; then
242 rpm_architecture="noarch"
243 deb_architecture="all"
246 # These python packages have binary components
247 if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
248 rpm_architecture="x86_64"
249 deb_architecture="amd64"
252 rpm_architecture=$arch
253 deb_architecture=$arch
256 if [[ "$FORMAT" == "deb" ]]; then
257 local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
259 # rpm packages get iteration 1 if we don't supply one
260 iteration=${iteration:-1}
261 local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
264 # See if we can skip building the package, only if it already exists in the
265 # processed/ directory. If so, move it back to the packages directory to make
266 # sure it gets picked up by the test and/or upload steps.
267 if [[ -e "processed/$complete_pkgname" ]]; then
268 echo "Package $complete_pkgname exists, not rebuilding!"
269 mv processed/$complete_pkgname .
272 echo "Package $complete_pkgname not found, building"
277 handle_rails_package() {
278 local pkgname="$1"; shift
280 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
283 local srcdir="$1"; shift
285 local license_path="$1"; shift
286 local version="$(version_from_git)"
287 local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
290 _build_rails_package_scripts "$pkgname" "$scripts_dir"
293 git rev-parse HEAD >git-commit.version
296 if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
297 echo "ERROR: $pkgname package prep failed" >&2
298 rm -rf "$scripts_dir"
302 local railsdir="/var/www/${pkgname%-server}/current"
303 local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
304 local license_arg="$license_path=$railsdir/$(basename "$license_path")"
305 local -a switches=(--after-install "$scripts_dir/postinst"
306 --before-remove "$scripts_dir/prerm"
307 --after-remove "$scripts_dir/postrm")
308 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
309 switches+=(--iteration $RAILS_PACKAGE_ITERATION)
311 # For some reason fpm excludes need to not start with /.
312 local exclude_root="${railsdir#/}"
313 # .git and packages are for the SSO server, which is built from its
315 local -a exclude_list=(.git packages tmp log coverage Capfile\* \
316 config/deploy\* config/application.yml)
317 # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
318 if [[ "$pkgname" != "arvados-workbench" ]]; then
319 exclude_list+=('config/database.yml')
321 for exclude in ${exclude_list[@]}; do
322 switches+=(-x "$exclude_root/$exclude")
324 fpm_build "${pos_args[@]}" "${switches[@]}" \
325 -x "$exclude_root/vendor/cache-*" \
326 -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
327 rm -rf "$scripts_dir"
330 # Build packages for everything
332 # The package source. Depending on the source type, this can be a
333 # path, or the name of the package in an upstream repository (e.g.,
337 # The name of the package to build.
340 # Optional: the vendor of the package. Should be "Curoverse, Inc." for
341 # packages of our own software. Passed to fpm --vendor.
344 # The type of source package. Passed to fpm -s. Default "python".
345 PACKAGE_TYPE=${1:-python}
347 # Optional: the package version number. Passed to fpm -v.
351 if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
355 local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
358 case "$PACKAGE_TYPE" in
360 # All Arvados Python2 packages depend on Python 2.7.
361 # Make sure we build with that for consistency.
363 set -- "$@" --python-bin python2.7 \
364 --python-easyinstall "$EASY_INSTALL2" \
365 --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
366 --prefix "$PYTHON2_PREFIX" \
367 --python-install-lib "$PYTHON2_INSTALL_LIB" \
368 --python-install-data . \
369 --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
370 --depends "$PYTHON2_PACKAGE"
373 # fpm does not actually support a python3 package type. Instead
374 # we recognize it as a convenience shortcut to add several
375 # necessary arguments to fpm's command line later, after we're
376 # done handling positional arguments.
379 set -- "$@" --python-bin python3 \
380 --python-easyinstall "$EASY_INSTALL3" \
381 --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
382 --prefix "$PYTHON3_PREFIX" \
383 --python-install-lib "$PYTHON3_INSTALL_LIB" \
384 --python-install-data . \
385 --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
386 --depends "$PYTHON3_PACKAGE"
390 declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
391 if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
392 # Dependencies are built from setup.py. Since setup.py will never
393 # refer to Debian package iterations, it doesn't make sense to
394 # enforce those in the .deb dependencies.
395 COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
398 # 12271 - As FPM-generated packages don't include scripts by default, the
399 # packages cleanup on upgrade depends on files being listed on the %files
400 # section in the generated SPEC files. To remove DIRECTORIES, they need to
401 # be listed in that sectiontoo, so we need to add this parameter to properly
402 # remove lingering dirs. But this only works for python2: if used on
403 # python33, it includes dirs like /opt/rh/python33 that belong to
405 if [[ "$FORMAT" = rpm ]] && [[ "$python" = python2.7 ]]; then
406 COMMAND_ARR+=('--rpm-auto-add-directories')
409 if [[ "${DEBUG:-0}" != "0" ]]; then
410 COMMAND_ARR+=('--verbose' '--log' 'info')
413 if [[ -n "$PACKAGE_NAME" ]]; then
414 COMMAND_ARR+=('-n' "$PACKAGE_NAME")
417 if [[ "$VENDOR" != "" ]]; then
418 COMMAND_ARR+=('--vendor' "$VENDOR")
421 if [[ "$VERSION" != "" ]]; then
422 COMMAND_ARR+=('-v' "$VERSION")
424 if [[ -n "$default_iteration_value" ]]; then
425 # We can always add an --iteration here. If another one is specified in $@,
426 # that will take precedence, as desired.
427 COMMAND_ARR+=(--iteration "$default_iteration_value")
430 if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
433 --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
434 --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
438 # Append --depends X and other arguments specified by fpm-info.sh in
439 # the package source dir. These are added last so they can override
440 # the arguments added by this script.
441 declare -a fpm_args=()
442 declare -a build_depends=()
443 declare -a fpm_depends=()
444 declare -a fpm_exclude=()
445 declare -a fpm_dirs=(
446 # source dir part of 'dir' package ("/source=/dest" => "/source"):
448 # backports ("llfuse>=1.0" => "backports/python-llfuse")
449 "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
450 if [[ -n "$PACKAGE_NAME" ]]; then
451 fpm_dirs+=("${WORKSPACE}/backports/${PACKAGE_NAME}")
453 for pkgdir in "${fpm_dirs[@]}"; do
454 fpminfo="$pkgdir/fpm-info.sh"
455 if [[ -e "$fpminfo" ]]; then
456 debug_echo "Loading fpm overrides from $fpminfo"
461 for pkg in "${build_depends[@]}"; do
462 if [[ $TARGET =~ debian|ubuntu ]]; then
463 pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
464 if [[ -e $pkg_deb ]]; then
465 echo "Installing build_dep $pkg from $pkg_deb"
468 echo "Attemping to install build_dep $pkg using apt-get"
469 apt-get install -y "$pkg"
471 apt-get -y -f install
473 pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
474 if [[ -e $pkg_rpm ]]; then
475 echo "Installing build_dep $pkg from $pkg_rpm"
478 echo "Attemping to install build_dep $pkg"
483 for i in "${fpm_depends[@]}"; do
484 COMMAND_ARR+=('--depends' "$i")
486 for i in "${fpm_exclude[@]}"; do
487 COMMAND_ARR+=('--exclude' "$i")
490 # Append remaining function arguments directly to fpm's command line.
495 COMMAND_ARR+=("${fpm_args[@]}")
497 COMMAND_ARR+=("$PACKAGE")
499 debug_echo -e "\n${COMMAND_ARR[@]}\n"
501 FPM_RESULTS=$("${COMMAND_ARR[@]}")
504 fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
506 # if something went wrong and debug is off, print out the fpm command that errored
507 if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
508 echo -e "\n${COMMAND_ARR[@]}\n"
512 # verify build results
519 if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
520 FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
523 if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
526 echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
531 elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
532 echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
534 elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
536 echo "Error building package for $1:\n $FPM_RESULTS"
543 if [[ "$FORMAT" == "deb" ]]; then
544 $SUDO apt-get install $PACKAGES --yes
545 elif [[ "$FORMAT" == "rpm" ]]; then
546 $SUDO yum -q -y install $PACKAGES
551 txt="********** $1 **********"
552 printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
556 if [[ "$1" != "0" ]]; then
557 title "!!!!!! $2 FAILED !!!!!!"
558 failures+=("$2 (`timer`)")
560 successes+=("$2 (`timer`)")
569 echo -n "$(($SECONDS - $t0))s"
573 for x in "${successes[@]}"
578 if [[ ${#failures[@]} == 0 ]]
580 echo "All test suites passed."
582 echo "Failures (${#failures[@]}):"
583 for x in "${failures[@]}"