Merge branch 'master' of git.curoverse.com:arvados into 13076-r-autogen-api
[arvados.git] / build / run-library.sh
1 #!/bin/bash
2 # Copyright (C) The Arvados Authors. All rights reserved.
3 #
4 # SPDX-License-Identifier: AGPL-3.0
5
6 # A library of functions shared by the various scripts in this directory.
7
8 # This is the timestamp about when we merged changed to include licenses
9 # with Arvados packages.  We use it as a heuristic to add revisions for
10 # older packages.
11 LICENSE_PACKAGE_TS=20151208015500
12
13 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
14     RAILS_PACKAGE_ITERATION=8
15 else
16     RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
17 fi
18
19 debug_echo () {
20     echo "$@" >"$STDOUT_IF_DEBUG"
21 }
22
23 find_easy_install() {
24     for version_suffix in "$@"; do
25         if "easy_install$version_suffix" --version >/dev/null 2>&1; then
26             echo "easy_install$version_suffix"
27             return 0
28         fi
29     done
30     cat >&2 <<EOF
31 $helpmessage
32
33 Error: easy_install$1 (from Python setuptools module) not found
34
35 EOF
36     exit 1
37 }
38
39 format_last_commit_here() {
40     local format="$1"; shift
41     TZ=UTC git log -n1 --first-parent "--format=format:$format" .
42 }
43
44 version_from_git() {
45     # Output the version being built, or if we're building a
46     # dev/prerelease, output a version number based on the git log for
47     # the current working directory.
48     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
49         echo "$ARVADOS_BUILDING_VERSION"
50         return
51     fi
52
53     local git_ts git_hash prefix
54     if [[ -n "$1" ]] ; then
55         prefix="$1"
56     else
57         prefix="0.1"
58     fi
59
60     declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
61     echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
62 }
63
64 nohash_version_from_git() {
65     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
66         echo "$ARVADOS_BUILDING_VERSION"
67         return
68     fi
69     version_from_git $1 | cut -d. -f1-3
70 }
71
72 timestamp_from_git() {
73     format_last_commit_here "%ct"
74 }
75
76 handle_python_package () {
77   # This function assumes the current working directory is the python package directory
78   if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
79     # This package doesn't need rebuilding.
80     return
81   fi
82   # Make sure only to use sdist - that's the only format pip can deal with (sigh)
83   python setup.py $DASHQ_UNLESS_DEBUG sdist
84 }
85
86 handle_ruby_gem() {
87     local gem_name="$1"; shift
88     local gem_version="$(nohash_version_from_git)"
89     local gem_src_dir="$(pwd)"
90
91     if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
92         return 0
93     fi
94
95     if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
96         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
97
98         # -q appears to be broken in gem version 2.2.2
99         $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
100     fi
101 }
102
103 # Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
104 package_go_binary() {
105     local src_path="$1"; shift
106     local prog="$1"; shift
107     local description="$1"; shift
108     local license_file="${1:-agpl-3.0.txt}"; shift
109
110     if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then
111         return 0
112     fi
113
114     debug_echo "package_go_binary $src_path as $prog"
115
116     local basename="${src_path##*/}"
117
118     mkdir -p "$GOPATH/src/git.curoverse.com"
119     ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
120     (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
121
122     cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
123     local version="$(version_from_git)"
124     local timestamp="$(timestamp_from_git)"
125
126     # Update the version number and build a new package if the vendor
127     # bundle has changed, or the command imports anything from the
128     # Arvados SDK and the SDK has changed.
129     declare -a checkdirs=(vendor)
130     if grep -qr git.curoverse.com/arvados .; then
131         checkdirs+=(sdk/go)
132         if [[ "$prog" -eq "crunch-dispatch-slurm" ]]; then
133           checkdirs+=(lib/dispatchcloud)
134         fi
135     fi
136     for dir in ${checkdirs[@]}; do
137         cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
138         ts="$(timestamp_from_git)"
139         if [[ "$ts" -gt "$timestamp" ]]; then
140             version=$(version_from_git)
141             timestamp="$ts"
142         fi
143     done
144
145     cd $WORKSPACE/packages/$TARGET
146     test_package_presence $prog $version go
147
148     if [[ "$?" != "0" ]]; then
149       return 1
150     fi
151
152     go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
153
154     local -a switches=()
155     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
156     if [[ -e "${systemd_unit}" ]]; then
157         switches+=(
158             --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
159             --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
160             "${systemd_unit}=/lib/systemd/system/${prog}.service")
161     fi
162     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
163
164     fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" 'Curoverse, Inc.' dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
165 }
166
167 default_iteration() {
168     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
169         echo "$ARVADOS_BUILDING_ITERATION"
170         return
171     fi
172     local package_name="$1"; shift
173     local package_version="$1"; shift
174     local package_type="$1"; shift
175     local iteration=1
176     if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
177            [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
178         iteration=2
179     fi
180     if [[ $package_type =~ ^python ]]; then
181       # Fix --iteration for #9242.
182       iteration=2
183     fi
184     echo $iteration
185 }
186
187 _build_rails_package_scripts() {
188     local pkgname="$1"; shift
189     local destdir="$1"; shift
190     local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
191     for scriptname in postinst prerm postrm; do
192         cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
193             >"$destdir/$scriptname" || return $?
194     done
195 }
196
197 test_rails_package_presence() {
198   local pkgname="$1"; shift
199   local srcdir="$1"; shift
200
201   if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
202     return 1
203   fi
204
205   tmppwd=`pwd`
206
207   cd $srcdir
208
209   local version="$(version_from_git)"
210
211   cd $tmppwd
212
213   test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
214 }
215
216 test_package_presence() {
217     local pkgname="$1"; shift
218     local version="$1"; shift
219     local pkgtype="$1"; shift
220     local iteration="$1"; shift
221     local arch="$1"; shift
222
223     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
224         return 1
225     fi
226
227     if [[ "$iteration" == "" ]]; then
228         iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
229     fi
230
231     if [[ "$arch" == "" ]]; then
232       rpm_architecture="x86_64"
233       deb_architecture="amd64"
234
235       if [[ "$pkgtype" =~ ^(python|python3)$ ]]; then
236         rpm_architecture="noarch"
237         deb_architecture="all"
238       fi
239
240       if [[ "$pkgtype" =~ ^(src)$ ]]; then
241         rpm_architecture="noarch"
242         deb_architecture="all"
243       fi
244
245       # These python packages have binary components
246       if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
247         rpm_architecture="x86_64"
248         deb_architecture="amd64"
249       fi
250     else
251       rpm_architecture=$arch
252       deb_architecture=$arch
253     fi
254
255     if [[ "$FORMAT" == "deb" ]]; then
256         local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
257     else
258         # rpm packages get iteration 1 if we don't supply one
259         iteration=${iteration:-1}
260         local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
261     fi
262
263     # See if we can skip building the package, only if it already exists in the
264     # processed/ directory. If so, move it back to the packages directory to make
265     # sure it gets picked up by the test and/or upload steps.
266     if [[ -e "processed/$complete_pkgname" ]]; then
267       echo "Package $complete_pkgname exists, not rebuilding!"
268       mv processed/$complete_pkgname .
269       return 1
270     else
271       echo "Package $complete_pkgname not found, building"
272       return 0
273     fi
274 }
275
276 handle_rails_package() {
277     local pkgname="$1"; shift
278
279     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
280         return 0
281     fi
282     local srcdir="$1"; shift
283     cd "$srcdir"
284     local license_path="$1"; shift
285     local version="$(version_from_git)"
286     local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
287     (
288         set -e
289         _build_rails_package_scripts "$pkgname" "$scripts_dir"
290         cd "$srcdir"
291         mkdir -p tmp
292         git rev-parse HEAD >git-commit.version
293         bundle package --all
294     )
295     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
296         echo "ERROR: $pkgname package prep failed" >&2
297         rm -rf "$scripts_dir"
298         EXITCODE=1
299         return 1
300     fi
301     local railsdir="/var/www/${pkgname%-server}/current"
302     local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
303     local license_arg="$license_path=$railsdir/$(basename "$license_path")"
304     local -a switches=(--after-install "$scripts_dir/postinst"
305                        --before-remove "$scripts_dir/prerm"
306                        --after-remove "$scripts_dir/postrm")
307     if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
308         switches+=(--iteration $RAILS_PACKAGE_ITERATION)
309     fi
310     # For some reason fpm excludes need to not start with /.
311     local exclude_root="${railsdir#/}"
312     # .git and packages are for the SSO server, which is built from its
313     # repository root.
314     local -a exclude_list=(.git packages tmp log coverage Capfile\* \
315                            config/deploy\* config/application.yml)
316     # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
317     if  [[ "$pkgname" != "arvados-workbench" ]]; then
318       exclude_list+=('config/database.yml')
319     fi
320     for exclude in ${exclude_list[@]}; do
321         switches+=(-x "$exclude_root/$exclude")
322     done
323     fpm_build "${pos_args[@]}" "${switches[@]}" \
324               -x "$exclude_root/vendor/cache-*" \
325               -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
326     rm -rf "$scripts_dir"
327 }
328
329 # Build packages for everything
330 fpm_build () {
331   # The package source.  Depending on the source type, this can be a
332   # path, or the name of the package in an upstream repository (e.g.,
333   # pip).
334   PACKAGE=$1
335   shift
336   # The name of the package to build.
337   PACKAGE_NAME=$1
338   shift
339   # Optional: the vendor of the package.  Should be "Curoverse, Inc." for
340   # packages of our own software.  Passed to fpm --vendor.
341   VENDOR=$1
342   shift
343   # The type of source package.  Passed to fpm -s.  Default "python".
344   PACKAGE_TYPE=${1:-python}
345   shift
346   # Optional: the package version number.  Passed to fpm -v.
347   VERSION=$1
348   shift
349
350   if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
351       return 0
352   fi
353
354   local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
355   local python=""
356
357   case "$PACKAGE_TYPE" in
358       python)
359           # All Arvados Python2 packages depend on Python 2.7.
360           # Make sure we build with that for consistency.
361           python=python2.7
362           set -- "$@" --python-bin python2.7 \
363               --python-easyinstall "$EASY_INSTALL2" \
364               --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
365               --prefix "$PYTHON2_PREFIX" \
366               --python-install-lib "$PYTHON2_INSTALL_LIB" \
367               --python-install-data . \
368               --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
369               --depends "$PYTHON2_PACKAGE"
370           ;;
371       python3)
372           # fpm does not actually support a python3 package type.  Instead
373           # we recognize it as a convenience shortcut to add several
374           # necessary arguments to fpm's command line later, after we're
375           # done handling positional arguments.
376           PACKAGE_TYPE=python
377           python=python3
378           set -- "$@" --python-bin python3 \
379               --python-easyinstall "$EASY_INSTALL3" \
380               --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
381               --prefix "$PYTHON3_PREFIX" \
382               --python-install-lib "$PYTHON3_INSTALL_LIB" \
383               --python-install-data . \
384               --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
385               --depends "$PYTHON3_PACKAGE"
386           ;;
387   esac
388
389   declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
390   if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
391       # Dependencies are built from setup.py.  Since setup.py will never
392       # refer to Debian package iterations, it doesn't make sense to
393       # enforce those in the .deb dependencies.
394       COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
395   fi
396
397   # 12271 - As FPM-generated packages don't include scripts by default, the
398   # packages cleanup on upgrade depends on files being listed on the %files
399   # section in the generated SPEC files. To remove DIRECTORIES, they need to
400   # be listed in that sectiontoo, so we need to add this parameter to properly
401   # remove lingering dirs. But this only works for python2: if used on
402   # python33, it includes dirs like /opt/rh/python33 that belong to
403   # other packages.
404   if [[ "$FORMAT" = rpm ]] && [[ "$python" = python2.7 ]]; then
405     COMMAND_ARR+=('--rpm-auto-add-directories')
406   fi
407
408   if [[ "${DEBUG:-0}" != "0" ]]; then
409     COMMAND_ARR+=('--verbose' '--log' 'info')
410   fi
411
412   if [[ -n "$PACKAGE_NAME" ]]; then
413     COMMAND_ARR+=('-n' "$PACKAGE_NAME")
414   fi
415
416   if [[ "$VENDOR" != "" ]]; then
417     COMMAND_ARR+=('--vendor' "$VENDOR")
418   fi
419
420   if [[ "$VERSION" != "" ]]; then
421     COMMAND_ARR+=('-v' "$VERSION")
422   fi
423   if [[ -n "$default_iteration_value" ]]; then
424       # We can always add an --iteration here.  If another one is specified in $@,
425       # that will take precedence, as desired.
426       COMMAND_ARR+=(--iteration "$default_iteration_value")
427   fi
428
429   if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
430   then
431       COMMAND_ARR+=(
432           --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
433           --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
434       )
435   fi
436
437   # Append --depends X and other arguments specified by fpm-info.sh in
438   # the package source dir. These are added last so they can override
439   # the arguments added by this script.
440   declare -a fpm_args=()
441   declare -a build_depends=()
442   declare -a fpm_depends=()
443   declare -a fpm_exclude=()
444   declare -a fpm_dirs=(
445       # source dir part of 'dir' package ("/source=/dest" => "/source"):
446       "${PACKAGE%%=/*}"
447       # backports ("llfuse>=1.0" => "backports/python-llfuse")
448       "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
449   if [[ -n "$PACKAGE_NAME" ]]; then
450       fpm_dirs+=("${WORKSPACE}/backports/${PACKAGE_NAME}")
451   fi
452   for pkgdir in "${fpm_dirs[@]}"; do
453       fpminfo="$pkgdir/fpm-info.sh"
454       if [[ -e "$fpminfo" ]]; then
455           debug_echo "Loading fpm overrides from $fpminfo"
456           source "$fpminfo"
457           break
458       fi
459   done
460   for pkg in "${build_depends[@]}"; do
461       if [[ $TARGET =~ debian|ubuntu ]]; then
462           pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
463           if [[ -e $pkg_deb ]]; then
464               echo "Installing build_dep $pkg from $pkg_deb"
465               dpkg -i "$pkg_deb"
466           else
467               echo "Attemping to install build_dep $pkg using apt-get"
468               apt-get install -y "$pkg"
469           fi
470           apt-get -y -f install
471       else
472           pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
473           if [[ -e $pkg_rpm ]]; then
474               echo "Installing build_dep $pkg from $pkg_rpm"
475               rpm -i "$pkg_rpm"
476           else
477               echo "Attemping to install build_dep $pkg"
478               rpm -i "$pkg"
479           fi
480       fi
481   done
482   for i in "${fpm_depends[@]}"; do
483     COMMAND_ARR+=('--depends' "$i")
484   done
485   for i in "${fpm_exclude[@]}"; do
486     COMMAND_ARR+=('--exclude' "$i")
487   done
488
489   # Append remaining function arguments directly to fpm's command line.
490   for i; do
491     COMMAND_ARR+=("$i")
492   done
493
494   COMMAND_ARR+=("${fpm_args[@]}")
495
496   COMMAND_ARR+=("$PACKAGE")
497
498   debug_echo -e "\n${COMMAND_ARR[@]}\n"
499
500   FPM_RESULTS=$("${COMMAND_ARR[@]}")
501   FPM_EXIT_CODE=$?
502
503   fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
504
505   # if something went wrong and debug is off, print out the fpm command that errored
506   if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
507     echo -e "\n${COMMAND_ARR[@]}\n"
508   fi
509 }
510
511 # verify build results
512 fpm_verify () {
513   FPM_EXIT_CODE=$1
514   shift
515   FPM_RESULTS=$@
516
517   FPM_PACKAGE_NAME=''
518   if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
519     FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
520   fi
521
522   if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
523     EXITCODE=1
524     echo
525     echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
526     echo
527     echo $FPM_RESULTS
528     echo
529     return 1
530   elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
531     echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
532     return 0
533   elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
534     EXITCODE=1
535     echo "Error building package for $1:\n $FPM_RESULTS"
536     return 1
537   fi
538 }
539
540 install_package() {
541   PACKAGES=$@
542   if [[ "$FORMAT" == "deb" ]]; then
543     $SUDO apt-get install $PACKAGES --yes
544   elif [[ "$FORMAT" == "rpm" ]]; then
545     $SUDO yum -q -y install $PACKAGES
546   fi
547 }
548
549 title () {
550     txt="********** $1 **********"
551     printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
552 }
553
554 checkexit() {
555     if [[ "$1" != "0" ]]; then
556         title "!!!!!! $2 FAILED !!!!!!"
557         failures+=("$2 (`timer`)")
558     else
559         successes+=("$2 (`timer`)")
560     fi
561 }
562
563 timer_reset() {
564     t0=$SECONDS
565 }
566
567 timer() {
568     echo -n "$(($SECONDS - $t0))s"
569 }
570
571 report_outcomes() {
572     for x in "${successes[@]}"
573     do
574         echo "Pass: $x"
575     done
576
577     if [[ ${#failures[@]} == 0 ]]
578     then
579         echo "All test suites passed."
580     else
581         echo "Failures (${#failures[@]}):"
582         for x in "${failures[@]}"
583         do
584             echo "Fail: $x"
585         done
586     fi
587 }