]> git.arvados.org - arvados.git/blob - build/run-library.sh
20311: Include CWL schemas in Python packages
[arvados.git] / build / run-library.sh
1 #!/bin/bash -xe
2 # Copyright (C) The Arvados Authors. All rights reserved.
3 #
4 # SPDX-License-Identifier: AGPL-3.0
5
6 # A library of functions shared by the various scripts in this directory.
7
8 # This is the timestamp about when we merged changed to include licenses
9 # with Arvados packages.  We use it as a heuristic to add revisions for
10 # older packages.
11 LICENSE_PACKAGE_TS=20151208015500
12 RAILS_PACKAGE_ITERATION="${ARVADOS_BUILDING_ITERATION:-1}"
13
14 debug_echo () {
15     echo "$@" >"$STDOUT_IF_DEBUG"
16 }
17
18 find_python_program() {
19     prog="$1"
20     shift
21     for prog in "$@"; do
22         if "$prog" --version >/dev/null 2>&1; then
23             echo "$prog"
24             return 0
25         fi
26     done
27     cat >&2 <<EOF
28 $helpmessage
29
30 Error: $prog (from Python setuptools module) not found
31
32 EOF
33     exit 1
34 }
35
36 # get_ci_scripts sets $CI_DIR to the path of a directory with CI scripts.
37 # If it is not already set, it uses the following to get them by creating a
38 # temporary Git worktree:
39 #  * CI_SRC: a Git checkout of the scripts (default $WORKSPACE)
40 #  * CI_REF: the reference used to create that (default `remotes/arvados-ci/ci-build`)
41 #  * CI_PATH: the path of CI scripts under the worktree (default `/jenkins`)
42 # The defaults are all suitable for jobs running under ci.arvados.org, but
43 # you can set them in the environment to customize the behavior, or just set
44 # $CI_DIR to a path that already has the scripts ready.
45 get_ci_scripts() {
46     if [ -n "${CI_DIR:-}" ]; then
47         return
48     fi
49     local clone_dir="$(mktemp --directory --tmpdir="${WORKSPACE_TMP:-}")" &&
50         git -C "${CI_SRC:-$WORKSPACE}" worktree add "$clone_dir" "${CI_REF:-remotes/arvados-ci/ci-build}" ||
51             return
52     CI_DIR="$clone_dir${CI_PATH:-/jenkins}"
53 }
54
55 format_last_commit_here() {
56     local format="$1"; shift
57     local dir="${1:-.}"; shift
58     TZ=UTC git log -n1 --first-parent "--format=format:$format" "$dir"
59 }
60
61 version_from_git() {
62     # Output the version being built, or if we're building a
63     # dev/prerelease, output a version number based on the git log for
64     # the given $subdir.
65     local subdir="$1"; shift
66     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
67         echo "$ARVADOS_BUILDING_VERSION"
68         return
69     fi
70
71     local git_ts git_hash
72     declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
73     ARVADOS_BUILDING_VERSION="$($WORKSPACE/build/version-at-commit.sh $git_hash)"
74     echo "$ARVADOS_BUILDING_VERSION"
75 }
76
77 nohash_version_from_git() {
78     local subdir="$1"; shift
79     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
80         echo "$ARVADOS_BUILDING_VERSION"
81         return
82     fi
83     version_from_git $subdir | cut -d. -f1-4
84 }
85
86 timestamp_from_git() {
87     local subdir="$1"; shift
88     format_last_commit_here "%ct" "$subdir"
89 }
90
91 # Usage: get_native_arch
92 get_native_arch() {
93   # Only amd64 and aarch64 are supported at the moment
94   local native_arch=""
95   case "$HOSTTYPE" in
96     x86_64)
97       native_arch="amd64"
98       ;;
99     *)
100       echo "Error: architecture not supported"
101       exit 1
102       ;;
103   esac
104   echo $native_arch
105 }
106
107 handle_ruby_gem() {
108     local gem_name="$1"; shift
109     local gem_version="$(nohash_version_from_git)"
110     local gem_src_dir="$(pwd)"
111
112     if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
113         return 0
114     fi
115
116     if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
117         find -maxdepth 1 -name "${gem_name}-*.gem" -delete
118
119         # -q appears to be broken in gem version 2.2.2
120         gem build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
121     fi
122 }
123
124 # Usage: package_workbench2
125 package_workbench2() {
126     local pkgname=arvados-workbench2
127     local src=services/workbench2
128     local dst=/var/www/arvados-workbench2/workbench2
129     local description="Arvados Workbench 2"
130     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
131         return 0
132     fi
133     cd "$WORKSPACE/$src"
134     local version="$(version_from_git)"
135     rm -rf ./build
136     NODE_ENV=production yarn install
137     VERSION="$version" BUILD_NUMBER="$(default_iteration "$pkgname" "$version" yarn)" GIT_COMMIT="$(git rev-parse HEAD | head -c9)" yarn build
138     cd "$WORKSPACE/packages/$TARGET"
139     fpm_build "${WORKSPACE}/$src" "${WORKSPACE}/$src/build/=$dst" "$pkgname" dir "$version" \
140               --license="GNU Affero General Public License, version 3.0" \
141               --description="${description}" \
142               --config-files="/etc/arvados/$pkgname/workbench2.example.json" \
143               "$WORKSPACE/services/workbench2/etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$pkgname/workbench2.example.json"
144 }
145
146 calculate_go_package_version() {
147   # $__returnvar has the nameref attribute set, which means it is a reference
148   # to another variable that is passed in as the first argument to this function.
149   # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
150   local -n __returnvar="$1"; shift
151   local oldpwd="$PWD"
152
153   cd "$WORKSPACE"
154   go mod download
155
156   # Update the version number and build a new package if the vendor
157   # bundle has changed, or the command imports anything from the
158   # Arvados SDK and the SDK has changed.
159   declare -a checkdirs=(go.mod go.sum)
160   while [ -n "$1" ]; do
161       checkdirs+=("$1")
162       shift
163   done
164   # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
165   # Everything depends on the build directory.
166   checkdirs+=(sdk/go lib build)
167   local timestamp=0
168   for dir in ${checkdirs[@]}; do
169       cd "$WORKSPACE"
170       ts="$(timestamp_from_git "$dir")"
171       if [[ "$ts" -gt "$timestamp" ]]; then
172           version=$(version_from_git "$dir")
173           timestamp="$ts"
174       fi
175   done
176   cd "$oldpwd"
177   __returnvar="$version"
178 }
179
180 # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64] "Compute foo to arbitrary precision" [apache-2.0.txt]
181 package_go_binary() {
182   local src_path="$1"; shift
183   local prog="$1"; shift
184   local package_format="$1"; shift
185   local target_arch="$1"; shift
186   local description="$1"; shift
187   local license_file="${1:-agpl-3.0.txt}"; shift
188
189   if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
190       debug_echo -e "Skipping build of $prog package."
191       return 0
192   fi
193
194   native_arch=$(get_native_arch)
195
196   if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
197     echo "Error: no cross compilation support for Go on $native_arch, can not build $prog for $target_arch"
198     return 1
199   fi
200
201   case "$package_format-$TARGET" in
202     # Ubuntu 20.04 does not support cross compilation because the
203     # libfuse package does not support multiarch. See
204     # <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477>.
205     # Red Hat-based distributions do not support native cross compilation at
206     # all (they use a qemu-based solution we haven't implemented yet).
207     deb-ubuntu2004|rpm-*)
208       cross_compilation=0
209       if [[ "$native_arch" == "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
210         echo "Error: no cross compilation support for Go on $native_arch for $TARGET, can not build $prog for $target_arch"
211         return 1
212       fi
213       ;;
214     *)
215       cross_compilation=1
216       ;;
217   esac
218
219   if [[ -n "$target_arch" ]]; then
220     archs=($target_arch)
221   else
222     # No target architecture specified, default to native target.
223     archs=($native_arch)
224   fi
225
226   for ta in ${archs[@]}; do
227     package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
228     retval=$?
229     if [[ $retval -ne 0 ]]; then
230       return $retval
231     fi
232   done
233 }
234
235 # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64] [amd64] [apache-2.0.txt]
236 package_go_binary_worker() {
237     local src_path="$1"; shift
238     local prog="$1"; shift
239     local package_format="$1"; shift
240     local description="$1"; shift
241     local native_arch="${1:-amd64}"; shift
242     local target_arch="${1:-amd64}"; shift
243     local license_file="${1:-agpl-3.0.txt}"; shift
244
245     debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
246     local basename="${src_path##*/}"
247     calculate_go_package_version go_package_version $src_path
248
249     cd $WORKSPACE/packages/$TARGET
250     test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
251     if [[ $? -ne 0 ]]; then
252       return 0
253     fi
254
255     echo "Building $package_format ($target_arch) package for $prog from $src_path"
256     GOARCH=${arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
257
258     local -a switches=()
259
260     binpath=$GOPATH/bin/${basename}
261     if [[ "${target_arch}" != "${native_arch}" ]]; then
262       switches+=("-a${target_arch}")
263       binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
264     fi
265
266     case "$package_format" in
267         # As of April 2024 we package identical Go binaries under different
268         # packages and names. This upsets the build id database, so don't
269         # register ourselves there.
270         rpm) switches+=(--rpm-rpmbuild-define="_build_id_links none") ;;
271     esac
272
273     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
274     if [[ -e "${systemd_unit}" ]]; then
275         switches+=(
276             --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
277             --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
278             "${systemd_unit}=/lib/systemd/system/${prog}.service")
279     fi
280     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
281
282     fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
283 }
284
285 # Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
286 package_go_so() {
287     local src_path="$1"; shift
288     local sofile="$1"; shift
289     local pkg="$1"; shift
290     local package_format="$1"; shift
291     local target_arch="$1"; shift # supported: amd64
292     local description="$1"; shift
293
294     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
295       debug_echo -e "Skipping build of $pkg package."
296       return 0
297     fi
298
299     debug_echo "package_go_so $src_path as $pkg"
300
301     calculate_go_package_version go_package_version $src_path
302     cd $WORKSPACE/packages/$TARGET
303     test_package_presence $pkg $go_package_version go || return 1
304     cd $WORKSPACE/$src_path
305     go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
306     cd $WORKSPACE/packages/$TARGET
307     local -a fpmargs=(
308         "--url=https://arvados.org"
309         "--license=Apache License, Version 2.0"
310         "--description=${description}"
311         "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
312     )
313     if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
314         fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go")
315     fi
316     if [[ -e "$WORKSPACE/$src_path/README" ]]; then
317         fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
318     fi
319     fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
320 }
321
322 default_iteration() {
323     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
324         echo "$ARVADOS_BUILDING_ITERATION"
325         return
326     fi
327     local package_name="$1"; shift
328     local package_version="$1"; shift
329     local package_type="$1"; shift
330     local iteration=1
331     if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
332            [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
333         iteration=2
334     fi
335     echo $iteration
336 }
337
338 _build_rails_package_scripts() {
339     local pkgname="$1"; shift
340     local destdir="$1"; shift
341     local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
342     for scriptname in postinst prerm postrm; do
343         cat "$srcdir/$pkgname.sh" "$srcdir/$scriptname.sh" \
344             >"$destdir/$scriptname" || return $?
345     done
346 }
347
348 rails_package_version() {
349     local pkgname="$1"; shift
350     local srcdir="$1"; shift
351     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
352         echo "$ARVADOS_BUILDING_VERSION"
353         return
354     fi
355     local version="$(version_from_git)"
356     if [ $pkgname = "arvados-api-server" ] ; then
357         calculate_go_package_version version cmd/arvados-server "$srcdir"
358     fi
359     echo $version
360 }
361
362 test_rails_package_presence() {
363   local pkgname="$1"; shift
364   local srcdir="$1"; shift
365
366   if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
367     return 1
368   fi
369
370   tmppwd=`pwd`
371
372   cd $srcdir
373
374   local version="$(rails_package_version "$pkgname" "$srcdir")"
375
376   cd $tmppwd
377
378   test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
379 }
380
381 get_complete_package_name() {
382   # if the errexit flag is set, unset it until this function returns
383   # otherwise, the shift calls below will abort the program if optional arguments are not supplied
384   if [ -o errexit ]; then
385     set +e
386     trap 'set -e' RETURN
387   fi
388   # $__returnvar has the nameref attribute set, which means it is a reference
389   # to another variable that is passed in as the first argument to this function.
390   # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
391   local -n __returnvar="$1"; shift
392   local pkgname="$1"; shift
393   local version="$1"; shift
394   local pkgtype="$1"; shift
395   local iteration="$1"; shift
396   local arch="$1"; shift
397   if [[ "$iteration" == "" ]]; then
398       iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
399   fi
400
401   if [[ "$arch" == "" ]]; then
402     native_arch=$(get_native_arch)
403     rpm_native_arch="x86_64"
404     rpm_architecture="$rpm_native_arch"
405     deb_architecture="$native_arch"
406
407     if [[ "$pkgtype" =~ ^(src)$ ]]; then
408       rpm_architecture="noarch"
409       deb_architecture="all"
410     fi
411   else
412     rpm_architecture=$arch
413     deb_architecture=$arch
414   fi
415
416   local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
417   if [[ "$FORMAT" == "rpm" ]]; then
418       # rpm packages get iteration 1 if we don't supply one
419       iteration=${iteration:-1}
420       complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
421   fi
422   __returnvar=${complete_pkgname}
423 }
424
425 # Test if the package already exists, if not return 0, if it does return 1
426 test_package_presence() {
427     local pkgname="$1"; shift
428     local version="$1"; shift
429     local pkgtype="$1"; shift
430     local iteration="$1"; shift
431     local arch="$1"; shift
432     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
433         return 1
434     fi
435
436     local full_pkgname
437     get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
438
439     # See if we can skip building the package, only if it already exists in the
440     # processed/ directory. If so, move it back to the packages directory to make
441     # sure it gets picked up by the test and/or upload steps.
442     # Get the list of packages from the repos
443
444     local pkg_url
445     if [[ "$FORCE_BUILD" == "1" ]]; then
446       echo "Package $full_pkgname build forced with --force-build, building"
447       return 0
448     elif [[ "$FORMAT" == "deb" ]]; then
449       local codename
450       case "$TARGET" in
451           debian11) codename=bullseye ;;
452           debian12) codename=bookworm ;;
453           ubuntu2004) codename=focal ;;
454           ubuntu2204) codename=jammy ;;
455           ubuntu2404) codename=noble ;;
456           *)
457               echo "FIXME: Don't know deb URL path for $TARGET, building"
458               return 0
459               ;;
460       esac
461       local repo_subdir
462       if [ ${pkgname:0:3} = "lib" ]; then
463         repo_subdir=${pkgname:0:4}
464       else
465         repo_subdir=${pkgname:0:1}
466       fi
467       pkg_url="http://apt.arvados.org/$codename/pool/main/$repo_subdir/$pkgname/$full_pkgname"
468     else
469       local rpm_root
470       case "$TARGET" in
471         rocky8 | rocky9) rpm_root="RHEL/${TARGET#rocky}/dev" ;;
472         *)
473           echo "FIXME: Don't know RPM URL path for $TARGET, building"
474           return 0
475           ;;
476       esac
477       pkg_url="https://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
478     fi
479
480     if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$pkg_url"; then
481       echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
482       return 1
483     elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
484       echo "Package $full_pkgname exists, not rebuilding!"
485       return 1
486     else
487       echo "Package $full_pkgname not found, building"
488       return 0
489     fi
490 }
491
492 handle_rails_package() {
493     local pkgname="$1"; shift
494
495     if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
496         return 0
497     fi
498     local srcdir="$1"; shift
499     cd "$srcdir"
500     local license_path="$1"; shift
501     local version="$(rails_package_version "$pkgname" "$srcdir")"
502     echo "$version" >package-build.version
503     local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
504     (
505         set -e
506         _build_rails_package_scripts "$pkgname" "$scripts_dir"
507         cd "$srcdir"
508         mkdir -p tmp
509         git rev-parse HEAD >git-commit.version
510         # Please make sure you read `bundle help config` carefully before you
511         # modify any of these settings. Some of their names are not intuitive.
512         #
513         # `bundle cache` caches from Git and paths, not just rubygems.org.
514         bundle config set cache_all true
515         # Disallow changes to Gemfile.
516         bundle config set deployment true
517         # Avoid loading system-wide gems (although this seems to not work 100%).
518         bundle config set disable_shared_gems true
519         # `bundle cache` only downloads gems, doesn't install them.
520         # Our Rails postinst script does the install step.
521         bundle config set no_install true
522         # As of April 2024/Bundler 2.4, `bundle cache` seems to skip downloading
523         # gems that are already available system-wide... and then it complains
524         # that your bundle is incomplete. Work around this by fetching gems
525         # manually.
526         # `--max-procs=6` is an abritrary number to download in parallel
527         # while being at least a little polite to rubygems.org.
528         # TODO: Once all our supported distros have Ruby 3+, we can modify
529         # the awk script to print "NAME:VERSION" output, and pipe that directly
530         # to `xargs -0r gem fetch` for reduced overhead.
531         mkdir -p vendor/cache
532         awk -- '
533 BEGIN { OFS="\0"; ORS="\0"; }
534 (/^[A-Z ]*$/) { level1=$0; }
535 (/^  [[:alpha:]]+:$/) { level2=substr($0, 3, length($0) - 3); next; }
536 (/^ {0,3}[[:alpha:]]/) { level2=""; next; }
537 (level1 == "GEM" && level2 == "specs" && NF == 2 && $1 ~ /^[[:alpha:]][-_[:alnum:]]*$/ && $2 ~ /\([[:digit:]]+[-_+.[:alnum:]]*\)$/) {
538     print "--version", substr($2, 2, length($2) - 2), $1;
539 }
540 ' Gemfile.lock | env -C vendor/cache xargs -0r --max-args=3 --max-procs=6 gem fetch
541         # Despite the bug, we still run `bundle cache` to make sure Bundler is
542         # happy for later steps.
543         # Tip: If this command removes "stale" gems downloaded in the previous
544         # step, that might mean those gems declare that the version of Ruby
545         # you're running is too new.
546         bundle cache
547     )
548     if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
549         echo "ERROR: $pkgname package prep failed" >&2
550         rm -rf "$scripts_dir"
551         EXITCODE=1
552         return 1
553     fi
554     local railsdir="/var/www/${pkgname%-server}/current"
555     local -a pos_args=("$srcdir/=$railsdir" "$pkgname" dir "$version")
556     local -a switches=(--after-install "$scripts_dir/postinst"
557                        --before-remove "$scripts_dir/prerm"
558                        --after-remove "$scripts_dir/postrm")
559     if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
560         switches+=(--iteration $RAILS_PACKAGE_ITERATION)
561     fi
562     # For some reason fpm excludes need to not start with /.
563     local exclude_root="${railsdir#/}"
564     for exclude in tmp log coverage Capfile\* \
565                        config/deploy\* \
566                        config/application.yml \
567                        config/database.yml \
568                        \*.service; do
569         switches+=(-x "$exclude_root/$exclude")
570     done
571     fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
572               -x "$exclude_root/vendor/cache-*" \
573               -x "$exclude_root/vendor/bundle" "$@" \
574               "$license_path=$railsdir/$(basename "$license_path")" \
575               "$srcdir/arvados-railsapi.service=/lib/systemd/system/arvados-railsapi.service"
576     rm -rf "$scripts_dir"
577 }
578
579 # Usage: handle_api_server [amd64]
580 handle_api_server () {
581   local target_arch="${1:-amd64}"; shift
582
583   if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then
584     debug_echo -e "Skipping build of arvados-api-server package."
585     return 0
586   fi
587
588   native_arch=$(get_native_arch)
589   if [[ "$target_arch" != "$native_arch" ]]; then
590     echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH"
591     echo
592     exit 1
593   fi
594
595   # Build the API server package
596   test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
597   if [[ "$?" == "0" ]]; then
598     calculate_go_package_version arvados_server_version cmd/arvados-server
599     arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
600     handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
601         "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
602         --description="Arvados API server - Arvados is a free and open source platform for big data science." \
603         --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
604   fi
605 }
606
607 # Usage: handle_arvados_src
608 handle_arvados_src () {
609   if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
610     debug_echo -e "Skipping build of arvados-src package."
611     return 0
612   fi
613   # arvados-src
614   (
615       cd "$WORKSPACE"
616       COMMIT_HASH=$(format_last_commit_here "%H")
617       arvados_src_version="$(version_from_git)"
618
619       cd $WORKSPACE/packages/$TARGET
620       test_package_presence arvados-src "$arvados_src_version" src ""
621
622       if [[ "$?" == "0" ]]; then
623         cd "$WORKSPACE"
624         SRC_BUILD_DIR=$(mktemp -d)
625         # mktemp creates the directory with 0700 permissions by default
626         chmod 755 $SRC_BUILD_DIR
627         git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
628         cd "$SRC_BUILD_DIR"
629
630         # go into detached-head state
631         git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
632         echo "$COMMIT_HASH" >git-commit.version
633
634         cd $WORKSPACE/packages/$TARGET
635         fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
636
637         rm -rf "$SRC_BUILD_DIR"
638       fi
639   )
640 }
641
642 setup_build_virtualenv() {
643     PYTHON_BUILDROOT="$(mktemp --directory --tmpdir pybuild.XXXXXXXX)"
644     "$PYTHON3_EXECUTABLE" -m venv "$PYTHON_BUILDROOT/venv"
645     "$PYTHON_BUILDROOT/venv/bin/pip" install -r "$WORKSPACE/build/requirements.build-packages.txt"
646     mkdir "$PYTHON_BUILDROOT/wheelhouse"
647 }
648
649 # Build python packages with a virtualenv built-in
650 # Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64]
651 fpm_build_virtualenv () {
652   local pkg=$1; shift
653   local pkg_dir=$1; shift
654   local package_format="$1"; shift
655   local target_arch="${1:-amd64}"; shift
656
657   fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" amd64 amd64
658 }
659
660 # Build python packages with a virtualenv built-in
661 # Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64] [amd64]
662 fpm_build_virtualenv_worker () {
663   PKG=$1; shift
664   PKG_DIR=$1; shift
665   local package_format="$1"; shift
666   local native_arch="${1:-amd64}"; shift
667   local target_arch=${1:-amd64}; shift
668
669   # Set up
670   STDOUT_IF_DEBUG=/dev/null
671   STDERR_IF_DEBUG=/dev/null
672   DASHQ_UNLESS_DEBUG=-q
673   if [[ "$DEBUG" != "0" ]]; then
674       STDOUT_IF_DEBUG=/dev/stdout
675       STDERR_IF_DEBUG=/dev/stderr
676       DASHQ_UNLESS_DEBUG=
677   fi
678   if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
679     ARVADOS_BUILDING_ITERATION=1
680   fi
681
682   PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
683   if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
684     PYTHON_PKG=$PACKAGE_PREFIX-$PKG
685   else
686     # Exception to our package naming convention
687     PYTHON_PKG=$PKG
688   fi
689
690   # We must always add a wheel to our repository, even if we're not building
691   # this distro package, because it might be a dependency for a later
692   # package we do build.
693   if [[ "$PKG_DIR" =~ ^.=[0-9]+\. ]]; then
694       # Not source to build, but a version to download.
695       # The rest of the function expects a filesystem path, so set one afterwards.
696       "$PYTHON_BUILDROOT/venv/bin/pip" download --dest="$PYTHON_BUILDROOT/wheelhouse" "$PKG$PKG_DIR" \
697           && PKG_DIR="$PYTHON_BUILDROOT/nonexistent"
698   else
699       # Make PKG_DIR absolute.
700       PKG_DIR="$(env -C "$WORKSPACE" readlink -e "$PKG_DIR")"
701       "$PYTHON_BUILDROOT/venv/bin/python" -m build --outdir="$PYTHON_BUILDROOT/wheelhouse" "$PKG_DIR"
702   fi
703   if [[ $? -ne 0 ]]; then
704     printf "Error, unable to download/build wheel for %s @ %s\n" "$PKG" "$PKG_DIR"
705     exit 1
706   fi
707
708   if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
709     return 0
710   elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
711     printf "Error, unable to update local wheel repository\n"
712     exit 1
713   fi
714
715   local venv_dir="/usr/lib/$PYTHON_PKG"
716   echo "Creating virtualenv..."
717   if ! "$PYTHON3_EXECUTABLE" -m venv "$venv_dir"; then
718     printf "Error, unable to run\n  %s -m venv %s\n" "$PYTHON3_EXECUTABLE" "$venv_dir"
719     exit 1
720   # We must have the dependency resolver introduced in late 2020 for the rest
721   # of our install process to work.
722   # <https://blog.python.org/2020/11/pip-20-3-release-new-resolver.html>
723   elif ! "$venv_dir/bin/pip" install "pip>=20.3"; then
724     printf "Error, unable to run\n  %s/bin/pip install 'pip>=20.3'\n" "$venv_dir"
725     exit 1
726   fi
727
728   local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
729   if [[ -z "$pip_wheel" ]]; then
730     printf "Error, unable to find built wheel for $PKG\n"
731     exit 1
732   elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
733     printf "Error, unable to run
734   %s/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url=file://%s %s
735 " "$venv_dir" "$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"
736     exit 1
737   fi
738
739   # Determine the package version from the wheel
740   PYTHON_VERSION="$("$venv_dir/bin/python" "$WORKSPACE/build/pypkg_info.py" metadata "$PKG" Version)"
741   UNFILTERED_PYTHON_VERSION="$(echo "$PYTHON_VERSION" | sed 's/\.dev/~dev/; s/\([0-9]\)rc/\1~rc/')"
742
743   # See if we actually need to build this package; does it exist already?
744   # We can't do this earlier than here, because we need PYTHON_VERSION.
745   if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" python3 "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
746     return 0
747   fi
748   echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
749
750   # Using `env -C` sets the directory where the package is built.
751   # Using `fpm --chdir` sets the root directory for source arguments.
752   declare -a COMMAND_ARR=(
753       env -C "$PYTHON_BUILDROOT" fpm
754       --chdir="$venv_dir"
755       --name="$PYTHON_PKG"
756       --version="$UNFILTERED_PYTHON_VERSION"
757       --input-type=dir
758       --output-type="$package_format"
759       --depends="$PYTHON3_PACKAGE"
760       --iteration="$ARVADOS_BUILDING_ITERATION"
761       --replaces="python-$PKG"
762       --url="https://arvados.org"
763   )
764   # Append fpm flags corresponding to Python package metadata.
765   readarray -d "" -O "${#COMMAND_ARR[@]}" -t COMMAND_ARR < \
766             <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" \
767                                       --delimiter=\\0 --format=fpm \
768                                       metadata "$PKG" License Summary)
769
770   if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
771     COMMAND_ARR+=("-a$target_arch")
772   fi
773
774   if [[ "$MAINTAINER" != "" ]]; then
775     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
776   fi
777
778   if [[ "$VENDOR" != "" ]]; then
779     COMMAND_ARR+=('--vendor' "$VENDOR")
780   fi
781
782   if [[ "$DEBUG" != "0" ]]; then
783     COMMAND_ARR+=('--verbose' '--log' 'info')
784   fi
785
786   systemd_unit="$PKG_DIR/$PKG.service"
787   if [[ -e "${systemd_unit}" ]]; then
788     COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
789     COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
790   fi
791
792   case "$package_format" in
793       deb)
794           COMMAND_ARR+=(
795               # Avoid warning
796               --deb-no-default-config-files
797           ) ;;
798       rpm)
799           COMMAND_ARR+=(
800               # Conflict with older packages we used to publish
801               --conflicts "rh-python36-python-$PKG"
802               # Do not generate /usr/lib/.build-id links on RH8+
803               # (otherwise our packages conflict with platform-python)
804               --rpm-rpmbuild-define "_build_id_links none"
805           ) ;;
806   esac
807
808   # Append --depends X and other arguments specified by fpm-info.sh in
809   # the package source dir. These are added last so they can override
810   # the arguments added by this script.
811   declare -a fpm_args=()
812   declare -a fpm_depends=()
813
814   fpminfo="$PKG_DIR/fpm-info.sh"
815   if [[ -e "$fpminfo" ]]; then
816     echo "Loading fpm overrides from $fpminfo"
817     if ! source "$fpminfo"; then
818       echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
819       exit 1
820     fi
821   fi
822
823   for i in "${fpm_depends[@]}"; do
824     COMMAND_ARR+=('--depends' "$i")
825   done
826
827   # make sure the systemd service file ends up in the right place
828   # used by arvados-docker-cleaner
829   if [[ -e "${systemd_unit}" ]]; then
830     COMMAND_ARR+=("share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
831   fi
832
833   COMMAND_ARR+=("${fpm_args[@]}")
834
835   while read -d "" binpath; do
836       COMMAND_ARR+=("$binpath=/usr/$binpath")
837   done < <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" --delimiter=\\0 binfiles "$PKG")
838
839   # the python3-arvados-cwl-runner package comes with cwltool, expose that version
840   if [[ "$PKG" == arvados-cwl-runner ]]; then
841     COMMAND_ARR+=("bin/cwltool=/usr/bin/cwltool")
842   fi
843
844   COMMAND_ARR+=(".=$venv_dir")
845
846   debug_echo -e "\n${COMMAND_ARR[@]}\n"
847
848   FPM_RESULTS=$("${COMMAND_ARR[@]}")
849   FPM_EXIT_CODE=$?
850
851   # if something went wrong and debug is off, print out the fpm command that errored
852   if ! fpm_verify $FPM_EXIT_CODE $FPM_RESULTS && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
853     echo "fpm returned an error executing the command:"
854     echo
855     echo -e "\n${COMMAND_ARR[@]}\n"
856   else
857     ls "$PYTHON_BUILDROOT"/*."$package_format"
858     mv "$PYTHON_BUILDROOT"/*."$package_format" "$WORKSPACE/packages/$TARGET/"
859   fi
860   echo
861 }
862
863 # Build packages for everything
864 fpm_build() {
865   # Source dir where fpm-info.sh (if any) will be found.
866   SRC_DIR=$1
867   shift
868   # The package source.  Depending on the source type, this can be a
869   # path, or the name of the package in an upstream repository (e.g.,
870   # pip).
871   PACKAGE=$1
872   shift
873   # The name of the package to build.
874   PACKAGE_NAME=$1
875   shift
876   # The type of source package.  Passed to fpm -s.  Default "dir".
877   PACKAGE_TYPE=${1:-dir}
878   shift
879   # Optional: the package version number.  Passed to fpm -v.
880   VERSION=$1
881   shift
882
883   if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
884       return 0
885   fi
886
887   local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
888
889   declare -a COMMAND_ARR=("fpm" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
890   if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
891       # Dependencies are built from Python package metadata.  Since that
892       # will never refer to Debian package iterations, it doesn't make sense
893       # to enforce those in the .deb dependencies.
894       COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
895   fi
896
897   if [[ "$DEBUG" != "0" ]]; then
898     COMMAND_ARR+=('--verbose' '--log' 'info')
899   fi
900
901   if [[ -n "$PACKAGE_NAME" ]]; then
902     COMMAND_ARR+=('-n' "$PACKAGE_NAME")
903   fi
904
905   if [[ "$MAINTAINER" != "" ]]; then
906     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
907   fi
908
909   if [[ "$VENDOR" != "" ]]; then
910     COMMAND_ARR+=('--vendor' "$VENDOR")
911   fi
912
913   if [[ "$VERSION" != "" ]]; then
914     COMMAND_ARR+=('-v' "$VERSION")
915   fi
916   if [[ -n "$default_iteration_value" ]]; then
917       # We can always add an --iteration here.  If another one is specified in $@,
918       # that will take precedence, as desired.
919       COMMAND_ARR+=(--iteration "$default_iteration_value")
920   fi
921
922   # Append --depends X and other arguments specified by fpm-info.sh in
923   # the package source dir. These are added last so they can override
924   # the arguments added by this script.
925   declare -a fpm_args=()
926   declare -a build_depends=()
927   declare -a fpm_depends=()
928   declare -a fpm_conflicts=()
929   declare -a fpm_exclude=()
930   if [[ ! -d "$SRC_DIR" ]]; then
931       echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
932       exit 1
933   fi
934   fpminfo="${SRC_DIR}/fpm-info.sh"
935   if [[ -e "$fpminfo" ]]; then
936       debug_echo "Loading fpm overrides from $fpminfo"
937       source "$fpminfo"
938   fi
939   for pkg in "${build_depends[@]}"; do
940       if [[ $TARGET =~ debian|ubuntu ]]; then
941           pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
942           if [[ -e $pkg_deb ]]; then
943               echo "Installing build_dep $pkg from $pkg_deb"
944               dpkg -i "$pkg_deb"
945           else
946               echo "Attemping to install build_dep $pkg using apt-get"
947               apt-get install -y "$pkg"
948           fi
949           apt-get -y -f install
950       else
951           pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
952           if [[ -e $pkg_rpm ]]; then
953               echo "Installing build_dep $pkg from $pkg_rpm"
954               rpm -i "$pkg_rpm"
955           else
956               echo "Attemping to install build_dep $pkg"
957               rpm -i "$pkg"
958           fi
959       fi
960   done
961   for i in "${fpm_depends[@]}"; do
962     COMMAND_ARR+=('--depends' "$i")
963   done
964   for i in "${fpm_conflicts[@]}"; do
965     COMMAND_ARR+=('--conflicts' "$i")
966   done
967   for i in "${fpm_exclude[@]}"; do
968     COMMAND_ARR+=('--exclude' "$i")
969   done
970
971   COMMAND_ARR+=("${fpm_args[@]}")
972
973   # Append remaining function arguments directly to fpm's command line.
974   for i; do
975     COMMAND_ARR+=("$i")
976   done
977
978   COMMAND_ARR+=("$PACKAGE")
979
980   debug_echo -e "\n${COMMAND_ARR[@]}\n"
981
982   FPM_RESULTS=$("${COMMAND_ARR[@]}")
983   FPM_EXIT_CODE=$?
984   echo "fpm: exit code $FPM_EXIT_CODE" >>$STDOUT_IF_DEBUG
985   echo "$FPM_RESULTS" >>$STDOUT_IF_DEBUG
986
987   fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
988
989   # if something went wrong and debug is off, print out the fpm command that errored
990   if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
991     echo -e "\n${COMMAND_ARR[@]}\n"
992   fi
993 }
994
995 # verify build results
996 fpm_verify () {
997   FPM_EXIT_CODE=$1
998   shift
999   FPM_RESULTS=$@
1000
1001   FPM_PACKAGE_NAME=''
1002   if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.~-]*\.)(deb|rpm) ]]; then
1003     FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
1004   fi
1005
1006   if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
1007     EXITCODE=1
1008     echo
1009     echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
1010     echo
1011     echo $FPM_RESULTS
1012     echo
1013     return 1
1014   elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
1015     echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
1016     return 0
1017   elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
1018     EXITCODE=1
1019     echo "Error building package for $1:\n $FPM_RESULTS"
1020     return 1
1021   fi
1022 }
1023
1024 install_package() {
1025   PACKAGES=$@
1026   if [[ "$FORMAT" == "deb" ]]; then
1027     $SUDO apt-get install $PACKAGES --yes
1028   elif [[ "$FORMAT" == "rpm" ]]; then
1029     $SUDO yum -q -y install $PACKAGES
1030   fi
1031 }
1032
1033 title() {
1034     printf '%s %s\n' "=======" "$1"
1035 }
1036
1037 checkexit() {
1038     if [[ "$1" != "0" ]]; then
1039         title "$2 -- FAILED"
1040         failures+=("$2 (`timer`)")
1041     else
1042         successes+=("$2 (`timer`)")
1043     fi
1044 }
1045
1046 timer_reset() {
1047     t0=$SECONDS
1048 }
1049
1050 timer() {
1051     if [[ -n "$t0" ]]; then
1052         echo -n "$(($SECONDS - $t0))s"
1053     fi
1054 }
1055
1056 report_outcomes() {
1057     for x in "${successes[@]}"
1058     do
1059         echo "Pass: $x"
1060     done
1061
1062     if [[ ${#failures[@]} == 0 ]]
1063     then
1064         if [[ ${#successes[@]} != 0 ]]; then
1065            echo "All test suites passed."
1066         fi
1067     else
1068         echo "Failures (${#failures[@]}):"
1069         for x in "${failures[@]}"
1070         do
1071             echo "Fail: $x"
1072         done
1073     fi
1074 }