2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 # A library of functions shared by the various scripts in this directory.
8 # This is the timestamp about when we merged changed to include licenses
9 # with Arvados packages. We use it as a heuristic to add revisions for
11 LICENSE_PACKAGE_TS=20151208015500
12 RAILS_PACKAGE_ITERATION="${ARVADOS_BUILDING_ITERATION:-1}"
15 echo "$@" >"$STDOUT_IF_DEBUG"
18 find_python_program() {
22 if "$prog" --version >/dev/null 2>&1; then
30 Error: $prog (from Python setuptools module) not found
36 # get_ci_scripts sets $CI_DIR to the path of a directory with CI scripts.
37 # If it is not already set, it uses the following to get them by creating a
38 # temporary Git worktree:
39 # * CI_SRC: a Git checkout of the scripts (default $WORKSPACE)
40 # * CI_REF: the reference used to create that (default `remotes/arvados-ci/ci-build`)
41 # * CI_PATH: the path of CI scripts under the worktree (default `/jenkins`)
42 # The defaults are all suitable for jobs running under ci.arvados.org, but
43 # you can set them in the environment to customize the behavior, or just set
44 # $CI_DIR to a path that already has the scripts ready.
46 if [ -n "${CI_DIR:-}" ]; then
49 local clone_dir="$(mktemp --directory --tmpdir="${WORKSPACE_TMP:-}")" &&
50 git -C "${CI_SRC:-$WORKSPACE}" worktree add "$clone_dir" "${CI_REF:-remotes/arvados-ci/ci-build}" ||
52 CI_DIR="$clone_dir${CI_PATH:-/jenkins}"
55 format_last_commit_here() {
56 local format="$1"; shift
57 local dir="${1:-.}"; shift
58 TZ=UTC git log -n1 --first-parent "--format=format:$format" "$dir"
62 # Output the version being built, or if we're building a
63 # dev/prerelease, output a version number based on the git log for
65 local subdir="$1"; shift
66 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
67 echo "$ARVADOS_BUILDING_VERSION"
72 declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
73 ARVADOS_BUILDING_VERSION="$($WORKSPACE/build/version-at-commit.sh $git_hash)"
74 echo "$ARVADOS_BUILDING_VERSION"
77 nohash_version_from_git() {
78 local subdir="$1"; shift
79 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
80 echo "$ARVADOS_BUILDING_VERSION"
83 version_from_git $subdir | cut -d. -f1-4
86 timestamp_from_git() {
87 local subdir="$1"; shift
88 format_last_commit_here "%ct" "$subdir"
91 # Usage: get_native_arch
93 # Only amd64 and aarch64 are supported at the moment
100 echo "Error: architecture not supported"
108 local gem_name="$1"; shift
109 local gem_version="$(nohash_version_from_git)"
110 local gem_src_dir="$(pwd)"
112 if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
116 if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
117 find -maxdepth 1 -name "${gem_name}-*.gem" -delete
119 # -q appears to be broken in gem version 2.2.2
120 gem build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
124 # Usage: package_workbench2
125 package_workbench2() {
126 local pkgname=arvados-workbench2
127 local src=services/workbench2
128 local dst=/var/www/arvados-workbench2/workbench2
129 local description="Arvados Workbench 2"
130 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
134 local version="$(version_from_git)"
136 NODE_ENV=production yarn install
137 VERSION="$version" BUILD_NUMBER="$(default_iteration "$pkgname" "$version" yarn)" GIT_COMMIT="$(git rev-parse HEAD | head -c9)" yarn build
138 cd "$WORKSPACE/packages/$TARGET"
139 fpm_build "${WORKSPACE}/$src" "${WORKSPACE}/$src/build/=$dst" "$pkgname" dir "$version" \
140 --license="GNU Affero General Public License, version 3.0" \
141 --description="${description}" \
142 --config-files="/etc/arvados/$pkgname/workbench2.example.json" \
143 "$WORKSPACE/services/workbench2/etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$pkgname/workbench2.example.json"
146 calculate_go_package_version() {
147 # $__returnvar has the nameref attribute set, which means it is a reference
148 # to another variable that is passed in as the first argument to this function.
149 # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
150 local -n __returnvar="$1"; shift
156 # Update the version number and build a new package if the vendor
157 # bundle has changed, or the command imports anything from the
158 # Arvados SDK and the SDK has changed.
159 declare -a checkdirs=(go.mod go.sum)
160 while [ -n "$1" ]; do
164 # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
165 # Everything depends on the build directory.
166 checkdirs+=(sdk/go lib build)
168 for dir in ${checkdirs[@]}; do
170 ts="$(timestamp_from_git "$dir")"
171 if [[ "$ts" -gt "$timestamp" ]]; then
172 version=$(version_from_git "$dir")
177 __returnvar="$version"
180 # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64] "Compute foo to arbitrary precision" [apache-2.0.txt]
181 package_go_binary() {
182 local src_path="$1"; shift
183 local prog="$1"; shift
184 local package_format="$1"; shift
185 local target_arch="$1"; shift
186 local description="$1"; shift
187 local license_file="${1:-agpl-3.0.txt}"; shift
189 if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
190 debug_echo -e "Skipping build of $prog package."
194 native_arch=$(get_native_arch)
196 if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
197 echo "Error: no cross compilation support for Go on $native_arch, can not build $prog for $target_arch"
201 case "$package_format-$TARGET" in
202 # Ubuntu 20.04 does not support cross compilation because the
203 # libfuse package does not support multiarch. See
204 # <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477>.
205 # Red Hat-based distributions do not support native cross compilation at
206 # all (they use a qemu-based solution we haven't implemented yet).
207 deb-ubuntu2004|rpm-*)
209 if [[ "$native_arch" == "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
210 echo "Error: no cross compilation support for Go on $native_arch for $TARGET, can not build $prog for $target_arch"
219 if [[ -n "$target_arch" ]]; then
222 # No target architecture specified, default to native target.
226 for ta in ${archs[@]}; do
227 package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
229 if [[ $retval -ne 0 ]]; then
235 # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64] [amd64] [apache-2.0.txt]
236 package_go_binary_worker() {
237 local src_path="$1"; shift
238 local prog="$1"; shift
239 local package_format="$1"; shift
240 local description="$1"; shift
241 local native_arch="${1:-amd64}"; shift
242 local target_arch="${1:-amd64}"; shift
243 local license_file="${1:-agpl-3.0.txt}"; shift
245 debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
246 local basename="${src_path##*/}"
247 calculate_go_package_version go_package_version $src_path
249 cd $WORKSPACE/packages/$TARGET
250 test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
251 if [[ $? -ne 0 ]]; then
255 echo "Building $package_format ($target_arch) package for $prog from $src_path"
256 GOARCH=${arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
260 binpath=$GOPATH/bin/${basename}
261 if [[ "${target_arch}" != "${native_arch}" ]]; then
262 switches+=("-a${target_arch}")
263 binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
266 case "$package_format" in
267 # As of April 2024 we package identical Go binaries under different
268 # packages and names. This upsets the build id database, so don't
269 # register ourselves there.
270 rpm) switches+=(--rpm-rpmbuild-define="_build_id_links none") ;;
273 systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
274 if [[ -e "${systemd_unit}" ]]; then
276 --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
277 --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
278 "${systemd_unit}=/lib/systemd/system/${prog}.service")
280 switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
282 fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
285 # Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
287 local src_path="$1"; shift
288 local sofile="$1"; shift
289 local pkg="$1"; shift
290 local package_format="$1"; shift
291 local target_arch="$1"; shift # supported: amd64
292 local description="$1"; shift
294 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
295 debug_echo -e "Skipping build of $pkg package."
299 debug_echo "package_go_so $src_path as $pkg"
301 calculate_go_package_version go_package_version $src_path
302 cd $WORKSPACE/packages/$TARGET
303 test_package_presence $pkg $go_package_version go || return 1
304 cd $WORKSPACE/$src_path
305 go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
306 cd $WORKSPACE/packages/$TARGET
308 "--url=https://arvados.org"
309 "--license=Apache License, Version 2.0"
310 "--description=${description}"
311 "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
313 if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
314 fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go")
316 if [[ -e "$WORKSPACE/$src_path/README" ]]; then
317 fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
319 fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
322 default_iteration() {
323 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
324 echo "$ARVADOS_BUILDING_ITERATION"
327 local package_name="$1"; shift
328 local package_version="$1"; shift
329 local package_type="$1"; shift
331 if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
332 [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
338 _build_rails_package_scripts() {
339 local pkgname="$1"; shift
340 local destdir="$1"; shift
341 local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
342 for scriptname in postinst prerm postrm; do
343 cat "$srcdir/$pkgname.sh" "$srcdir/$scriptname.sh" \
344 >"$destdir/$scriptname" || return $?
348 rails_package_version() {
349 local pkgname="$1"; shift
350 local srcdir="$1"; shift
351 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
352 echo "$ARVADOS_BUILDING_VERSION"
355 local version="$(version_from_git)"
356 if [ $pkgname = "arvados-api-server" ] ; then
357 calculate_go_package_version version cmd/arvados-server "$srcdir"
362 test_rails_package_presence() {
363 local pkgname="$1"; shift
364 local srcdir="$1"; shift
366 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
374 local version="$(rails_package_version "$pkgname" "$srcdir")"
378 test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
381 get_complete_package_name() {
382 # if the errexit flag is set, unset it until this function returns
383 # otherwise, the shift calls below will abort the program if optional arguments are not supplied
384 if [ -o errexit ]; then
388 # $__returnvar has the nameref attribute set, which means it is a reference
389 # to another variable that is passed in as the first argument to this function.
390 # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
391 local -n __returnvar="$1"; shift
392 local pkgname="$1"; shift
393 local version="$1"; shift
394 local pkgtype="$1"; shift
395 local iteration="$1"; shift
396 local arch="$1"; shift
397 if [[ "$iteration" == "" ]]; then
398 iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
401 if [[ "$arch" == "" ]]; then
402 native_arch=$(get_native_arch)
403 rpm_native_arch="x86_64"
404 rpm_architecture="$rpm_native_arch"
405 deb_architecture="$native_arch"
407 if [[ "$pkgtype" =~ ^(src)$ ]]; then
408 rpm_architecture="noarch"
409 deb_architecture="all"
412 rpm_architecture=$arch
413 deb_architecture=$arch
416 local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
417 if [[ "$FORMAT" == "rpm" ]]; then
418 # rpm packages get iteration 1 if we don't supply one
419 iteration=${iteration:-1}
420 complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
422 __returnvar=${complete_pkgname}
425 # Test if the package already exists, if not return 0, if it does return 1
426 test_package_presence() {
427 local pkgname="$1"; shift
428 local version="$1"; shift
429 local pkgtype="$1"; shift
430 local iteration="$1"; shift
431 local arch="$1"; shift
432 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
437 get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
439 # See if we can skip building the package, only if it already exists in the
440 # processed/ directory. If so, move it back to the packages directory to make
441 # sure it gets picked up by the test and/or upload steps.
442 # Get the list of packages from the repos
445 if [[ "$FORCE_BUILD" == "1" ]]; then
446 echo "Package $full_pkgname build forced with --force-build, building"
448 elif [[ "$FORMAT" == "deb" ]]; then
451 debian11) codename=bullseye ;;
452 debian12) codename=bookworm ;;
453 ubuntu2004) codename=focal ;;
454 ubuntu2204) codename=jammy ;;
455 ubuntu2404) codename=noble ;;
457 echo "FIXME: Don't know deb URL path for $TARGET, building"
462 if [ ${pkgname:0:3} = "lib" ]; then
463 repo_subdir=${pkgname:0:4}
465 repo_subdir=${pkgname:0:1}
467 pkg_url="http://apt.arvados.org/$codename/pool/main/$repo_subdir/$pkgname/$full_pkgname"
471 rocky8 | rocky9) rpm_root="RHEL/${TARGET#rocky}/dev" ;;
473 echo "FIXME: Don't know RPM URL path for $TARGET, building"
477 pkg_url="https://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
480 if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$pkg_url"; then
481 echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
483 elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
484 echo "Package $full_pkgname exists, not rebuilding!"
487 echo "Package $full_pkgname not found, building"
492 handle_rails_package() {
493 local pkgname="$1"; shift
495 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
498 local srcdir="$1"; shift
500 local license_path="$1"; shift
501 local version="$(rails_package_version "$pkgname" "$srcdir")"
502 echo "$version" >package-build.version
503 local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
506 _build_rails_package_scripts "$pkgname" "$scripts_dir"
509 git rev-parse HEAD >git-commit.version
510 # Please make sure you read `bundle help config` carefully before you
511 # modify any of these settings. Some of their names are not intuitive.
513 # `bundle cache` caches from Git and paths, not just rubygems.org.
514 bundle config set cache_all true
515 # Disallow changes to Gemfile.
516 bundle config set deployment true
517 # Avoid loading system-wide gems (although this seems to not work 100%).
518 bundle config set disable_shared_gems true
519 # `bundle cache` only downloads gems, doesn't install them.
520 # Our Rails postinst script does the install step.
521 bundle config set no_install true
522 # As of April 2024/Bundler 2.4, `bundle cache` seems to skip downloading
523 # gems that are already available system-wide... and then it complains
524 # that your bundle is incomplete. Work around this by fetching gems
526 # `--max-procs=6` is an abritrary number to download in parallel
527 # while being at least a little polite to rubygems.org.
528 # TODO: Once all our supported distros have Ruby 3+, we can modify
529 # the awk script to print "NAME:VERSION" output, and pipe that directly
530 # to `xargs -0r gem fetch` for reduced overhead.
531 mkdir -p vendor/cache
533 BEGIN { OFS="\0"; ORS="\0"; }
534 (/^[A-Z ]*$/) { level1=$0; }
535 (/^ [[:alpha:]]+:$/) { level2=substr($0, 3, length($0) - 3); next; }
536 (/^ {0,3}[[:alpha:]]/) { level2=""; next; }
537 (level1 == "GEM" && level2 == "specs" && NF == 2 && $1 ~ /^[[:alpha:]][-_[:alnum:]]*$/ && $2 ~ /\([[:digit:]]+[-_+.[:alnum:]]*\)$/) {
538 print "--version", substr($2, 2, length($2) - 2), $1;
540 ' Gemfile.lock | env -C vendor/cache xargs -0r --max-args=3 --max-procs=6 gem fetch
541 # Despite the bug, we still run `bundle cache` to make sure Bundler is
542 # happy for later steps.
543 # Tip: If this command removes "stale" gems downloaded in the previous
544 # step, that might mean those gems declare that the version of Ruby
545 # you're running is too new.
548 if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
549 echo "ERROR: $pkgname package prep failed" >&2
550 rm -rf "$scripts_dir"
554 local railsdir="/var/www/${pkgname%-server}/current"
555 local -a pos_args=("$srcdir/=$railsdir" "$pkgname" dir "$version")
556 local -a switches=(--after-install "$scripts_dir/postinst"
557 --before-remove "$scripts_dir/prerm"
558 --after-remove "$scripts_dir/postrm")
559 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
560 switches+=(--iteration $RAILS_PACKAGE_ITERATION)
562 # For some reason fpm excludes need to not start with /.
563 local exclude_root="${railsdir#/}"
564 for exclude in tmp log coverage Capfile\* \
566 config/application.yml \
567 config/database.yml \
569 switches+=(-x "$exclude_root/$exclude")
571 fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
572 -x "$exclude_root/vendor/cache-*" \
573 -x "$exclude_root/vendor/bundle" "$@" \
574 "$license_path=$railsdir/$(basename "$license_path")" \
575 "$srcdir/arvados-railsapi.service=/lib/systemd/system/arvados-railsapi.service"
576 rm -rf "$scripts_dir"
579 # Usage: handle_api_server [amd64]
580 handle_api_server () {
581 local target_arch="${1:-amd64}"; shift
583 if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then
584 debug_echo -e "Skipping build of arvados-api-server package."
588 native_arch=$(get_native_arch)
589 if [[ "$target_arch" != "$native_arch" ]]; then
590 echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH"
595 # Build the API server package
596 test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
597 if [[ "$?" == "0" ]]; then
598 calculate_go_package_version arvados_server_version cmd/arvados-server
599 arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
600 handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
601 "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
602 --description="Arvados API server - Arvados is a free and open source platform for big data science." \
603 --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
607 # Usage: handle_arvados_src
608 handle_arvados_src () {
609 if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
610 debug_echo -e "Skipping build of arvados-src package."
616 COMMIT_HASH=$(format_last_commit_here "%H")
617 arvados_src_version="$(version_from_git)"
619 cd $WORKSPACE/packages/$TARGET
620 test_package_presence arvados-src "$arvados_src_version" src ""
622 if [[ "$?" == "0" ]]; then
624 SRC_BUILD_DIR=$(mktemp -d)
625 # mktemp creates the directory with 0700 permissions by default
626 chmod 755 $SRC_BUILD_DIR
627 git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
630 # go into detached-head state
631 git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
632 echo "$COMMIT_HASH" >git-commit.version
634 cd $WORKSPACE/packages/$TARGET
635 fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
637 rm -rf "$SRC_BUILD_DIR"
642 setup_build_virtualenv() {
643 PYTHON_BUILDROOT="$(mktemp --directory --tmpdir pybuild.XXXXXXXX)"
644 "$PYTHON3_EXECUTABLE" -m venv "$PYTHON_BUILDROOT/venv"
645 "$PYTHON_BUILDROOT/venv/bin/pip" install -r "$WORKSPACE/build/requirements.build-packages.txt"
646 mkdir "$PYTHON_BUILDROOT/wheelhouse"
649 # Build python packages with a virtualenv built-in
650 # Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64]
651 fpm_build_virtualenv () {
653 local pkg_dir=$1; shift
654 local package_format="$1"; shift
655 local target_arch="${1:-amd64}"; shift
657 fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" amd64 amd64
660 # Build python packages with a virtualenv built-in
661 # Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64] [amd64]
662 fpm_build_virtualenv_worker () {
665 local package_format="$1"; shift
666 local native_arch="${1:-amd64}"; shift
667 local target_arch=${1:-amd64}; shift
670 STDOUT_IF_DEBUG=/dev/null
671 STDERR_IF_DEBUG=/dev/null
672 DASHQ_UNLESS_DEBUG=-q
673 if [[ "$DEBUG" != "0" ]]; then
674 STDOUT_IF_DEBUG=/dev/stdout
675 STDERR_IF_DEBUG=/dev/stderr
678 if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
679 ARVADOS_BUILDING_ITERATION=1
682 PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
683 if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
684 PYTHON_PKG=$PACKAGE_PREFIX-$PKG
686 # Exception to our package naming convention
690 # We must always add a wheel to our repository, even if we're not building
691 # this distro package, because it might be a dependency for a later
692 # package we do build.
693 if [[ "$PKG_DIR" =~ ^.=[0-9]+\. ]]; then
694 # Not source to build, but a version to download.
695 # The rest of the function expects a filesystem path, so set one afterwards.
696 "$PYTHON_BUILDROOT/venv/bin/pip" download --dest="$PYTHON_BUILDROOT/wheelhouse" "$PKG$PKG_DIR" \
697 && PKG_DIR="$PYTHON_BUILDROOT/nonexistent"
699 # Make PKG_DIR absolute.
700 PKG_DIR="$(env -C "$WORKSPACE" readlink -e "$PKG_DIR")"
701 "$PYTHON_BUILDROOT/venv/bin/python" -m build --outdir="$PYTHON_BUILDROOT/wheelhouse" "$PKG_DIR"
703 if [[ $? -ne 0 ]]; then
704 printf "Error, unable to download/build wheel for %s @ %s\n" "$PKG" "$PKG_DIR"
708 if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
710 elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
711 printf "Error, unable to update local wheel repository\n"
715 local venv_dir="/usr/lib/$PYTHON_PKG"
716 echo "Creating virtualenv..."
717 if ! "$PYTHON3_EXECUTABLE" -m venv "$venv_dir"; then
718 printf "Error, unable to run\n %s -m venv %s\n" "$PYTHON3_EXECUTABLE" "$venv_dir"
720 # We must have the dependency resolver introduced in late 2020 for the rest
721 # of our install process to work.
722 # <https://blog.python.org/2020/11/pip-20-3-release-new-resolver.html>
723 elif ! "$venv_dir/bin/pip" install "pip>=20.3"; then
724 printf "Error, unable to run\n %s/bin/pip install 'pip>=20.3'\n" "$venv_dir"
728 local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
729 if [[ -z "$pip_wheel" ]]; then
730 printf "Error, unable to find built wheel for $PKG\n"
732 elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
733 printf "Error, unable to run
734 %s/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url=file://%s %s
735 " "$venv_dir" "$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"
739 # Determine the package version from the wheel
740 PYTHON_VERSION="$("$venv_dir/bin/python" "$WORKSPACE/build/pypkg_info.py" metadata "$PKG" Version)"
741 UNFILTERED_PYTHON_VERSION="$(echo "$PYTHON_VERSION" | sed 's/\.dev/~dev/; s/\([0-9]\)rc/\1~rc/')"
743 # See if we actually need to build this package; does it exist already?
744 # We can't do this earlier than here, because we need PYTHON_VERSION.
745 if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" python3 "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
748 echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
750 # Using `env -C` sets the directory where the package is built.
751 # Using `fpm --chdir` sets the root directory for source arguments.
752 declare -a COMMAND_ARR=(
753 env -C "$PYTHON_BUILDROOT" fpm
756 --version="$UNFILTERED_PYTHON_VERSION"
758 --output-type="$package_format"
759 --depends="$PYTHON3_PACKAGE"
760 --iteration="$ARVADOS_BUILDING_ITERATION"
761 --replaces="python-$PKG"
762 --url="https://arvados.org"
764 # Append fpm flags corresponding to Python package metadata.
765 readarray -d "" -O "${#COMMAND_ARR[@]}" -t COMMAND_ARR < \
766 <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" \
767 --delimiter=\\0 --format=fpm \
768 metadata "$PKG" License Summary)
770 if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
771 COMMAND_ARR+=("-a$target_arch")
774 if [[ "$MAINTAINER" != "" ]]; then
775 COMMAND_ARR+=('--maintainer' "$MAINTAINER")
778 if [[ "$VENDOR" != "" ]]; then
779 COMMAND_ARR+=('--vendor' "$VENDOR")
782 if [[ "$DEBUG" != "0" ]]; then
783 COMMAND_ARR+=('--verbose' '--log' 'info')
786 systemd_unit="$PKG_DIR/$PKG.service"
787 if [[ -e "${systemd_unit}" ]]; then
788 COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
789 COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
792 case "$package_format" in
796 --deb-no-default-config-files
800 # Conflict with older packages we used to publish
801 --conflicts "rh-python36-python-$PKG"
802 # Do not generate /usr/lib/.build-id links on RH8+
803 # (otherwise our packages conflict with platform-python)
804 --rpm-rpmbuild-define "_build_id_links none"
808 # Append --depends X and other arguments specified by fpm-info.sh in
809 # the package source dir. These are added last so they can override
810 # the arguments added by this script.
811 declare -a fpm_args=()
812 declare -a fpm_depends=()
814 fpminfo="$PKG_DIR/fpm-info.sh"
815 if [[ -e "$fpminfo" ]]; then
816 echo "Loading fpm overrides from $fpminfo"
817 if ! source "$fpminfo"; then
818 echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
823 for i in "${fpm_depends[@]}"; do
824 COMMAND_ARR+=('--depends' "$i")
827 # make sure the systemd service file ends up in the right place
828 # used by arvados-docker-cleaner
829 if [[ -e "${systemd_unit}" ]]; then
830 COMMAND_ARR+=("share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
833 COMMAND_ARR+=("${fpm_args[@]}")
835 while read -d "" binpath; do
836 COMMAND_ARR+=("$binpath=/usr/$binpath")
837 done < <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" --delimiter=\\0 binfiles "$PKG")
839 # the python3-arvados-cwl-runner package comes with cwltool, expose that version
840 if [[ "$PKG" == arvados-cwl-runner ]]; then
841 COMMAND_ARR+=("bin/cwltool=/usr/bin/cwltool")
844 COMMAND_ARR+=(".=$venv_dir")
846 debug_echo -e "\n${COMMAND_ARR[@]}\n"
848 FPM_RESULTS=$("${COMMAND_ARR[@]}")
851 # if something went wrong and debug is off, print out the fpm command that errored
852 if ! fpm_verify $FPM_EXIT_CODE $FPM_RESULTS && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
853 echo "fpm returned an error executing the command:"
855 echo -e "\n${COMMAND_ARR[@]}\n"
857 ls "$PYTHON_BUILDROOT"/*."$package_format"
858 mv "$PYTHON_BUILDROOT"/*."$package_format" "$WORKSPACE/packages/$TARGET/"
863 # Build packages for everything
865 # Source dir where fpm-info.sh (if any) will be found.
868 # The package source. Depending on the source type, this can be a
869 # path, or the name of the package in an upstream repository (e.g.,
873 # The name of the package to build.
876 # The type of source package. Passed to fpm -s. Default "dir".
877 PACKAGE_TYPE=${1:-dir}
879 # Optional: the package version number. Passed to fpm -v.
883 if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
887 local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
889 declare -a COMMAND_ARR=("fpm" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
890 if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
891 # Dependencies are built from Python package metadata. Since that
892 # will never refer to Debian package iterations, it doesn't make sense
893 # to enforce those in the .deb dependencies.
894 COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
897 if [[ "$DEBUG" != "0" ]]; then
898 COMMAND_ARR+=('--verbose' '--log' 'info')
901 if [[ -n "$PACKAGE_NAME" ]]; then
902 COMMAND_ARR+=('-n' "$PACKAGE_NAME")
905 if [[ "$MAINTAINER" != "" ]]; then
906 COMMAND_ARR+=('--maintainer' "$MAINTAINER")
909 if [[ "$VENDOR" != "" ]]; then
910 COMMAND_ARR+=('--vendor' "$VENDOR")
913 if [[ "$VERSION" != "" ]]; then
914 COMMAND_ARR+=('-v' "$VERSION")
916 if [[ -n "$default_iteration_value" ]]; then
917 # We can always add an --iteration here. If another one is specified in $@,
918 # that will take precedence, as desired.
919 COMMAND_ARR+=(--iteration "$default_iteration_value")
922 # Append --depends X and other arguments specified by fpm-info.sh in
923 # the package source dir. These are added last so they can override
924 # the arguments added by this script.
925 declare -a fpm_args=()
926 declare -a build_depends=()
927 declare -a fpm_depends=()
928 declare -a fpm_conflicts=()
929 declare -a fpm_exclude=()
930 if [[ ! -d "$SRC_DIR" ]]; then
931 echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
934 fpminfo="${SRC_DIR}/fpm-info.sh"
935 if [[ -e "$fpminfo" ]]; then
936 debug_echo "Loading fpm overrides from $fpminfo"
939 for pkg in "${build_depends[@]}"; do
940 if [[ $TARGET =~ debian|ubuntu ]]; then
941 pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
942 if [[ -e $pkg_deb ]]; then
943 echo "Installing build_dep $pkg from $pkg_deb"
946 echo "Attemping to install build_dep $pkg using apt-get"
947 apt-get install -y "$pkg"
949 apt-get -y -f install
951 pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
952 if [[ -e $pkg_rpm ]]; then
953 echo "Installing build_dep $pkg from $pkg_rpm"
956 echo "Attemping to install build_dep $pkg"
961 for i in "${fpm_depends[@]}"; do
962 COMMAND_ARR+=('--depends' "$i")
964 for i in "${fpm_conflicts[@]}"; do
965 COMMAND_ARR+=('--conflicts' "$i")
967 for i in "${fpm_exclude[@]}"; do
968 COMMAND_ARR+=('--exclude' "$i")
971 COMMAND_ARR+=("${fpm_args[@]}")
973 # Append remaining function arguments directly to fpm's command line.
978 COMMAND_ARR+=("$PACKAGE")
980 debug_echo -e "\n${COMMAND_ARR[@]}\n"
982 FPM_RESULTS=$("${COMMAND_ARR[@]}")
984 echo "fpm: exit code $FPM_EXIT_CODE" >>$STDOUT_IF_DEBUG
985 echo "$FPM_RESULTS" >>$STDOUT_IF_DEBUG
987 fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
989 # if something went wrong and debug is off, print out the fpm command that errored
990 if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
991 echo -e "\n${COMMAND_ARR[@]}\n"
995 # verify build results
1002 if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.~-]*\.)(deb|rpm) ]]; then
1003 FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
1006 if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
1009 echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
1014 elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
1015 echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
1017 elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
1019 echo "Error building package for $1:\n $FPM_RESULTS"
1026 if [[ "$FORMAT" == "deb" ]]; then
1027 $SUDO apt-get install $PACKAGES --yes
1028 elif [[ "$FORMAT" == "rpm" ]]; then
1029 $SUDO yum -q -y install $PACKAGES
1034 printf '%s %s\n' "=======" "$1"
1038 if [[ "$1" != "0" ]]; then
1039 title "$2 -- FAILED"
1040 failures+=("$2 (`timer`)")
1042 successes+=("$2 (`timer`)")
1051 if [[ -n "$t0" ]]; then
1052 echo -n "$(($SECONDS - $t0))s"
1057 for x in "${successes[@]}"
1062 if [[ ${#failures[@]} == 0 ]]
1064 if [[ ${#successes[@]} != 0 ]]; then
1065 echo "All test suites passed."
1068 echo "Failures (${#failures[@]}):"
1069 for x in "${failures[@]}"