2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 # A library of functions shared by the various scripts in this directory.
8 # This is the timestamp about when we merged changed to include licenses
9 # with Arvados packages. We use it as a heuristic to add revisions for
11 LICENSE_PACKAGE_TS=20151208015500
13 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
14 RAILS_PACKAGE_ITERATION=1
16 RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
20 echo "$@" >"$STDOUT_IF_DEBUG"
23 find_python_program() {
27 if "$prog" --version >/dev/null 2>&1; then
35 Error: $prog (from Python setuptools module) not found
41 format_last_commit_here() {
42 local format="$1"; shift
43 local dir="${1:-.}"; shift
44 TZ=UTC git log -n1 --first-parent "--format=format:$format" "$dir"
48 # Output the version being built, or if we're building a
49 # dev/prerelease, output a version number based on the git log for
51 local subdir="$1"; shift
52 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
53 echo "$ARVADOS_BUILDING_VERSION"
58 declare $(format_last_commit_here "git_ts=%ct git_hash=%h" "$subdir")
59 ARVADOS_BUILDING_VERSION="$($WORKSPACE/build/version-at-commit.sh $git_hash)"
60 echo "$ARVADOS_BUILDING_VERSION"
63 nohash_version_from_git() {
64 local subdir="$1"; shift
65 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
66 echo "$ARVADOS_BUILDING_VERSION"
69 version_from_git $subdir | cut -d. -f1-4
72 timestamp_from_git() {
73 local subdir="$1"; shift
74 format_last_commit_here "%ct" "$subdir"
77 calculate_python_sdk_cwl_package_versions() {
78 python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
79 cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
82 # Usage: get_native_arch
84 # Only amd64 and aarch64 are supported at the moment
94 echo "Error: architecture not supported"
102 local gem_name="$1"; shift
103 local gem_version="$(nohash_version_from_git)"
104 local gem_src_dir="$(pwd)"
106 if [[ -n "$ONLY_BUILD" ]] && [[ "$gem_name" != "$ONLY_BUILD" ]] ; then
110 if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
111 find -maxdepth 1 -name "${gem_name}-*.gem" -delete
113 # -q appears to be broken in gem version 2.2.2
114 $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
118 # Usage: package_workbench2
119 package_workbench2() {
120 local pkgname=arvados-workbench2
121 local src=services/workbench2
122 local dst=/var/www/arvados-workbench2/workbench2
123 local description="Arvados Workbench 2"
125 local version="$(version_from_git)"
127 NODE_ENV=production yarn install
128 VERSION="$version" BUILD_NUMBER="$(default_iteration "$pkgname" "$version" yarn)" GIT_COMMIT="$(git rev-parse HEAD | head -c9)" yarn build
129 cd "$WORKSPACE/packages/$TARGET"
130 fpm_build "${WORKSPACE}/$src" "${WORKSPACE}/$src/build/=$dst" "$pkgname" dir "$version" \
131 --license="GNU Affero General Public License, version 3.0" \
132 --description="${description}" \
133 --config-files="/etc/arvados/$pkgname/workbench2.example.json" \
134 "$WORKSPACE/services/workbench2/etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$pkgname/workbench2.example.json"
137 calculate_go_package_version() {
138 # $__returnvar has the nameref attribute set, which means it is a reference
139 # to another variable that is passed in as the first argument to this function.
140 # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
141 local -n __returnvar="$1"; shift
147 # Update the version number and build a new package if the vendor
148 # bundle has changed, or the command imports anything from the
149 # Arvados SDK and the SDK has changed.
150 declare -a checkdirs=(go.mod go.sum)
151 while [ -n "$1" ]; do
155 # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
156 # Everything depends on the build directory.
157 checkdirs+=(sdk/go lib build)
159 for dir in ${checkdirs[@]}; do
161 ts="$(timestamp_from_git "$dir")"
162 if [[ "$ts" -gt "$timestamp" ]]; then
163 version=$(version_from_git "$dir")
168 __returnvar="$version"
171 # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
172 package_go_binary() {
173 local src_path="$1"; shift
174 local prog="$1"; shift
175 local package_format="$1"; shift
176 local target_arch="$1"; shift
177 local description="$1"; shift
178 local license_file="${1:-agpl-3.0.txt}"; shift
180 if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
181 debug_echo -e "Skipping build of $prog package."
185 native_arch=$(get_native_arch)
187 if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
188 echo "Error: no cross compilation support for Go on $native_arch, can not build $prog for $target_arch"
192 case "$package_format-$TARGET" in
193 # Ubuntu 20.04 does not support cross compilation because the
194 # libfuse package does not support multiarch. See
195 # <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477>.
196 # Red Hat-based distributions do not support native cross compilation at
197 # all (they use a qemu-based solution we haven't implemented yet).
198 deb-ubuntu2004|rpm-*)
200 if [[ "$native_arch" == "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
201 echo "Error: no cross compilation support for Go on $native_arch for $TARGET, can not build $prog for $target_arch"
210 if [[ -n "$target_arch" ]]; then
213 # No target architecture specified, default to native target. When on amd64
214 # also crosscompile arm64 (when supported).
216 if [[ $cross_compilation -ne 0 ]]; then
221 for ta in ${archs[@]}; do
222 package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
224 if [[ $retval -ne 0 ]]; then
230 # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
231 package_go_binary_worker() {
232 local src_path="$1"; shift
233 local prog="$1"; shift
234 local package_format="$1"; shift
235 local description="$1"; shift
236 local native_arch="${1:-amd64}"; shift
237 local target_arch="${1:-amd64}"; shift
238 local license_file="${1:-agpl-3.0.txt}"; shift
240 debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
241 local basename="${src_path##*/}"
242 calculate_go_package_version go_package_version $src_path
244 cd $WORKSPACE/packages/$TARGET
245 test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
246 if [[ $? -ne 0 ]]; then
250 echo "Building $package_format ($target_arch) package for $prog from $src_path"
251 if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
252 CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
254 GOARCH=${arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
259 binpath=$GOPATH/bin/${basename}
260 if [[ "${target_arch}" != "${native_arch}" ]]; then
261 switches+=("-a${target_arch}")
262 binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
265 case "$package_format" in
266 # As of April 2024 we package identical Go binaries under different
267 # packages and names. This upsets the build id database, so don't
268 # register ourselves there.
269 rpm) switches+=(--rpm-rpmbuild-define="_build_id_links none") ;;
272 systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
273 if [[ -e "${systemd_unit}" ]]; then
275 --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
276 --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
277 "${systemd_unit}=/lib/systemd/system/${prog}.service")
279 switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
281 fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
284 # Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
286 local src_path="$1"; shift
287 local sofile="$1"; shift
288 local pkg="$1"; shift
289 local package_format="$1"; shift
290 local target_arch="$1"; shift # supported: amd64, arm64
291 local description="$1"; shift
293 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
294 debug_echo -e "Skipping build of $pkg package."
298 debug_echo "package_go_so $src_path as $pkg"
300 calculate_go_package_version go_package_version $src_path
301 cd $WORKSPACE/packages/$TARGET
302 test_package_presence $pkg $go_package_version go || return 1
303 cd $WORKSPACE/$src_path
304 go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
305 cd $WORKSPACE/packages/$TARGET
307 "--url=https://arvados.org"
308 "--license=Apache License, Version 2.0"
309 "--description=${description}"
310 "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
312 if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
313 fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/doc/$pkg/pam-configs-arvados-go")
315 if [[ -e "$WORKSPACE/$src_path/README" ]]; then
316 fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
318 fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
321 default_iteration() {
322 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
323 echo "$ARVADOS_BUILDING_ITERATION"
326 local package_name="$1"; shift
327 local package_version="$1"; shift
328 local package_type="$1"; shift
330 if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
331 [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
337 _build_rails_package_scripts() {
338 local pkgname="$1"; shift
339 local destdir="$1"; shift
340 local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
341 for scriptname in postinst prerm postrm; do
342 cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
343 >"$destdir/$scriptname" || return $?
347 rails_package_version() {
348 local pkgname="$1"; shift
349 local srcdir="$1"; shift
350 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
351 echo "$ARVADOS_BUILDING_VERSION"
354 local version="$(version_from_git)"
355 if [ $pkgname = "arvados-api-server" ] ; then
356 calculate_go_package_version version cmd/arvados-server "$srcdir"
361 test_rails_package_presence() {
362 local pkgname="$1"; shift
363 local srcdir="$1"; shift
365 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
373 local version="$(rails_package_version "$pkgname" "$srcdir")"
377 test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
380 get_complete_package_name() {
381 # if the errexit flag is set, unset it until this function returns
382 # otherwise, the shift calls below will abort the program if optional arguments are not supplied
383 if [ -o errexit ]; then
387 # $__returnvar has the nameref attribute set, which means it is a reference
388 # to another variable that is passed in as the first argument to this function.
389 # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
390 local -n __returnvar="$1"; shift
391 local pkgname="$1"; shift
392 local version="$1"; shift
393 local pkgtype="$1"; shift
394 local iteration="$1"; shift
395 local arch="$1"; shift
396 if [[ "$iteration" == "" ]]; then
397 iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
400 if [[ "$arch" == "" ]]; then
401 native_arch=$(get_native_arch)
402 rpm_native_arch="x86_64"
403 if [[ "$HOSTTYPE" == "aarch64" ]]; then
404 rpm_native_arch="arm64"
406 rpm_architecture="$rpm_native_arch"
407 deb_architecture="$native_arch"
409 if [[ "$pkgtype" =~ ^(src)$ ]]; then
410 rpm_architecture="noarch"
411 deb_architecture="all"
414 rpm_architecture=$arch
415 deb_architecture=$arch
418 local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
419 if [[ "$FORMAT" == "rpm" ]]; then
420 # rpm packages get iteration 1 if we don't supply one
421 iteration=${iteration:-1}
422 complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
424 __returnvar=${complete_pkgname}
427 # Test if the package already exists, if not return 0, if it does return 1
428 test_package_presence() {
429 local pkgname="$1"; shift
430 local version="$1"; shift
431 local pkgtype="$1"; shift
432 local iteration="$1"; shift
433 local arch="$1"; shift
434 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
439 get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
441 # See if we can skip building the package, only if it already exists in the
442 # processed/ directory. If so, move it back to the packages directory to make
443 # sure it gets picked up by the test and/or upload steps.
444 # Get the list of packages from the repos
446 if [[ "$FORCE_BUILD" == "1" ]]; then
447 echo "Package $full_pkgname build forced with --force-build, building"
448 elif [[ "$FORMAT" == "deb" ]]; then
450 dd[debian11]=bullseye
451 dd[debian12]=bookworm
455 if [ ${pkgname:0:3} = "lib" ]; then
456 repo_subdir=${pkgname:0:4}
458 repo_subdir=${pkgname:0:1}
461 repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
462 echo "${repo_pkg_list}" |grep -q ${full_pkgname}
463 if [ $? -eq 0 ] ; then
464 echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
465 curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
467 elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
468 echo "Package $full_pkgname exists, not rebuilding!"
471 echo "Package $full_pkgname not found, building"
477 rocky8) rpm_root="CentOS/8/dev" ;;
479 echo "FIXME: Don't know RPM URL path for $TARGET, building"
483 local rpm_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
485 if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$rpm_url"; then
486 echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
488 elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
489 echo "Package $full_pkgname exists, not rebuilding!"
492 echo "Package $full_pkgname not found, building"
498 handle_rails_package() {
499 local pkgname="$1"; shift
501 if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
504 local srcdir="$1"; shift
506 local license_path="$1"; shift
507 local version="$(rails_package_version "$pkgname" "$srcdir")"
508 echo "$version" >package-build.version
509 local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
512 _build_rails_package_scripts "$pkgname" "$scripts_dir"
515 git rev-parse HEAD >git-commit.version
516 bundle config set cache_all true
519 if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
520 echo "ERROR: $pkgname package prep failed" >&2
521 rm -rf "$scripts_dir"
525 local railsdir="/var/www/${pkgname%-server}/current"
526 local -a pos_args=("$srcdir/=$railsdir" "$pkgname" dir "$version")
527 local license_arg="$license_path=$railsdir/$(basename "$license_path")"
528 local -a switches=(--after-install "$scripts_dir/postinst"
529 --before-remove "$scripts_dir/prerm"
530 --after-remove "$scripts_dir/postrm")
531 if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
532 switches+=(--iteration $RAILS_PACKAGE_ITERATION)
534 # For some reason fpm excludes need to not start with /.
535 local exclude_root="${railsdir#/}"
536 for exclude in tmp log coverage Capfile\* \
538 config/application.yml \
539 config/database.yml; do
540 switches+=(-x "$exclude_root/$exclude")
542 fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
543 -x "$exclude_root/vendor/cache-*" \
544 -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
545 rm -rf "$scripts_dir"
548 # Usage: handle_api_server [amd64|arm64]
549 handle_api_server () {
550 local target_arch="${1:-amd64}"; shift
552 if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then
553 debug_echo -e "Skipping build of arvados-api-server package."
557 native_arch=$(get_native_arch)
558 if [[ "$target_arch" != "$native_arch" ]]; then
559 echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH"
564 # Build the API server package
565 test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
566 if [[ "$?" == "0" ]]; then
567 calculate_go_package_version arvados_server_version cmd/arvados-server
568 arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
569 handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
570 "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
571 --description="Arvados API server - Arvados is a free and open source platform for big data science." \
572 --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
576 # Usage: handle_arvados_src
577 handle_arvados_src () {
578 if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then
579 debug_echo -e "Skipping build of arvados-src package."
585 COMMIT_HASH=$(format_last_commit_here "%H")
586 arvados_src_version="$(version_from_git)"
588 cd $WORKSPACE/packages/$TARGET
589 test_package_presence arvados-src "$arvados_src_version" src ""
591 if [[ "$?" == "0" ]]; then
593 SRC_BUILD_DIR=$(mktemp -d)
594 # mktemp creates the directory with 0700 permissions by default
595 chmod 755 $SRC_BUILD_DIR
596 git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
599 # go into detached-head state
600 git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
601 echo "$COMMIT_HASH" >git-commit.version
603 cd $WORKSPACE/packages/$TARGET
604 fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
606 rm -rf "$SRC_BUILD_DIR"
611 setup_build_virtualenv() {
612 PYTHON_BUILDROOT="$(mktemp --directory --tmpdir pybuild.XXXXXXXX)"
613 "$PYTHON3_EXECUTABLE" -m venv "$PYTHON_BUILDROOT/venv"
614 "$PYTHON_BUILDROOT/venv/bin/pip" install --upgrade build piprepo setuptools wheel
615 mkdir "$PYTHON_BUILDROOT/wheelhouse"
618 # Build python packages with a virtualenv built-in
619 # Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64]
620 fpm_build_virtualenv () {
622 local pkg_dir=$1; shift
623 local package_format="$1"; shift
624 local target_arch="${1:-amd64}"; shift
626 native_arch=$(get_native_arch)
627 if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then
628 fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$target_arch"
629 elif [[ -z "$target_arch" ]]; then
630 fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$native_arch"
632 echo "Error: no cross compilation support for Python yet, can not build $pkg for $target_arch"
637 # Build python packages with a virtualenv built-in
638 # Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64|arm64] [amd64|arm64]
639 fpm_build_virtualenv_worker () {
642 local package_format="$1"; shift
643 local native_arch="${1:-amd64}"; shift
644 local target_arch=${1:-amd64}; shift
647 STDOUT_IF_DEBUG=/dev/null
648 STDERR_IF_DEBUG=/dev/null
649 DASHQ_UNLESS_DEBUG=-q
650 if [[ "$DEBUG" != "0" ]]; then
651 STDOUT_IF_DEBUG=/dev/stdout
652 STDERR_IF_DEBUG=/dev/stderr
655 if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
656 ARVADOS_BUILDING_ITERATION=1
659 PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
660 if [[ "$PKG" != "arvados-docker-cleaner" ]]; then
661 PYTHON_PKG=$PACKAGE_PREFIX-$PKG
663 # Exception to our package naming convention
667 # We must always add a wheel to our repository, even if we're not building
668 # this distro package, because it might be a dependency for a later
669 # package we do build.
670 if [[ "$PKG_DIR" =~ ^.=[0-9]+\. ]]; then
671 # Not source to build, but a version to download.
672 # The rest of the function expects a filesystem path, so set one afterwards.
673 "$PYTHON_BUILDROOT/venv/bin/pip" download --dest="$PYTHON_BUILDROOT/wheelhouse" "$PKG$PKG_DIR" \
674 && PKG_DIR="$PYTHON_BUILDROOT/nonexistent"
676 # Make PKG_DIR absolute.
677 PKG_DIR="$(env -C "$WORKSPACE" readlink -e "$PKG_DIR")"
678 if [[ -e "$PKG_DIR/pyproject.toml" ]]; then
679 "$PYTHON_BUILDROOT/venv/bin/python" -m build --outdir="$PYTHON_BUILDROOT/wheelhouse" "$PKG_DIR"
681 env -C "$PKG_DIR" "$PYTHON_BUILDROOT/venv/bin/python" setup.py bdist_wheel --dist-dir="$PYTHON_BUILDROOT/wheelhouse"
684 if [[ $? -ne 0 ]]; then
685 printf "Error, unable to download/build wheel for %s @ %s" "$PKG" "$PKG_DIR"
687 elif ! "$PYTHON_BUILDROOT/venv/bin/piprepo" build "$PYTHON_BUILDROOT/wheelhouse"; then
688 printf "Error, unable to update local wheel repository"
692 if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
696 local venv_dir="$PYTHON_BUILDROOT/$PYTHON_PKG"
697 echo "Creating virtualenv..."
698 if ! "$PYTHON3_EXECUTABLE" -m venv "$venv_dir"; then
699 printf "Error, unable to run\n %s -m venv %s\n" "$PYTHON3_EXECUTABLE" "$venv_dir"
701 # We must have the dependency resolver introduced in late 2020 for the rest
702 # of our install process to work.
703 # <https://blog.python.org/2020/11/pip-20-3-release-new-resolver.html>
704 elif ! "$venv_dir/bin/pip" install "pip>=20.3"; then
705 printf "Error, unable to run\n %s/bin/pip install 'pip>=20.3'\n" "$venv_dir"
709 local pip_wheel="$(ls --sort=time --reverse "$PYTHON_BUILDROOT/wheelhouse/$(echo "$PKG" | sed s/-/_/g)-"*.whl | tail -n1)"
710 if [[ -z "$pip_wheel" ]]; then
711 printf "Error, unable to find built wheel for $PKG"
713 elif ! "$venv_dir/bin/pip" install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url="file://$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"; then
714 printf "Error, unable to run
715 %s/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG --extra-index-url=file://%s %s
716 " "$venv_dir" "$PYTHON_BUILDROOT/wheelhouse/simple" "$pip_wheel"
720 # Determine the package version from the wheel
721 PYTHON_VERSION="$("$venv_dir/bin/python" "$WORKSPACE/build/pypkg_info.py" metadata "$PKG" Version)"
722 UNFILTERED_PYTHON_VERSION="$(echo "$PYTHON_VERSION" | sed 's/\.dev/~dev/; s/\([0-9]\)rc/\1~rc/')"
724 # See if we actually need to build this package; does it exist already?
725 # We can't do this earlier than here, because we need PYTHON_VERSION.
726 if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" python3 "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then
729 echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR"
731 # Replace the shebang lines in all python scripts, and handle the activate
732 # scripts too. This is a functional replacement of the 237 line
733 # virtualenv_tools.py script that doesn't work in python3 without serious
734 # patching, minus the parts we don't need (modifying pyc files, etc).
735 local sys_venv_dir="/usr/lib/$PYTHON_PKG"
736 local sys_venv_py="$sys_venv_dir/bin/python$PYTHON3_VERSION"
737 find "$venv_dir/bin" -type f | while read binfile; do
738 if file --mime "$binfile" | grep -q binary; then
739 : # Nothing to do for binary files
740 elif [[ "$binfile" =~ /activate(.csh|.fish|)$ ]]; then
741 sed -ri "s@VIRTUAL_ENV(=| )\".*\"@VIRTUAL_ENV\\1\"$sys_venv_dir\"@" "$binfile"
743 # Replace shebang line
744 sed -ri "1 s@^#\![^[:space:]]+/bin/python[0-9.]*@#\!$sys_venv_py@" "$binfile"
748 # Using `env -C` sets the directory where the package is built.
749 # Using `fpm --chdir` sets the root directory for source arguments.
750 declare -a COMMAND_ARR=(
751 env -C "$PYTHON_BUILDROOT" fpm
754 --version="$UNFILTERED_PYTHON_VERSION"
756 --output-type="$package_format"
757 --depends="$PYTHON3_PACKAGE"
758 --iteration="$ARVADOS_BUILDING_ITERATION"
759 --replaces="python-$PKG"
760 --url="https://arvados.org"
762 # Append fpm flags corresponding to Python package metadata.
763 readarray -d "" -O "${#COMMAND_ARR[@]}" -t COMMAND_ARR < \
764 <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" \
765 --delimiter=\\0 --format=fpm \
766 metadata "$PKG" License Summary)
768 if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then
769 COMMAND_ARR+=("-a$target_arch")
772 if [[ "$MAINTAINER" != "" ]]; then
773 COMMAND_ARR+=('--maintainer' "$MAINTAINER")
776 if [[ "$VENDOR" != "" ]]; then
777 COMMAND_ARR+=('--vendor' "$VENDOR")
780 if [[ "$DEBUG" != "0" ]]; then
781 COMMAND_ARR+=('--verbose' '--log' 'info')
784 systemd_unit="$PKG_DIR/$PKG.service"
785 if [[ -e "${systemd_unit}" ]]; then
786 COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
787 COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
790 case "$package_format" in
794 --deb-no-default-config-files
798 # Conflict with older packages we used to publish
799 --conflicts "rh-python36-python-$PKG"
800 # Do not generate /usr/lib/.build-id links on RH8+
801 # (otherwise our packages conflict with platform-python)
802 --rpm-rpmbuild-define "_build_id_links none"
806 # Append --depends X and other arguments specified by fpm-info.sh in
807 # the package source dir. These are added last so they can override
808 # the arguments added by this script.
809 declare -a fpm_args=()
810 declare -a fpm_depends=()
812 fpminfo="$PKG_DIR/fpm-info.sh"
813 if [[ -e "$fpminfo" ]]; then
814 echo "Loading fpm overrides from $fpminfo"
815 if ! source "$fpminfo"; then
816 echo "Error, unable to source $WORKSPACE/$PKG_DIR/fpm-info.sh for $PKG"
821 for i in "${fpm_depends[@]}"; do
822 COMMAND_ARR+=('--depends' "$i")
825 # make sure the systemd service file ends up in the right place
826 # used by arvados-docker-cleaner
827 if [[ -e "${systemd_unit}" ]]; then
828 COMMAND_ARR+=("share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
831 COMMAND_ARR+=("${fpm_args[@]}")
833 while read -d "" binpath; do
834 COMMAND_ARR+=("$binpath=/usr/$binpath")
835 done < <("$venv_dir/bin/python3" "$WORKSPACE/build/pypkg_info.py" --delimiter=\\0 binfiles "$PKG")
837 # the python3-arvados-cwl-runner package comes with cwltool, expose that version
838 if [[ "$PKG" == arvados-cwl-runner ]]; then
839 COMMAND_ARR+=("bin/cwltool=/usr/bin/cwltool")
842 COMMAND_ARR+=(".=$sys_venv_dir")
844 debug_echo -e "\n${COMMAND_ARR[@]}\n"
846 FPM_RESULTS=$("${COMMAND_ARR[@]}")
849 # if something went wrong and debug is off, print out the fpm command that errored
850 if ! fpm_verify $FPM_EXIT_CODE $FPM_RESULTS && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
851 echo "fpm returned an error executing the command:"
853 echo -e "\n${COMMAND_ARR[@]}\n"
855 ls "$PYTHON_BUILDROOT"/*."$package_format"
856 mv "$PYTHON_BUILDROOT"/*."$package_format" "$WORKSPACE/packages/$TARGET/"
861 # Build packages for everything
863 # Source dir where fpm-info.sh (if any) will be found.
866 # The package source. Depending on the source type, this can be a
867 # path, or the name of the package in an upstream repository (e.g.,
871 # The name of the package to build.
874 # The type of source package. Passed to fpm -s. Default "dir".
875 PACKAGE_TYPE=${1:-dir}
877 # Optional: the package version number. Passed to fpm -v.
881 if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
885 local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
887 declare -a COMMAND_ARR=("fpm" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
888 if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
889 # Dependencies are built from setup.py. Since setup.py will never
890 # refer to Debian package iterations, it doesn't make sense to
891 # enforce those in the .deb dependencies.
892 COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
895 if [[ "$DEBUG" != "0" ]]; then
896 COMMAND_ARR+=('--verbose' '--log' 'info')
899 if [[ -n "$PACKAGE_NAME" ]]; then
900 COMMAND_ARR+=('-n' "$PACKAGE_NAME")
903 if [[ "$MAINTAINER" != "" ]]; then
904 COMMAND_ARR+=('--maintainer' "$MAINTAINER")
907 if [[ "$VENDOR" != "" ]]; then
908 COMMAND_ARR+=('--vendor' "$VENDOR")
911 if [[ "$VERSION" != "" ]]; then
912 COMMAND_ARR+=('-v' "$VERSION")
914 if [[ -n "$default_iteration_value" ]]; then
915 # We can always add an --iteration here. If another one is specified in $@,
916 # that will take precedence, as desired.
917 COMMAND_ARR+=(--iteration "$default_iteration_value")
920 # Append --depends X and other arguments specified by fpm-info.sh in
921 # the package source dir. These are added last so they can override
922 # the arguments added by this script.
923 declare -a fpm_args=()
924 declare -a build_depends=()
925 declare -a fpm_depends=()
926 declare -a fpm_conflicts=()
927 declare -a fpm_exclude=()
928 if [[ ! -d "$SRC_DIR" ]]; then
929 echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
932 fpminfo="${SRC_DIR}/fpm-info.sh"
933 if [[ -e "$fpminfo" ]]; then
934 debug_echo "Loading fpm overrides from $fpminfo"
937 for pkg in "${build_depends[@]}"; do
938 if [[ $TARGET =~ debian|ubuntu ]]; then
939 pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
940 if [[ -e $pkg_deb ]]; then
941 echo "Installing build_dep $pkg from $pkg_deb"
944 echo "Attemping to install build_dep $pkg using apt-get"
945 apt-get install -y "$pkg"
947 apt-get -y -f install
949 pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
950 if [[ -e $pkg_rpm ]]; then
951 echo "Installing build_dep $pkg from $pkg_rpm"
954 echo "Attemping to install build_dep $pkg"
959 for i in "${fpm_depends[@]}"; do
960 COMMAND_ARR+=('--depends' "$i")
962 for i in "${fpm_conflicts[@]}"; do
963 COMMAND_ARR+=('--conflicts' "$i")
965 for i in "${fpm_exclude[@]}"; do
966 COMMAND_ARR+=('--exclude' "$i")
969 COMMAND_ARR+=("${fpm_args[@]}")
971 # Append remaining function arguments directly to fpm's command line.
976 COMMAND_ARR+=("$PACKAGE")
978 debug_echo -e "\n${COMMAND_ARR[@]}\n"
980 FPM_RESULTS=$("${COMMAND_ARR[@]}")
982 echo "fpm: exit code $FPM_EXIT_CODE" >>$STDOUT_IF_DEBUG
983 echo "$FPM_RESULTS" >>$STDOUT_IF_DEBUG
985 fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
987 # if something went wrong and debug is off, print out the fpm command that errored
988 if [[ 0 -ne $? ]] && [[ "$STDOUT_IF_DEBUG" == "/dev/null" ]]; then
989 echo -e "\n${COMMAND_ARR[@]}\n"
993 # verify build results
1000 if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.~-]*\.)(deb|rpm) ]]; then
1001 FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
1004 if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
1007 echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
1012 elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
1013 echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
1015 elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
1017 echo "Error building package for $1:\n $FPM_RESULTS"
1024 if [[ "$FORMAT" == "deb" ]]; then
1025 $SUDO apt-get install $PACKAGES --yes
1026 elif [[ "$FORMAT" == "rpm" ]]; then
1027 $SUDO yum -q -y install $PACKAGES
1032 printf '%s %s\n' "=======" "$1"
1036 if [[ "$1" != "0" ]]; then
1037 title "$2 -- FAILED"
1038 failures+=("$2 (`timer`)")
1040 successes+=("$2 (`timer`)")
1049 if [[ -n "$t0" ]]; then
1050 echo -n "$(($SECONDS - $t0))s"
1055 for x in "${successes[@]}"
1060 if [[ ${#failures[@]} == 0 ]]
1062 if [[ ${#successes[@]} != 0 ]]; then
1063 echo "All test suites passed."
1066 echo "Failures (${#failures[@]}):"
1067 for x in "${failures[@]}"