native_arch=$(get_native_arch)
if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
- echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
+ echo "Error: no cross compilation support for Go on $native_arch, can not build $prog for $target_arch"
return 1
fi
+ case "$package_format-$TARGET" in
+ # Older Debian/Ubuntu do not support cross compilation because the
+ # libfuse package does not support multiarch. See
+ # <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477>.
+ # Red Hat-based distributions do not support native cross compilation at
+ # all (they use a qemu-based solution we haven't implemented yet).
+ deb-debian10|deb-ubuntu1804|deb-ubuntu2004|rpm-*)
+ cross_compilation=0
+ if [[ "$native_arch" == "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
+ echo "Error: no cross compilation support for Go on $native_arch for $TARGET, can not build $prog for $target_arch"
+ return 1
+ fi
+ ;;
+ *)
+ cross_compilation=1
+ ;;
+ esac
+
if [[ -n "$target_arch" ]]; then
- # A target architecture has been specified
- package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
- return $?
+ archs=($target_arch)
else
- # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
- # but only when building deb packages (centos does not have support for crosscompiling userspace).
+ # No target architecture specified, default to native target. When on amd64
+ # also crosscompile arm64 (when supported).
archs=($native_arch)
- if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+ if [[ $cross_compilation -ne 0 ]]; then
archs+=("arm64")
fi
- for ta in ${archs[@]}; do
- package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
- retval=$?
- if [[ $retval -ne 0 ]]; then
- return $retval
- fi
- done
fi
+
+ for ta in ${archs[@]}; do
+ package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
+ retval=$?
+ if [[ $retval -ne 0 ]]; then
+ return $retval
+ fi
+ done
}
# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
local target_arch="${1:-amd64}"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then
- echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch"
- return 1
- fi
-
debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
local basename="${src_path##*/}"
calculate_go_package_version go_package_version $src_path
echo "Building $package_format ($target_arch) package for $prog from $src_path"
if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
- CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
else
- GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+ GOARCH=${arch} go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
fi
local -a switches=()
return 0
fi
else
- centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
-
- repo_pkg_list=$(curl -s -o - ${centos_repo})
- echo ${repo_pkg_list} |grep -q ${full_pkgname}
- if [ $? -eq 0 ]; then
+ local rpm_root
+ case "$TARGET" in
+ centos7) rpm_root="CentOS/7/dev" ;;
+ rocky8) rpm_root="CentOS/8/dev" ;;
+ *)
+ echo "FIXME: Don't know RPM URL path for $TARGET, building"
+ return 0
+ ;;
+ esac
+ local rpm_url="http://rpm.arvados.org/$rpm_root/$arch/$full_pkgname"
+
+ if curl -fs -o "$WORKSPACE/packages/$TARGET/$full_pkgname" "$rpm_url"; then
echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
- curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" ${centos_repo}${full_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
+ elif [[ -f "$WORKSPACE/packages/$TARGET/processed/$full_pkgname" ]]; then
echo "Package $full_pkgname exists, not rebuilding!"
return 1
else
cd "$srcdir"
mkdir -p tmp
git rev-parse HEAD >git-commit.version
- bundle package --all
+ bundle config set cache_all true
+ bundle package
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
# Build the workbench server package
test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
if [[ "$?" == "0" ]] ; then
+ calculate_go_package_version arvados_server_version cmd/arvados-server
+ arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
(
set -e
- calculate_go_package_version arvados_server_version cmd/arvados-server
- arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
-
# The workbench package has a build-time dependency on the arvados-server
# package for config manipulation, so install it first.
cd $WORKSPACE/cmd/arvados-server
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
+ bundle config set --local system 'true' >"$STDOUT_IF_DEBUG"
+ bundle install >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
rm -rf "$WORKSPACE/cwltest"
fi
git clone https://github.com/common-workflow-language/cwltest.git
+
+ # The subsequent release of cwltest confirms that files exist on disk, since
+ # our files are in Keep, all the tests fail.
+ # We should add [optional] Arvados support to cwltest so it can access
+ # Keep but for the time being just package the last working version.
+ (cd cwltest && git checkout 2.3.20230108193615)
+
# signal to our build script that we want a cwltest executable installed in /usr/bin/
mkdir cwltest/bin && touch cwltest/bin/cwltest
fpm_build_virtualenv "cwltest" "cwltest" "$package_format" "$target_arch"
)
}
-# Usage: handle_libarvados_perl
-handle_libarvados_perl () {
- if [[ -n "$ONLY_BUILD" ]] || [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then
- debug_echo -e "Skipping build of libarvados-perl package."
- return 0
- fi
- cd "$WORKSPACE/sdk/perl"
- libarvados_perl_version="$(version_from_git)"
-
- cd $WORKSPACE/packages/$TARGET
- test_package_presence libarvados-perl "$libarvados_perl_version"
-
- if [[ "$?" == "0" ]]; then
- cd "$WORKSPACE/sdk/perl"
-
- if [[ -e Makefile ]]; then
- make realclean >"$STDOUT_IF_DEBUG"
- fi
- find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
- -delete
- rm -rf install
-
- perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
- make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
- fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
- dir "$(version_from_git)" install/man/=/usr/share/man \
- "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
- mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
- fi
-}
-
# Build python packages with a virtualenv built-in
# Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64]
fpm_build_virtualenv () {
PYTHON_PKG=$pkg
fi
- # arvados-python-client sdist should always be built, to be available
- # for other dependent packages.
- if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$pkg" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
- debug_echo -e "Skipping build of $pkg package."
- return 0
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]]; then
+ # arvados-python-client sdist should always be built if we are building a
+ # python package.
+ if [[ "$ONLY_BUILD" != "python3-arvados-cwl-runner" ]] &&
+ [[ "$ONLY_BUILD" != "python3-arvados-fuse" ]] &&
+ [[ "$ONLY_BUILD" != "python3-crunchstat-summary" ]] &&
+ [[ "$ONLY_BUILD" != "arvados-docker-cleaner" ]] &&
+ [[ "$ONLY_BUILD" != "python3-arvados-user-activity" ]]; then
+ debug_echo -e "Skipping build of $pkg package."
+ return 0
+ fi
fi
if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then
ARVADOS_BUILDING_ITERATION=1
fi
- local python=python3
+ local python=$PYTHON3_EXECUTABLE
pip=pip3
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
PACKAGE_PATH=`(cd dist; ls *tar.gz)`
if [[ "arvados-python-client" == "$PKG" ]]; then
- PYSDK_PATH=`pwd`/dist/
+ PYSDK_PATH="-f $(pwd)/dist/"
fi
if [[ -n "$ONLY_BUILD" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
echo "wheel version: `build/usr/share/$python/dist/$PYTHON_PKG/bin/wheel version`"
if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
- build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
+ build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PYSDK_PATH $PACKAGE_PATH
else
# centos7 needs these special tweaks to install python-arvados-fuse
build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
- PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH
+ PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PYSDK_PATH $PACKAGE_PATH
fi
if [[ "$?" != "0" ]]; then
echo "Error, unable to run"
- echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -f $PYSDK_PATH $PACKAGE_PATH"
+ echo " build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PYSDK_PATH $PACKAGE_PATH"
exit 1
fi
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
- if [[ "$package_format" == "rpm" ]]; then
- # Make sure to conflict with the old rh-python36 packages we used to publish
- COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
- fi
-
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
fi
COMMAND_ARR+=('--depends' "$PYTHON3_PACKAGE")
-
- # avoid warning
- COMMAND_ARR+=('--deb-no-default-config-files')
+ case "$package_format" in
+ deb)
+ COMMAND_ARR+=(
+ # Avoid warning
+ --deb-no-default-config-files
+ ) ;;
+ rpm)
+ COMMAND_ARR+=(
+ # Conflict with older packages we used to publish
+ --conflicts "rh-python36-python-$PKG"
+ # Do not generate /usr/lib/.build-id links on RH8+
+ # (otherwise our packages conflict with platform-python)
+ --rpm-rpmbuild-define "_build_id_links none"
+ ) ;;
+ esac
# Append --depends X and other arguments specified by fpm-info.sh in
# the package source dir. These are added last so they can override
projects / arvados.git / blobdiff