From 964c763379e2ea98e46584267342ced694e3349a Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Mon, 3 Jan 2022 16:55:42 -0500 Subject: [PATCH] 17417: refactor, reduce use of global variables. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- build/run-build-packages.sh | 42 +++++++++++------------ build/run-library.sh | 67 +++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 47 deletions(-) diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 17455bc435..2e22a9e53b 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -276,47 +276,47 @@ fi # Go binaries cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) -package_go_binary cmd/arvados-client arvados-client \ +package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \ "Arvados command line tool (beta)" -package_go_binary cmd/arvados-server arvados-server \ +package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \ "Arvados server daemons" -package_go_binary cmd/arvados-server arvados-controller \ +package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \ "Arvados cluster controller daemon" -package_go_binary cmd/arvados-server arvados-dispatch-cloud \ +package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \ "Arvados cluster cloud dispatch" -package_go_binary cmd/arvados-server arvados-dispatch-lsf \ +package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \ "Dispatch Arvados containers to an LSF cluster" -package_go_binary services/arv-git-httpd arvados-git-httpd \ +package_go_binary services/arv-git-httpd arvados-git-httpd "$FORMAT" "$ARCH" \ "Provide authenticated http access to Arvados-hosted git repositories" -package_go_binary services/crunch-dispatch-local crunch-dispatch-local \ +package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \ "Dispatch Crunch containers on the local system" -package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \ +package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \ "Dispatch Crunch containers to a SLURM cluster" -package_go_binary cmd/arvados-server crunch-run \ +package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \ "Supervise a single Crunch container" -package_go_binary services/crunchstat crunchstat \ +package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \ "Gather cpu/memory/network statistics of running Crunch jobs" -package_go_binary services/health arvados-health \ +package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \ "Check health of all Arvados cluster services" -package_go_binary services/keep-balance keep-balance \ +package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" -package_go_binary services/keepproxy keepproxy \ +package_go_binary services/keepproxy keepproxy "$FORMAT" "$ARCH" \ "Make a Keep cluster accessible to clients that are not on the LAN" -package_go_binary cmd/arvados-server keepstore \ +package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \ "Keep storage daemon, accessible to clients on the LAN" -package_go_binary services/keep-web keep-web \ +package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \ "Static web hosting service for user data stored in Arvados Keep" -package_go_binary cmd/arvados-server arvados-ws \ +package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \ "Arvados Websocket server" -package_go_binary tools/sync-groups arvados-sync-groups \ +package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \ "Synchronize remote groups into Arvados from an external source" -package_go_binary tools/keep-block-check keep-block-check \ +package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \ "Verify that all data from one set of Keep servers to another was copied" -package_go_binary tools/keep-rsync keep-rsync \ +package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \ "Copy all data from one set of Keep servers to another" -package_go_binary tools/keep-exercise keep-exercise \ +package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \ "Performance testing tool for Arvados Keep" -package_go_so lib/pam pam_arvados.so libpam-arvados-go \ +package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \ "Arvados PAM authentication module" # The Python SDK - Python3 package diff --git a/build/run-library.sh b/build/run-library.sh index eb7f086278..92140bbdc8 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -130,10 +130,12 @@ calculate_go_package_version() { __returnvar="$version" } -# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt] +# Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt] package_go_binary() { local src_path="$1"; shift local prog="$1"; shift + local package_format="$1"; shift + local target_arch="$1"; shift local description="$1"; shift local license_file="${1:-agpl-3.0.txt}"; shift @@ -145,55 +147,66 @@ package_go_binary() { fi fi + # Only amd64 and aarch64 are supported native_arch="amd64" if [[ "$HOSTTYPE" == "aarch64" ]]; then native_arch="arm64" fi - if [[ -n "$ARCH" ]]; then - if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ARCH" ]]; then - package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ARCH" "$license_file" - else - echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ARCH" - fi + if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then + echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch" + return 1 + fi + + if [[ -n "$target_arch" ]]; then + # A target architecture has been specified + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file" + return $? else + # No target architecture specified, default to native target. When on amd64 also crosscompile arm64 + # but only when building deb packages (centos does not have support for crosscompiling userspace). archs=($native_arch) - if [[ "$native_arch" == "amd64" ]]; then + if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then archs=('amd64' 'arm64') fi - for arch in $archs; do - package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file" + for ta in $archs; do + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file" + retval=$? + if [[ "$retval" != 0 ]]; then + return $retval + fi done fi } -# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt] +# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt] package_go_binary_worker() { local src_path="$1"; shift local prog="$1"; shift + local package_format="$1"; shift local description="$1"; shift local native_arch="${1:-amd64}"; shift - local arch="${1:-amd64}"; shift + local target_arch="${1:-amd64}"; shift local license_file="${1:-agpl-3.0.txt}"; shift - debug_echo "package_go_binary $src_path as $prog" + if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then + echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch" + return 1 + fi + + debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)" local basename="${src_path##*/}" calculate_go_package_version go_package_version $src_path cd $WORKSPACE/packages/$TARGET - test_package_presence "$prog" "$go_package_version" "go" "" "$arch" + test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch" if [[ "$?" != "0" ]]; then return 1 fi - echo "BUILDING ${arch}" - if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then - if [[ "$FORMAT" == "deb" ]]; then - CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" - else - echo "Error: no cross compilation support for Go on $native_arch ($FORMAT), can not build $prog for $ARCH" - return - fi + echo "Building $pachage_format ($target_arch) package for $prog from $src_path" + if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then + CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" else GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" fi @@ -201,9 +214,9 @@ package_go_binary_worker() { local -a switches=() binpath=$GOPATH/bin/${basename} - if [[ "${arch}" != "${native_arch}" ]]; then - switches+=("-a${arch}") - binpath="$GOPATH/bin/linux_${arch}/${basename}" + if [[ "${target_arch}" != "${native_arch}" ]]; then + switches+=("-a${target_arch}") + binpath="$GOPATH/bin/linux_${target_arch}/${basename}" fi systemd_unit="$WORKSPACE/${src_path}/${prog}.service" @@ -218,11 +231,13 @@ package_go_binary_worker() { fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}" } -# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library" +# Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library" package_go_so() { local src_path="$1"; shift local sofile="$1"; shift local pkg="$1"; shift + local package_format="$1"; shift + local target_arch="$1"; shift # supported: amd64, arm64 local description="$1"; shift if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then -- 2.30.2