From 1154089b34acd6e4d1426132456e1cf6bde9a241 Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Tue, 4 Jan 2022 13:55:18 -0500 Subject: [PATCH] 17417: more refactoring: reduce use of global variables, move more code into wrapper functions in run_library. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- .gitignore | 1 + build/run-build-packages.sh | 170 ++-------------- build/run-library.sh | 396 +++++++++++++++++++++++++++--------- 3 files changed, 324 insertions(+), 243 deletions(-) diff --git a/.gitignore b/.gitignore index 231424accd..07482bde73 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ _version.py *.log arvados-snakeoil-ca.pem .vagrant +packages diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 2e22a9e53b..7aa9f9bd2f 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -200,31 +200,7 @@ fi # Perl packages debug_echo -e "\nPerl packages\n" -if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then - cd "$WORKSPACE/sdk/perl" - libarvados_perl_version="$(version_from_git)" - - cd $WORKSPACE/packages/$TARGET - test_package_presence libarvados-perl "$libarvados_perl_version" - - if [[ "$?" == "0" ]]; then - cd "$WORKSPACE/sdk/perl" - - if [[ -e Makefile ]]; then - make realclean >"$STDOUT_IF_DEBUG" - fi - find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ - -delete - rm -rf install - - perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ - make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ - fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \ - dir "$(version_from_git)" install/man/=/usr/share/man \ - "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \ - mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" - fi -fi +handle_libarvados_perl # Ruby gems debug_echo -e "\nRuby gems\n" @@ -240,38 +216,11 @@ handle_ruby_gem arvados-cli cd "$WORKSPACE/services/login-sync" handle_ruby_gem arvados-login-sync -# Python packages -debug_echo -e "\nPython packages\n" +# arvados-src +handle_arvados_src -if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-src" == "$ONLY_BUILD" ]] ; then - # arvados-src - ( - cd "$WORKSPACE" - COMMIT_HASH=$(format_last_commit_here "%H") - arvados_src_version="$(version_from_git)" - - cd $WORKSPACE/packages/$TARGET - test_package_presence arvados-src "$arvados_src_version" src "" - - if [[ "$?" == "0" ]]; then - cd "$WORKSPACE" - SRC_BUILD_DIR=$(mktemp -d) - # mktemp creates the directory with 0700 permissions by default - chmod 755 $SRC_BUILD_DIR - git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" - cd "$SRC_BUILD_DIR" - - # go into detached-head state - git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" - echo "$COMMIT_HASH" >git-commit.version - - cd $WORKSPACE/packages/$TARGET - fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" - - rm -rf "$SRC_BUILD_DIR" - fi - ) -fi +# Go packages +debug_echo -e "\nGo packages\n" # Go binaries cd $WORKSPACE/packages/$TARGET @@ -319,23 +268,26 @@ package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \ package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \ "Arvados PAM authentication module" +# Python packages +debug_echo -e "\nPython packages\n" + # The Python SDK - Python3 package -fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" +fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH" # Arvados cwl runner - Python3 package -fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3" +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH" # The FUSE driver - Python3 package -fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3" +fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH" # The Arvados crunchstat-summary tool -fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3" +fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH" # The Docker image cleaner -fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" +fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH" # The Arvados user activity tool -fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3" +fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH" # The python->python3 metapackages build_metapackage "arvados-fuse" "services/fuse" @@ -345,94 +297,16 @@ build_metapackage "crunchstat-summary" "tools/crunchstat-summary" build_metapackage "arvados-docker-cleaner" "services/dockercleaner" build_metapackage "arvados-user-activity" "tools/user-activity" -if [[ -z "$ONLY_BUILD" ]] || [[ "cwltest" == "$ONLY_BUILD" ]] ; then - # The cwltest package, which lives out of tree - cd "$WORKSPACE" - if [[ -e "$WORKSPACE/cwltest" ]]; then - rm -rf "$WORKSPACE/cwltest" - fi - git clone https://github.com/common-workflow-language/cwltest.git - # signal to our build script that we want a cwltest executable installed in /usr/bin/ - mkdir cwltest/bin && touch cwltest/bin/cwltest - fpm_build_virtualenv "cwltest" "cwltest" "python3" - # The python->python3 metapackage - build_metapackage "cwltest" "cwltest" - cd "$WORKSPACE" - rm -rf "$WORKSPACE/cwltest" -fi - -calculate_go_package_version arvados_server_version cmd/arvados-server -arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") +# The cwltest package, which lives out of tree +handle_cwltest "$FORMAT" "$ARCH" -# Build the API server package -test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" -if [[ "$?" == "0" ]]; then - handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ - "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados API server - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" -fi +# Rails packages +debug_echo -e "\nRails packages\n" -# Build the workbench server package -if [[ "$HOSTTYPE" == "x86_64" ]]; then - test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" - if [[ "$?" == "0" ]] ; then - ( - set -e - - # The workbench package has a build-time dependency on the arvados-server - # package for config manipulation, so install it first. - cd $WORKSPACE/cmd/arvados-server - get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go - - arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}" - if [[ ! -e ${arvados_server_pkg_path} ]]; then - arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}" - fi - if [[ "$FORMAT" == "deb" ]]; then - dpkg -i ${arvados_server_pkg_path} - else - rpm -i ${arvados_server_pkg_path} - fi - - cd "$WORKSPACE/apps/workbench" - - # We need to bundle to be ready even when we build a package without vendor directory - # because asset compilation requires it. - bundle install --system >"$STDOUT_IF_DEBUG" - - # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, - # and we want that in the package, so it's easier to not exclude the tmp directory - # from the package - empty it instead. - rm -rf tmp - mkdir tmp - - # Set up an appropriate config.yml - arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x - mkdir -p /etc/arvados/ - mv /tmp/x /etc/arvados/config.yml - perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml - - ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG" - ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG" - - # Remove generated configuration files so they don't go in the package. - rm -rf /etc/arvados/ - ) - - if [[ "$?" != "0" ]]; then - echo "ERROR: Asset precompilation failed" - EXITCODE=1 - else - handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ - "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" - fi - fi -else - echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)." -fi +# The rails api server package +handle_api_server "$ARCH" +# The rails workbench package +handle_workbench "$ARCH" # clean up temporary GOPATH rm -rf "$GOPATH" diff --git a/build/run-library.sh b/build/run-library.sh index 92140bbdc8..ed3211c2a1 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -79,6 +79,25 @@ calculate_python_sdk_cwl_package_versions() { cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py) } +# Usage: get_native_arch +get_native_arch() { + # Only amd64 and aarch64 are supported at the moment + local native_arch="" + case "$HOSTTYPE" in + x86_64) + native_arch="amd64" + ;; + aarch64) + native_arch="arm64" + ;; + *) + echo "Error: architecture not supported" + exit 1 + ;; + esac + echo $native_arch +} + handle_ruby_gem() { local gem_name="$1"; shift local gem_version="$(nohash_version_from_git)" @@ -132,51 +151,48 @@ calculate_go_package_version() { # Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt] package_go_binary() { - local src_path="$1"; shift - local prog="$1"; shift - local package_format="$1"; shift - local target_arch="$1"; shift - local description="$1"; shift - local license_file="${1:-agpl-3.0.txt}"; shift - - if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then - # arvados-workbench depends on arvados-server at build time, so even when - # only arvados-workbench is being built, we need to build arvados-server too - if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then - return 0 - fi + local src_path="$1"; shift + local prog="$1"; shift + local package_format="$1"; shift + local target_arch="$1"; shift + local description="$1"; shift + local license_file="${1:-agpl-3.0.txt}"; shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then + # arvados-workbench depends on arvados-server at build time, so even when + # only arvados-workbench is being built, we need to build arvados-server too + if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then + debug_echo -e "Skipping build of $prog package." + return 0 fi + fi - # Only amd64 and aarch64 are supported - native_arch="amd64" - if [[ "$HOSTTYPE" == "aarch64" ]]; then - native_arch="arm64" - fi + native_arch=$(get_native_arch) - if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then - echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch" - return 1 - fi + if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then + echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch" + return 1 + fi - if [[ -n "$target_arch" ]]; then - # A target architecture has been specified - package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file" - return $? - else - # No target architecture specified, default to native target. When on amd64 also crosscompile arm64 - # but only when building deb packages (centos does not have support for crosscompiling userspace). - archs=($native_arch) - if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then - archs=('amd64' 'arm64') - fi - for ta in $archs; do - package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file" - retval=$? - if [[ "$retval" != 0 ]]; then - return $retval - fi - done + if [[ -n "$target_arch" ]]; then + # A target architecture has been specified + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file" + return $? + else + # No target architecture specified, default to native target. When on amd64 also crosscompile arm64 + # but only when building deb packages (centos does not have support for crosscompiling userspace). + archs=($native_arch) + if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then + archs=('amd64' 'arm64') fi + for ta in $archs; do + package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file" + retval=$? + if [[ "$retval" != 0 ]]; then + return $retval + fi + done + fi } # Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt] @@ -204,7 +220,7 @@ package_go_binary_worker() { return 1 fi - echo "Building $pachage_format ($target_arch) package for $prog from $src_path" + echo "Building $package_format ($target_arch) package for $prog from $src_path" if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path" else @@ -241,6 +257,7 @@ package_go_so() { local description="$1"; shift if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then + debug_echo -e "Skipping build of $pkg package." return 0 fi @@ -347,10 +364,9 @@ get_complete_package_name() { fi if [[ "$arch" == "" ]]; then - native_arch="amd64" + native_arch=$(get_native_arch) rpm_native_arch="x86_64" if [[ "$HOSTTYPE" == "aarch64" ]]; then - native_arch="arm64" rpm_native_arch="arm64" fi rpm_architecture="$rpm_native_arch" @@ -360,12 +376,6 @@ get_complete_package_name() { rpm_architecture="noarch" deb_architecture="all" fi - - # These python packages have binary components - if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then - rpm_architecture="$rpm_native_arch" - deb_architecture="$native_arch" - fi else rpm_architecture=$arch deb_architecture=$arch @@ -502,40 +512,247 @@ handle_rails_package() { rm -rf "$scripts_dir" } +# Usage: handle_api_server [amd64|arm64] +handle_api_server () { + local target_arch="${1:-amd64}"; shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-api-server" ]] ; then + debug_echo -e "Skipping build of arvados-api-server package." + return 0 + fi + + native_arch=$(get_native_arch) + if [[ "$target_arch" != "$native_arch" ]]; then + echo "Error: no cross compilation support for Rails yet, can not build arvados-api-server for $ARCH" + echo + exit 1 + fi + + # Build the API server package + test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" + if [[ "$?" == "0" ]]; then + calculate_go_package_version arvados_server_version cmd/arvados-server + arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") + handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ + "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ + --description="Arvados API server - Arvados is a free and open source platform for big data science." \ + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" + fi +} + +# Usage: handle_workbench [amd64|arm64] +handle_workbench () { + local target_arch="${1:-amd64}"; shift + if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-workbench" ]] ; then + debug_echo -e "Skipping build of arvados-workbench package." + return 0 + fi + + native_arch=$(get_native_arch) + if [[ "$target_arch" != "$native_arch" ]]; then + echo "Error: no cross compilation support for Rails yet, can not build arvados-workbench for $native_arch" + echo + exit 1 + fi + + if [[ "$native_arch" != "amd64" ]]; then + echo "Error: building the arvados-workbench package is not yet supported on this architecture ($native_arch)." + echo + exit 1 + fi + + # Build the workbench server package + test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" + if [[ "$?" == "0" ]] ; then + ( + set -e + + calculate_go_package_version arvados_server_version cmd/arvados-server + arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") + + # The workbench package has a build-time dependency on the arvados-server + # package for config manipulation, so install it first. + cd $WORKSPACE/cmd/arvados-server + get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go + + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}" + if [[ ! -e ${arvados_server_pkg_path} ]]; then + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}" + fi + if [[ "$FORMAT" == "deb" ]]; then + dpkg -i ${arvados_server_pkg_path} + else + rpm -i ${arvados_server_pkg_path} + fi + + cd "$WORKSPACE/apps/workbench" + + # We need to bundle to be ready even when we build a package without vendor directory + # because asset compilation requires it. + bundle install --system >"$STDOUT_IF_DEBUG" + + # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, + # and we want that in the package, so it's easier to not exclude the tmp directory + # from the package - empty it instead. + rm -rf tmp + mkdir tmp + + # Set up an appropriate config.yml + arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x + mkdir -p /etc/arvados/ + mv /tmp/x /etc/arvados/config.yml + perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml + + ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG" + ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG" + + # Remove generated configuration files so they don't go in the package. + rm -rf /etc/arvados/ + ) + + if [[ "$?" != "0" ]]; then + echo "ERROR: Asset precompilation failed" + EXITCODE=1 + else + handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ + "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ + --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" + fi + fi +} + +# Usage: handle_cwltest [deb|rpm] [amd64|arm64] +handle_cwltest () { + local package_format="$1"; shift + local target_arch="${1:-amd64}"; shift + + if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "python3-cwltest" ]] ; then + debug_echo -e "Skipping build of cwltest package." + return 0 + fi + cd "$WORKSPACE" + if [[ -e "$WORKSPACE/cwltest" ]]; then + rm -rf "$WORKSPACE/cwltest" + fi + git clone https://github.com/common-workflow-language/cwltest.git + # signal to our build script that we want a cwltest executable installed in /usr/bin/ + mkdir cwltest/bin && touch cwltest/bin/cwltest + fpm_build_virtualenv "cwltest" "cwltest" "$package_format" "$target_arch" + # The python->python3 metapackage + build_metapackage "cwltest" "cwltest" + cd "$WORKSPACE" + rm -rf "$WORKSPACE/cwltest" +} + +# Usage: handle_arvados_src +handle_arvados_src () { + if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "arvados-src" ]] ; then + debug_echo -e "Skipping build of arvados-src package." + return 0 + fi + # arvados-src + ( + cd "$WORKSPACE" + COMMIT_HASH=$(format_last_commit_here "%H") + arvados_src_version="$(version_from_git)" + + cd $WORKSPACE/packages/$TARGET + test_package_presence arvados-src "$arvados_src_version" src "" + + if [[ "$?" == "0" ]]; then + cd "$WORKSPACE" + SRC_BUILD_DIR=$(mktemp -d) + # mktemp creates the directory with 0700 permissions by default + chmod 755 $SRC_BUILD_DIR + git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" + cd "$SRC_BUILD_DIR" + + # go into detached-head state + git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" + echo "$COMMIT_HASH" >git-commit.version + + cd $WORKSPACE/packages/$TARGET + fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" + + rm -rf "$SRC_BUILD_DIR" + fi + ) +} + +# Usage: handle_libarvados_perl +handle_libarvados_perl () { + if [[ -n "$ONLY_BUILD" ]] || [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then + debug_echo -e "Skipping build of libarvados-perl package." + return 0 + fi + cd "$WORKSPACE/sdk/perl" + libarvados_perl_version="$(version_from_git)" + + cd $WORKSPACE/packages/$TARGET + test_package_presence libarvados-perl "$libarvados_perl_version" + + if [[ "$?" == "0" ]]; then + cd "$WORKSPACE/sdk/perl" + + if [[ -e Makefile ]]; then + make realclean >"$STDOUT_IF_DEBUG" + fi + find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ + -delete + rm -rf install + + perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ + make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ + fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \ + dir "$(version_from_git)" install/man/=/usr/share/man \ + "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \ + mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" + fi +} + # Build python packages with a virtualenv built-in +# Usage: fpm_build_virtualenv arvados-python-client sdk/python [deb|rpm] [amd64|arm64] fpm_build_virtualenv () { - PKG=$1 - shift - PKG_DIR=$1 - shift - PACKAGE_TYPE=${1:-python} - shift - native_arch="amd64" - if [[ "$HOSTTYPE" == "aarch64" ]]; then - native_arch="arm64" + local pkg=$1; shift + local pkg_dir=$1; shift + local package_format="$1"; shift + local target_arch="${1:-amd64}"; shift + + native_arch=$(get_native_arch) + + if [[ "$pkg" != "arvados-docker-cleaner" ]]; then + PYTHON_PKG=$PYTHON3_PKG_PREFIX-$pkg + else + # Exception to our package naming convention + PYTHON_PKG=$pkg fi - if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then - fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH" - elif [[ -z "$ARCH" ]]; then - for arch in $native_arch; do - fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch" - done + # arvados-python-client sdist should always be built, to be available + # for other dependent packages. + if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$pkg" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then + debug_echo -e "Skipping build of $pkg package." + return 0 + fi + + if [[ -n "$target_arch" ]] && [[ "$native_arch" == "$target_arch" ]]; then + fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$target_arch" + elif [[ -z "$target_arch" ]]; then + fpm_build_virtualenv_worker "$pkg" "$pkg_dir" "$package_format" "$native_arch" "$native_arch" else - echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH" + echo "Error: no cross compilation support for Python yet, can not build $pkg for $target_arch" + return 1 fi } # Build python packages with a virtualenv built-in +# Usage: fpm_build_virtualenv_worker arvados-python-client sdk/python python3 [deb|rpm] [amd64|arm64] [amd64|arm64] fpm_build_virtualenv_worker () { - PKG=$1 - shift - PKG_DIR=$1 - shift - PACKAGE_TYPE=${1:-python} - shift - arch=${1:-amd64} - shift + PKG=$1; shift + PKG_DIR=$1; shift + local package_format="$1"; shift + local native_arch="${1:-amd64}"; shift + local target_arch=${1:-amd64}; shift # Set up STDOUT_IF_DEBUG=/dev/null @@ -550,14 +767,9 @@ fpm_build_virtualenv_worker () { ARVADOS_BUILDING_ITERATION=1 fi - local python="" - case "$PACKAGE_TYPE" in - python3) - python=python3 - pip=pip3 - PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX - ;; - esac + local python=python3 + pip=pip3 + PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX if [[ "$PKG" != "arvados-docker-cleaner" ]]; then PYTHON_PKG=$PACKAGE_PREFIX-$PKG @@ -566,12 +778,6 @@ fpm_build_virtualenv_worker () { PYTHON_PKG=$PKG fi - # arvados-python-client sdist should always be built, to be available - # for other dependent packages. - if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then - return 0 - fi - cd $WORKSPACE/$PKG_DIR rm -rf dist/* @@ -611,11 +817,11 @@ fpm_build_virtualenv_worker () { # We can't do this earlier than here, because we need PYTHON_VERSION... # This isn't so bad; the sdist call above is pretty quick compared to # the invocation of virtualenv and fpm, below. - if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then + if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$python" "$ARVADOS_BUILDING_ITERATION" "$target_arch"; then return 0 fi - echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR" + echo "Building $package_format ($target_arch) package for $PKG from $PKG_DIR" # Package the sdist in a virtualenv echo "Creating virtualenv..." @@ -698,10 +904,10 @@ fpm_build_virtualenv_worker () { # Finally, generate the package echo "Creating package..." - declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT") + declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$package_format") - if [[ "${arch}" != "amd64" ]]; then - COMMAND_ARR+=("-a${arch}") + if [[ -n "$target_arch" ]] && [[ "$target_arch" != "amd64" ]]; then + COMMAND_ARR+=("-a$target_arch") fi if [[ "$MAINTAINER" != "" ]]; then @@ -722,7 +928,7 @@ fpm_build_virtualenv_worker () { LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"` COMMAND_ARR+=('--license' "$LICENSE_STRING") - if [[ "$FORMAT" == "rpm" ]]; then + if [[ "$package_format" == "rpm" ]]; then # Make sure to conflict with the old rh-python36 packages we used to publish COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG") fi @@ -808,8 +1014,8 @@ fpm_build_virtualenv_worker () { echo echo -e "\n${COMMAND_ARR[@]}\n" else - echo `ls *$FORMAT` - mv $WORKSPACE/$PKG_DIR/dist/*$FORMAT $WORKSPACE/packages/$TARGET/ + echo `ls *$package_format` + mv $WORKSPACE/$PKG_DIR/dist/*$package_format $WORKSPACE/packages/$TARGET/ fi echo } -- 2.30.2