X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/42fff42165a0fa1602758a078746f8697f265f83..9acc8cb9cd9ca4429712b0d31b647e9a6ecf2d96:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index f9db56a5ff..adcbab8a09 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -3,14 +3,13 @@ # # SPDX-License-Identifier: AGPL-3.0 -. `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1 -. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1 +. "$(dirname "$(readlink -f "$0")")"/run-library.sh || exit 1 read -rd "\000" helpmessage < Distribution to build packages for (default: debian10) --only-build - Build only a specific package (or $ONLY_BUILD from environment) + Build only a specific package (or ONLY_BUILD from environment) +--arch + Build a specific architecture (or ARCH from environment, defaults to native architecture) --force-build Build even if the package exists upstream or if it has already been built locally @@ -50,7 +51,7 @@ TARGET=debian10 COMMAND= PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,build-bundle-packages,debug,target:,only-build:,force-build \ + help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -73,6 +74,9 @@ while [ $# -gt 0 ]; do --force-build) FORCE_BUILD=1 ;; + --arch) + ARCH="$2"; shift + ;; --debug) DEBUG=1 ;; @@ -104,7 +108,8 @@ fi declare -a PYTHON3_BACKPORTS -PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') +PYTHON3_EXECUTABLE=python3 +PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') ## These defaults are suitable for any Debian-based distribution. # You can customize them as needed in distro sections below. @@ -118,12 +123,19 @@ case "$TARGET" in debian*) FORMAT=deb ;; + ubuntu1804) + FORMAT=deb + PYTHON3_EXECUTABLE=python3.8 + PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') + PYTHON3_PACKAGE=python$PYTHON3_VERSION + PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages + ;; ubuntu*) FORMAT=deb ;; centos*) FORMAT=rpm - PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PACKAGE=$(rpm -qf "$(which python"$PYTHON3_VERSION")" --queryformat '%{NAME}\n') PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE PYTHON3_PREFIX=/usr PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages @@ -136,7 +148,7 @@ case "$TARGET" in esac -if ! [[ -n "$WORKSPACE" ]]; then +if [[ -z "$WORKSPACE" ]]; then echo >&2 "$helpmessage" echo >&2 echo >&2 "Error: WORKSPACE environment variable not set" @@ -147,7 +159,7 @@ fi # Test for fpm fpm --version >/dev/null 2>&1 -if [[ "$?" != 0 ]]; then +if [[ $? -ne 0 ]]; then echo >&2 "$helpmessage" echo >&2 echo >&2 "Error: fpm not found" @@ -155,8 +167,8 @@ if [[ "$?" != 0 ]]; then exit 1 fi -RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`" -RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized +RUN_BUILD_PACKAGES_PATH="$(dirname "$0")" +RUN_BUILD_PACKAGES_PATH="$(cd "$RUN_BUILD_PACKAGES_PATH" && pwd)" # absolutized and normalized if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then # error; for some reason, the path is not accessible # to the script (e.g. permissions re-evaled after suid) @@ -178,155 +190,113 @@ fi chmod o+r "$WORKSPACE" -R # More cleanup - make sure all executables that we'll package are 755 -cd "$WORKSPACE" -find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {} +cd "$WORKSPACE" || exit 1 +find . -type d -name 'bin' -print0 |xargs -0 -I {} find {} -type f -print0 |xargs -0 -I {} chmod 755 {} # Now fix our umask to something better suited to building and publishing # gems and packages umask 0022 -debug_echo "umask is" `umask` +debug_echo "umask is" "$(umask)" if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then - mkdir -p $WORKSPACE/packages/$TARGET + mkdir -p "$WORKSPACE/packages/$TARGET" chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET" fi +# Required due to CVE-2022-24765 +git config --global --add safe.directory /arvados + # Perl packages debug_echo -e "\nPerl packages\n" -if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then - cd "$WORKSPACE/sdk/perl" - libarvados_perl_version="$(version_from_git)" - - cd $WORKSPACE/packages/$TARGET - test_package_presence libarvados-perl "$libarvados_perl_version" - - if [[ "$?" == "0" ]]; then - cd "$WORKSPACE/sdk/perl" - - if [[ -e Makefile ]]; then - make realclean >"$STDOUT_IF_DEBUG" - fi - find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ - -delete - rm -rf install - - perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ - make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ - fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \ - dir "$(version_from_git)" install/man/=/usr/share/man \ - "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \ - mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" - fi -fi +handle_libarvados_perl # Ruby gems debug_echo -e "\nRuby gems\n" FPM_GEM_PREFIX=$($GEM environment gemdir) -cd "$WORKSPACE/sdk/ruby" +cd "$WORKSPACE/sdk/ruby" || exit 1 handle_ruby_gem arvados -cd "$WORKSPACE/sdk/cli" +cd "$WORKSPACE/sdk/cli" || exit 1 handle_ruby_gem arvados-cli -cd "$WORKSPACE/services/login-sync" +cd "$WORKSPACE/services/login-sync" || exit 1 handle_ruby_gem arvados-login-sync -# Python packages -debug_echo -e "\nPython packages\n" - # arvados-src -( - cd "$WORKSPACE" - COMMIT_HASH=$(format_last_commit_here "%H") - arvados_src_version="$(version_from_git)" - - cd $WORKSPACE/packages/$TARGET - test_package_presence arvados-src $arvados_src_version src "" - - if [[ "$?" == "0" ]]; then - cd "$WORKSPACE" - SRC_BUILD_DIR=$(mktemp -d) - # mktemp creates the directory with 0700 permissions by default - chmod 755 $SRC_BUILD_DIR - git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" - cd "$SRC_BUILD_DIR" +handle_arvados_src - # go into detached-head state - git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" - echo "$COMMIT_HASH" >git-commit.version - - cd $WORKSPACE/packages/$TARGET - fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" - - rm -rf "$SRC_BUILD_DIR" - fi -) +# Go packages +debug_echo -e "\nGo packages\n" # Go binaries -cd $WORKSPACE/packages/$TARGET -export GOPATH=$(mktemp -d) -package_go_binary cmd/arvados-client arvados-client \ +export GOPATH=~/go +package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \ "Arvados command line tool (beta)" -package_go_binary cmd/arvados-server arvados-server \ +package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \ "Arvados server daemons" -package_go_binary cmd/arvados-server arvados-controller \ +package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \ "Arvados cluster controller daemon" -package_go_binary cmd/arvados-server arvados-dispatch-cloud \ +package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \ "Arvados cluster cloud dispatch" -package_go_binary services/arv-git-httpd arvados-git-httpd \ +package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \ + "Dispatch Arvados containers to an LSF cluster" +package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \ "Provide authenticated http access to Arvados-hosted git repositories" -package_go_binary services/crunch-dispatch-local crunch-dispatch-local \ +package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \ "Dispatch Crunch containers on the local system" -package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \ +package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \ "Dispatch Crunch containers to a SLURM cluster" -package_go_binary cmd/arvados-server crunch-run \ +package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \ "Supervise a single Crunch container" -package_go_binary services/crunchstat crunchstat \ +package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \ "Gather cpu/memory/network statistics of running Crunch jobs" -package_go_binary services/health arvados-health \ +package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \ "Check health of all Arvados cluster services" -package_go_binary services/keep-balance keep-balance \ +package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" -package_go_binary services/keepproxy keepproxy \ +package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \ "Make a Keep cluster accessible to clients that are not on the LAN" -package_go_binary services/keepstore keepstore \ +package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \ "Keep storage daemon, accessible to clients on the LAN" -package_go_binary services/keep-web keep-web \ +package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \ "Static web hosting service for user data stored in Arvados Keep" -package_go_binary cmd/arvados-server arvados-ws \ +package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \ "Arvados Websocket server" -package_go_binary tools/sync-groups arvados-sync-groups \ +package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \ "Synchronize remote groups into Arvados from an external source" -package_go_binary tools/keep-block-check keep-block-check \ +package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \ "Verify that all data from one set of Keep servers to another was copied" -package_go_binary tools/keep-rsync keep-rsync \ +package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \ "Copy all data from one set of Keep servers to another" -package_go_binary tools/keep-exercise keep-exercise \ +package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \ "Performance testing tool for Arvados Keep" -package_go_so lib/pam pam_arvados.so libpam-arvados-go \ +package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \ "Arvados PAM authentication module" +# Python packages +debug_echo -e "\nPython packages\n" + # The Python SDK - Python3 package -fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" +fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH" # Arvados cwl runner - Python3 package -fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3" +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH" # The FUSE driver - Python3 package -fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3" +fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH" # The Arvados crunchstat-summary tool -fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3" +fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH" # The Docker image cleaner -fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" +fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH" # The Arvados user activity tool -fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3" +fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH" # The python->python3 metapackages build_metapackage "arvados-fuse" "services/fuse" @@ -337,87 +307,15 @@ build_metapackage "arvados-docker-cleaner" "services/dockercleaner" build_metapackage "arvados-user-activity" "tools/user-activity" # The cwltest package, which lives out of tree -cd "$WORKSPACE" -if [[ -e "$WORKSPACE/cwltest" ]]; then - rm -rf "$WORKSPACE/cwltest" -fi -git clone https://github.com/common-workflow-language/cwltest.git -# signal to our build script that we want a cwltest executable installed in /usr/bin/ -mkdir cwltest/bin && touch cwltest/bin/cwltest -fpm_build_virtualenv "cwltest" "cwltest" "python3" -# The python->python3 metapackage -build_metapackage "cwltest" "cwltest" -cd "$WORKSPACE" -rm -rf "$WORKSPACE/cwltest" - -calculate_go_package_version arvados_server_version cmd/arvados-server -arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") - -# Build the API server package -test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" -if [[ "$?" == "0" ]]; then - handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ - "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados API server - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" -fi +handle_cwltest "$FORMAT" "$ARCH" -# Build the workbench server package -test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" -if [[ "$?" == "0" ]] ; then - ( - set -e - - # The workbench package has a build-time dependency on the arvados-server - # package for config manipulation, so install it first. - cd $WORKSPACE/cmd/arvados-server - get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go - - arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}" - if [[ ! -e ${arvados_server_pkg_path} ]]; then - arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}" - fi - if [[ "$FORMAT" == "deb" ]]; then - dpkg -i ${arvados_server_pkg_path} - else - rpm -i ${arvados_server_pkg_path} - fi - - cd "$WORKSPACE/apps/workbench" - - # We need to bundle to be ready even when we build a package without vendor directory - # because asset compilation requires it. - bundle install --system >"$STDOUT_IF_DEBUG" - - # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, - # and we want that in the package, so it's easier to not exclude the tmp directory - # from the package - empty it instead. - rm -rf tmp - mkdir tmp - - # Set up an appropriate config.yml - arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x - mkdir -p /etc/arvados/ - mv /tmp/x /etc/arvados/config.yml - perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml - - RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >"$STDOUT_IF_DEBUG" - RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >"$STDOUT_IF_DEBUG" - - # Remove generated configuration files so they don't go in the package. - rm -rf /etc/arvados/ - ) - - if [[ "$?" != "0" ]]; then - echo "ERROR: Asset precompilation failed" - EXITCODE=1 - else - handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ - "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" - fi -fi +# Rails packages +debug_echo -e "\nRails packages\n" + +# The rails api server package +handle_api_server "$ARCH" +# The rails workbench package +handle_workbench "$ARCH" # clean up temporary GOPATH rm -rf "$GOPATH"