X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/469ea187586ea8017e26874c2d80414ce7571fae..aca95ae8737d90adcee57929caa4239d2a7e1f66:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 3e1ed6a94d..ada3bf8b6c 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -9,16 +9,16 @@ read -rd "\000" helpmessage < [options] Options: --build-bundle-packages (default: false) - Build api server and workbench packages with vendor/bundle included + Build api server package with vendor/bundle included --debug Output debug information (default: false) --target - Distribution to build packages for (default: debian10) + Distribution to build packages for --only-build Build only a specific package (or ONLY_BUILD from environment) --arch @@ -47,8 +47,8 @@ VENDOR="The Arvados Project" DEBUG=${ARVADOS_DEBUG:-0} FORCE_BUILD=${FORCE_BUILD:-0} EXITCODE=0 -TARGET=debian10 COMMAND= +TARGET= PARSEDOPTS=$(getopt --name "$0" --longoptions \ help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \ @@ -93,6 +93,14 @@ while [ $# -gt 0 ]; do shift done +if [[ -z "$TARGET" ]]; then + echo "FATAL: --target must be specified" >&2 + exit 2 +elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then + echo "FATAL: unknown build target '$TARGET'" >&2 + exit 2 +fi + if [[ "$COMMAND" != "" ]]; then COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET" fi @@ -106,47 +114,42 @@ if [[ "$DEBUG" != 0 ]]; then DASHQ_UNLESS_DEBUG= fi -declare -a PYTHON3_BACKPORTS - -PYTHON3_EXECUTABLE=python3 -PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') - -## These defaults are suitable for any Debian-based distribution. -# You can customize them as needed in distro sections below. -PYTHON3_PACKAGE=python$PYTHON3_VERSION +# The next section defines a bunch of constants used to build distro packages +# for our Python tools. Because those packages include C extensions, they need +# to depend on and refer to a specific minor version of Python 3. The logic +# below should Just Work for most cases, but you can override variables for a +# specific distro if you need to to do something weird. +# * PYTHON3_VERSION: The major+minor version of Python we build against +# (e.g., "3.11") +# * PYTHON3_EXECUTABLE: The command to run that version of Python, +# either a full path or something in $PATH (e.g., "python3.11") +# * PYTHON3_PACKAGE: The name of the distro package that provides +# $PYTHON3_EXECUTABLE. Our Python packages will all depend on this. +# * PYTHON3_PKG_PREFIX: The prefix used in the names of all of our Python +# packages. This should match distro convention. PYTHON3_PKG_PREFIX=python3 -PYTHON3_PREFIX=/usr -PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages -## End Debian Python defaults. - case "$TARGET" in - debian*) - FORMAT=deb - ;; - ubuntu1804) - FORMAT=deb - PYTHON3_EXECUTABLE=python3.8 - PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))') - PYTHON3_PACKAGE=python$PYTHON3_VERSION - PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages + centos*|rocky*) + FORMAT=rpm ;; - ubuntu*) + debian*|ubuntu*) FORMAT=deb ;; - centos*) - FORMAT=rpm - PYTHON3_PACKAGE=$(rpm -qf "$(which python"$PYTHON3_VERSION")" --queryformat '%{NAME}\n') - PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE - PYTHON3_PREFIX=/usr - PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages - export PYCURL_SSL_LIBRARY=nss - ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 exit 1 ;; esac - +: "${PYTHON3_VERSION:=$("${PYTHON3_EXECUTABLE:-python3}" -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')}" +: "${PYTHON3_EXECUTABLE:=python$PYTHON3_VERSION}" +case "$FORMAT" in + deb) + : "${PYTHON3_PACKAGE:=python$PYTHON3_VERSION}" + ;; + rpm) + : "${PYTHON3_PACKAGE:=$(rpm -qf "$(command -v "$PYTHON3_EXECUTABLE")" --queryformat '%{NAME}\n')}" + ;; +esac if [[ -z "$WORKSPACE" ]]; then echo >&2 "$helpmessage" @@ -207,11 +210,6 @@ fi # Required due to CVE-2022-24765 git config --global --add safe.directory /arvados -# Perl packages -debug_echo -e "\nPerl packages\n" - -handle_libarvados_perl - # Ruby gems debug_echo -e "\nRuby gems\n" @@ -252,8 +250,6 @@ package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \ "Dispatch Crunch containers to a SLURM cluster" package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \ "Supervise a single Crunch container" -package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \ - "Gather cpu/memory/network statistics of running Crunch jobs" package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \ "Check health of all Arvados cluster services" package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \ @@ -268,6 +264,8 @@ package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \ "Arvados Websocket server" package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \ "Synchronize remote groups into Arvados from an external source" +package_go_binary tools/sync-users arvados-sync-users "$FORMAT" "$ARCH" \ + "Synchronize remote users into Arvados from an external source" package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \ "Verify that all data from one set of Keep servers to another was copied" package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \ @@ -280,42 +278,25 @@ package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \ # Python packages debug_echo -e "\nPython packages\n" -# The Python SDK - Python3 package +# Before a Python package can be built, its dependencies must already be built. +# This list is ordered accordingly. +setup_build_virtualenv +fpm_build_virtualenv cwltest "==2.3.20230108193615" "$FORMAT" "$ARCH" fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH" - -# Arvados cwl runner - Python3 package -fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH" - -# The FUSE driver - Python3 package -fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH" - -# The Arvados crunchstat-summary tool fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH" - -# The Docker image cleaner +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH" fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH" - -# The Arvados user activity tool +fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH" fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH" -# The python->python3 metapackages -build_metapackage "arvados-fuse" "services/fuse" -build_metapackage "arvados-python-client" "services/fuse" -build_metapackage "arvados-cwl-runner" "sdk/cwl" -build_metapackage "crunchstat-summary" "tools/crunchstat-summary" -build_metapackage "arvados-docker-cleaner" "services/dockercleaner" -build_metapackage "arvados-user-activity" "tools/user-activity" - -# The cwltest package, which lives out of tree -handle_cwltest "$FORMAT" "$ARCH" +# Workbench2 +package_workbench2 # Rails packages debug_echo -e "\nRails packages\n" # The rails api server package handle_api_server "$ARCH" -# The rails workbench package -handle_workbench "$ARCH" # clean up temporary GOPATH rm -rf "$GOPATH"