2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 . "$(dirname "$(readlink -f "$0")")"/run-library.sh || exit 1
8 read -rd "\000" helpmessage <<EOF
9 $(basename "$0"): Build Arvados packages
12 WORKSPACE=/path/to/arvados $(basename "$0") --target <target> [options]
16 --build-bundle-packages (default: false)
17 Build api server package with vendor/bundle included
19 Output debug information (default: false)
21 Distribution to build packages for
22 --only-build <package>
23 Build only a specific package (or ONLY_BUILD from environment)
25 Build a specific architecture (or ARCH from environment, defaults to native architecture)
27 Build even if the package exists upstream or if it has already been
30 Build command to execute (defaults to the run command defined in the
33 WORKSPACE=path Path to the Arvados source tree to build packages from
37 # Begin of user configuration
39 # set to --no-cache-dir to disable pip caching
42 MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
43 VENDOR="The Arvados Project"
45 # End of user configuration
47 DEBUG=${ARVADOS_DEBUG:-0}
48 FORCE_BUILD=${FORCE_BUILD:-0}
53 PARSEDOPTS=$(getopt --name "$0" --longoptions \
54 help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
60 eval set -- "$PARSEDOPTS"
61 while [ $# -gt 0 ]; do
64 echo >&2 "$helpmessage"
72 ONLY_BUILD="$2"; shift
88 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
96 if [[ -z "$TARGET" ]]; then
97 echo "FATAL: --target must be specified" >&2
99 elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then
100 echo "FATAL: unknown build target '$TARGET'" >&2
104 if [[ "$COMMAND" != "" ]]; then
105 COMMAND="bash /jenkins/$COMMAND --target $TARGET"
108 STDOUT_IF_DEBUG=/dev/null
109 STDERR_IF_DEBUG=/dev/null
110 DASHQ_UNLESS_DEBUG=-q
111 if [[ "$DEBUG" != 0 ]]; then
112 STDOUT_IF_DEBUG=/dev/stdout
113 STDERR_IF_DEBUG=/dev/stderr
117 # The next section defines a bunch of constants used to build distro packages
118 # for our Python tools. Because those packages include C extensions, they need
119 # to depend on and refer to a specific minor version of Python 3. The logic
120 # below should Just Work for most cases, but you can override variables for a
121 # specific distro if you need to to do something weird.
122 # * PYTHON3_VERSION: The major+minor version of Python we build against
124 # * PYTHON3_EXECUTABLE: The command to run that version of Python,
125 # either a full path or something in $PATH (e.g., "python3.11")
126 # * PYTHON3_PACKAGE: The name of the distro package that provides
127 # $PYTHON3_EXECUTABLE. Our Python packages will all depend on this.
128 # * PYTHON3_PKG_PREFIX: The prefix used in the names of all of our Python
129 # packages. This should match distro convention.
130 PYTHON3_PKG_PREFIX=python3
139 echo -e "$0: Unknown target '$TARGET'.\n" >&2
143 : "${PYTHON3_VERSION:=$("${PYTHON3_EXECUTABLE:-python3}" -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')}"
144 : "${PYTHON3_EXECUTABLE:=python$PYTHON3_VERSION}"
147 : "${PYTHON3_PACKAGE:=python$PYTHON3_VERSION}"
150 : "${PYTHON3_PACKAGE:=$(rpm -qf "$(command -v "$PYTHON3_EXECUTABLE")" --queryformat '%{NAME}\n')}"
154 if [[ -z "$WORKSPACE" ]]; then
155 echo >&2 "$helpmessage"
157 echo >&2 "Error: WORKSPACE environment variable not set"
163 fpm --version >/dev/null 2>&1
165 if [[ $? -ne 0 ]]; then
166 echo >&2 "$helpmessage"
168 echo >&2 "Error: fpm not found"
173 RUN_BUILD_PACKAGES_PATH="$(dirname "$0")"
174 RUN_BUILD_PACKAGES_PATH="$(cd "$RUN_BUILD_PACKAGES_PATH" && pwd)" # absolutized and normalized
175 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
176 # error; for some reason, the path is not accessible
177 # to the script (e.g. permissions re-evaled after suid)
181 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
182 debug_echo "Workspace is $WORKSPACE"
184 # Make all files world-readable -- jenkins runs with umask 027, and has checked
185 # out our git tree here
186 chmod o+r "$WORKSPACE" -R
188 # More cleanup - make sure all executables that we'll package are 755
189 cd "$WORKSPACE" || exit 1
190 find . -type d -name 'bin' -print0 |xargs -0 -I {} find {} -type f -print0 |xargs -0 -I {} chmod 755 {}
192 # Now fix our umask to something better suited to building and publishing
196 debug_echo "umask is" "$(umask)"
198 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
199 mkdir -p "$WORKSPACE/packages/$TARGET"
200 chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
203 # Required due to CVE-2022-24765
204 git config --global --add safe.directory /arvados
207 debug_echo -e "\nRuby gems\n"
209 FPM_GEM_PREFIX=$(gem environment gemdir)
211 cd "$WORKSPACE/sdk/ruby" || exit 1
212 handle_ruby_gem arvados
214 cd "$WORKSPACE/sdk/cli" || exit 1
215 handle_ruby_gem arvados-cli
217 cd "$WORKSPACE/services/login-sync" || exit 1
218 handle_ruby_gem arvados-login-sync
224 debug_echo -e "\nGo packages\n"
228 package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
229 "Arvados command line tool (beta)"
230 package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \
231 "Arvados server daemons"
232 package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \
233 "Arvados cluster controller daemon"
234 package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \
235 "Arvados cluster cloud dispatch"
236 package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
237 "Dispatch Arvados containers to an LSF cluster"
238 package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
239 "Dispatch Crunch containers on the local system"
240 package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \
241 "Dispatch Crunch containers to a SLURM cluster"
242 package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
243 "Supervise a single Crunch container"
244 package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \
245 "Check health of all Arvados cluster services"
246 package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \
247 "Rebalance and garbage-collect data blocks stored in Arvados Keep"
248 package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \
249 "Make a Keep cluster accessible to clients that are not on the LAN"
250 package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
251 "Keep storage daemon, accessible to clients on the LAN"
252 package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \
253 "Static web hosting service for user data stored in Arvados Keep"
254 package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
255 "Arvados Websocket server"
256 package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \
257 "Synchronize remote groups into Arvados from an external source"
258 package_go_binary tools/sync-users arvados-sync-users "$FORMAT" "$ARCH" \
259 "Synchronize remote users into Arvados from an external source"
260 package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \
261 "Verify that all data from one set of Keep servers to another was copied"
262 package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \
263 "Copy all data from one set of Keep servers to another"
264 package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
265 "Performance testing tool for Arvados Keep"
266 package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
267 "Arvados PAM authentication module"
270 debug_echo -e "\nPython packages\n"
272 # Before a Python package can be built, its dependencies must already be built.
273 # This list is ordered accordingly.
274 setup_build_virtualenv
275 fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
276 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
277 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
278 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
279 fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
280 fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
281 fpm_build_virtualenv "arvados-cluster-activity" "tools/cluster-activity" "$FORMAT" "$ARCH"
287 debug_echo -e "\nRails packages\n"
289 # The rails api server package
290 handle_api_server "$ARCH"
292 # clean up temporary GOPATH