2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 . "$(dirname "$(readlink -f "$0")")"/run-library.sh || exit 1
8 read -rd "\000" helpmessage <<EOF
9 $(basename "$0"): Build Arvados packages
12 WORKSPACE=/path/to/arvados $(basename "$0") --target <target> [options]
16 --build-bundle-packages (default: false)
17 Build api server package with vendor/bundle included
19 Output debug information (default: false)
21 Distribution to build packages for
22 --only-build <package>
23 Build only a specific package (or ONLY_BUILD from environment)
25 Build a specific architecture (or ARCH from environment, defaults to native architecture)
27 Build even if the package exists upstream or if it has already been
30 Build command to execute (defaults to the run command defined in the
33 WORKSPACE=path Path to the Arvados source tree to build packages from
37 # Begin of user configuration
39 # set to --no-cache-dir to disable pip caching
42 MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
43 VENDOR="The Arvados Project"
45 # End of user configuration
47 DEBUG=${ARVADOS_DEBUG:-0}
48 FORCE_BUILD=${FORCE_BUILD:-0}
53 PARSEDOPTS=$(getopt --name "$0" --longoptions \
54 help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
60 eval set -- "$PARSEDOPTS"
61 while [ $# -gt 0 ]; do
64 echo >&2 "$helpmessage"
72 ONLY_BUILD="$2"; shift
88 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
96 if [[ -z "$TARGET" ]]; then
97 echo "FATAL: --target must be specified" >&2
99 elif [[ ! -d "$WORKSPACE/build/package-build-dockerfiles/$TARGET" ]]; then
100 echo "FATAL: unknown build target '$TARGET'" >&2
104 if [[ "$COMMAND" != "" ]]; then
105 COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
108 STDOUT_IF_DEBUG=/dev/null
109 STDERR_IF_DEBUG=/dev/null
110 DASHQ_UNLESS_DEBUG=-q
111 if [[ "$DEBUG" != 0 ]]; then
112 STDOUT_IF_DEBUG=/dev/stdout
113 STDERR_IF_DEBUG=/dev/stderr
117 # The next section defines a bunch of constants used to build distro packages
118 # for our Python tools. Because those packages include C extensions, they need
119 # to depend on and refer to a specific minor version of Python 3. The logic
120 # below should Just Work for most cases, but you can override variables for a
121 # specific distro if you need to to do something weird.
122 # * PYTHON3_VERSION: The major+minor version of Python we build against
124 # * PYTHON3_EXECUTABLE: The command to run that version of Python,
125 # either a full path or something in $PATH (e.g., "python3.11")
126 # * PYTHON3_PACKAGE: The name of the distro package that provides
127 # $PYTHON3_EXECUTABLE. Our Python packages will all depend on this.
128 # * PYTHON3_PKG_PREFIX: The prefix used in the names of all of our Python
129 # packages. This should match distro convention.
130 PYTHON3_PKG_PREFIX=python3
139 echo -e "$0: Unknown target '$TARGET'.\n" >&2
143 : "${PYTHON3_VERSION:=$("${PYTHON3_EXECUTABLE:-python3}" -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')}"
144 : "${PYTHON3_EXECUTABLE:=python$PYTHON3_VERSION}"
147 : "${PYTHON3_PACKAGE:=python$PYTHON3_VERSION}"
150 : "${PYTHON3_PACKAGE:=$(rpm -qf "$(command -v "$PYTHON3_EXECUTABLE")" --queryformat '%{NAME}\n')}"
154 if [[ -z "$WORKSPACE" ]]; then
155 echo >&2 "$helpmessage"
157 echo >&2 "Error: WORKSPACE environment variable not set"
163 fpm --version >/dev/null 2>&1
165 if [[ $? -ne 0 ]]; then
166 echo >&2 "$helpmessage"
168 echo >&2 "Error: fpm not found"
173 RUN_BUILD_PACKAGES_PATH="$(dirname "$0")"
174 RUN_BUILD_PACKAGES_PATH="$(cd "$RUN_BUILD_PACKAGES_PATH" && pwd)" # absolutized and normalized
175 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
176 # error; for some reason, the path is not accessible
177 # to the script (e.g. permissions re-evaled after suid)
181 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
182 debug_echo "Workspace is $WORKSPACE"
184 if [[ -f /etc/profile.d/rvm.sh ]]; then
185 source /etc/profile.d/rvm.sh
186 GEM="rvm-exec default gem"
191 # Make all files world-readable -- jenkins runs with umask 027, and has checked
192 # out our git tree here
193 chmod o+r "$WORKSPACE" -R
195 # More cleanup - make sure all executables that we'll package are 755
196 cd "$WORKSPACE" || exit 1
197 find . -type d -name 'bin' -print0 |xargs -0 -I {} find {} -type f -print0 |xargs -0 -I {} chmod 755 {}
199 # Now fix our umask to something better suited to building and publishing
203 debug_echo "umask is" "$(umask)"
205 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
206 mkdir -p "$WORKSPACE/packages/$TARGET"
207 chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
210 # Required due to CVE-2022-24765
211 git config --global --add safe.directory /arvados
214 debug_echo -e "\nRuby gems\n"
216 FPM_GEM_PREFIX=$($GEM environment gemdir)
218 cd "$WORKSPACE/sdk/ruby" || exit 1
219 handle_ruby_gem arvados
221 cd "$WORKSPACE/sdk/cli" || exit 1
222 handle_ruby_gem arvados-cli
224 cd "$WORKSPACE/services/login-sync" || exit 1
225 handle_ruby_gem arvados-login-sync
231 debug_echo -e "\nGo packages\n"
235 package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
236 "Arvados command line tool (beta)"
237 package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \
238 "Arvados server daemons"
239 package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \
240 "Arvados cluster controller daemon"
241 package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \
242 "Arvados cluster cloud dispatch"
243 package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
244 "Dispatch Arvados containers to an LSF cluster"
245 package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \
246 "Provide authenticated http access to Arvados-hosted git repositories"
247 package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
248 "Dispatch Crunch containers on the local system"
249 package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \
250 "Dispatch Crunch containers to a SLURM cluster"
251 package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
252 "Supervise a single Crunch container"
253 package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \
254 "Check health of all Arvados cluster services"
255 package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \
256 "Rebalance and garbage-collect data blocks stored in Arvados Keep"
257 package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \
258 "Make a Keep cluster accessible to clients that are not on the LAN"
259 package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
260 "Keep storage daemon, accessible to clients on the LAN"
261 package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \
262 "Static web hosting service for user data stored in Arvados Keep"
263 package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
264 "Arvados Websocket server"
265 package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \
266 "Synchronize remote groups into Arvados from an external source"
267 package_go_binary tools/sync-users arvados-sync-users "$FORMAT" "$ARCH" \
268 "Synchronize remote users into Arvados from an external source"
269 package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \
270 "Verify that all data from one set of Keep servers to another was copied"
271 package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \
272 "Copy all data from one set of Keep servers to another"
273 package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
274 "Performance testing tool for Arvados Keep"
275 package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
276 "Arvados PAM authentication module"
279 debug_echo -e "\nPython packages\n"
281 # The Python SDK - Python3 package
282 fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
284 # Arvados cwl runner - Python3 package
285 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
287 # The FUSE driver - Python3 package
288 fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
290 # The Arvados crunchstat-summary tool
291 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
293 # The Docker image cleaner
294 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
296 # The Arvados user activity tool
297 fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
299 # The cwltest package, which lives out of tree
300 handle_cwltest "$FORMAT" "$ARCH"
306 debug_echo -e "\nRails packages\n"
308 # The rails api server package
309 handle_api_server "$ARCH"
311 # clean up temporary GOPATH