2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 . `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
7 . `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
9 read -rd "\000" helpmessage <<EOF
10 $(basename $0): Build Arvados packages
13 WORKSPACE=/path/to/arvados $(basename $0) [options]
17 --build-bundle-packages (default: false)
18 Build api server and workbench packages with vendor/bundle included
20 Output debug information (default: false)
22 Distribution to build packages for (default: debian10)
23 --only-build <package>
24 Build only a specific package (or $ONLY_BUILD from environment)
26 Build a specific architecture (or $ARCH from environment, defaults to native architecture)
28 Build even if the package exists upstream or if it has already been
31 Build command to execute (defaults to the run command defined in the
34 WORKSPACE=path Path to the Arvados source tree to build packages from
38 # Begin of user configuration
40 # set to --no-cache-dir to disable pip caching
43 MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
44 VENDOR="The Arvados Project"
46 # End of user configuration
48 DEBUG=${ARVADOS_DEBUG:-0}
49 FORCE_BUILD=${FORCE_BUILD:-0}
54 PARSEDOPTS=$(getopt --name "$0" --longoptions \
55 help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
61 eval set -- "$PARSEDOPTS"
62 while [ $# -gt 0 ]; do
65 echo >&2 "$helpmessage"
73 ONLY_BUILD="$2"; shift
89 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
97 if [[ "$COMMAND" != "" ]]; then
98 COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
101 STDOUT_IF_DEBUG=/dev/null
102 STDERR_IF_DEBUG=/dev/null
103 DASHQ_UNLESS_DEBUG=-q
104 if [[ "$DEBUG" != 0 ]]; then
105 STDOUT_IF_DEBUG=/dev/stdout
106 STDERR_IF_DEBUG=/dev/stderr
110 declare -a PYTHON3_BACKPORTS
112 PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
114 ## These defaults are suitable for any Debian-based distribution.
115 # You can customize them as needed in distro sections below.
116 PYTHON3_PACKAGE=python$PYTHON3_VERSION
117 PYTHON3_PKG_PREFIX=python3
119 PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
120 ## End Debian Python defaults.
131 PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
132 PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
134 PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
135 export PYCURL_SSL_LIBRARY=nss
138 echo -e "$0: Unknown target '$TARGET'.\n" >&2
144 if ! [[ -n "$WORKSPACE" ]]; then
145 echo >&2 "$helpmessage"
147 echo >&2 "Error: WORKSPACE environment variable not set"
153 fpm --version >/dev/null 2>&1
155 if [[ "$?" != 0 ]]; then
156 echo >&2 "$helpmessage"
158 echo >&2 "Error: fpm not found"
163 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
164 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
165 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
166 # error; for some reason, the path is not accessible
167 # to the script (e.g. permissions re-evaled after suid)
171 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
172 debug_echo "Workspace is $WORKSPACE"
174 if [[ -f /etc/profile.d/rvm.sh ]]; then
175 source /etc/profile.d/rvm.sh
176 GEM="rvm-exec default gem"
181 # Make all files world-readable -- jenkins runs with umask 027, and has checked
182 # out our git tree here
183 chmod o+r "$WORKSPACE" -R
185 # More cleanup - make sure all executables that we'll package are 755
187 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
189 # Now fix our umask to something better suited to building and publishing
193 debug_echo "umask is" `umask`
195 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
196 mkdir -p $WORKSPACE/packages/$TARGET
197 chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
201 debug_echo -e "\nPerl packages\n"
203 handle_libarvados_perl
206 debug_echo -e "\nRuby gems\n"
208 FPM_GEM_PREFIX=$($GEM environment gemdir)
210 cd "$WORKSPACE/sdk/ruby"
211 handle_ruby_gem arvados
213 cd "$WORKSPACE/sdk/cli"
214 handle_ruby_gem arvados-cli
216 cd "$WORKSPACE/services/login-sync"
217 handle_ruby_gem arvados-login-sync
223 debug_echo -e "\nGo packages\n"
226 cd $WORKSPACE/packages/$TARGET
228 package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
229 "Arvados command line tool (beta)"
230 package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \
231 "Arvados server daemons"
232 package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \
233 "Arvados cluster controller daemon"
234 package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \
235 "Arvados cluster cloud dispatch"
236 package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
237 "Dispatch Arvados containers to an LSF cluster"
238 package_go_binary services/arv-git-httpd arvados-git-httpd "$FORMAT" "$ARCH" \
239 "Provide authenticated http access to Arvados-hosted git repositories"
240 package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
241 "Dispatch Crunch containers on the local system"
242 package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \
243 "Dispatch Crunch containers to a SLURM cluster"
244 package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
245 "Supervise a single Crunch container"
246 package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \
247 "Gather cpu/memory/network statistics of running Crunch jobs"
248 package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \
249 "Check health of all Arvados cluster services"
250 package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \
251 "Rebalance and garbage-collect data blocks stored in Arvados Keep"
252 package_go_binary services/keepproxy keepproxy "$FORMAT" "$ARCH" \
253 "Make a Keep cluster accessible to clients that are not on the LAN"
254 package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
255 "Keep storage daemon, accessible to clients on the LAN"
256 package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \
257 "Static web hosting service for user data stored in Arvados Keep"
258 package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
259 "Arvados Websocket server"
260 package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \
261 "Synchronize remote groups into Arvados from an external source"
262 package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \
263 "Verify that all data from one set of Keep servers to another was copied"
264 package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \
265 "Copy all data from one set of Keep servers to another"
266 package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
267 "Performance testing tool for Arvados Keep"
268 package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
269 "Arvados PAM authentication module"
272 debug_echo -e "\nPython packages\n"
274 # The Python SDK - Python3 package
275 fpm_build_virtualenv "arvados-python-client" "sdk/python" "$FORMAT" "$ARCH"
277 # Arvados cwl runner - Python3 package
278 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "$FORMAT" "$ARCH"
280 # The FUSE driver - Python3 package
281 fpm_build_virtualenv "arvados-fuse" "services/fuse" "$FORMAT" "$ARCH"
283 # The Arvados crunchstat-summary tool
284 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "$FORMAT" "$ARCH"
286 # The Docker image cleaner
287 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "$FORMAT" "$ARCH"
289 # The Arvados user activity tool
290 fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "$FORMAT" "$ARCH"
292 # The python->python3 metapackages
293 build_metapackage "arvados-fuse" "services/fuse"
294 build_metapackage "arvados-python-client" "services/fuse"
295 build_metapackage "arvados-cwl-runner" "sdk/cwl"
296 build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
297 build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
298 build_metapackage "arvados-user-activity" "tools/user-activity"
300 # The cwltest package, which lives out of tree
301 handle_cwltest "$FORMAT" "$ARCH"
304 debug_echo -e "\nRails packages\n"
306 # The rails api server package
307 handle_api_server "$ARCH"
308 # The rails workbench package
309 handle_workbench "$ARCH"
311 # clean up temporary GOPATH