2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 . `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
7 . `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
9 read -rd "\000" helpmessage <<EOF
10 $(basename $0): Build Arvados packages
13 WORKSPACE=/path/to/arvados $(basename $0) [options]
17 --build-bundle-packages (default: false)
18 Build api server and workbench packages with vendor/bundle included
20 Output debug information (default: false)
22 Distribution to build packages for (default: debian10)
23 --only-build <package>
24 Build only a specific package (or $ONLY_BUILD from environment)
25 --only-arch <architecture>
26 Build only a specific architecture (or $ONLY_ARCH from environment)
28 Build even if the package exists upstream or if it has already been
31 Build command to execute (defaults to the run command defined in the
34 WORKSPACE=path Path to the Arvados source tree to build packages from
38 # Begin of user configuration
40 # set to --no-cache-dir to disable pip caching
43 MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
44 VENDOR="The Arvados Project"
46 # End of user configuration
48 DEBUG=${ARVADOS_DEBUG:-0}
49 FORCE_BUILD=${FORCE_BUILD:-0}
54 PARSEDOPTS=$(getopt --name "$0" --longoptions \
55 help,build-bundle-packages,debug,target:,only-build:,only-arch:,force-build \
61 eval set -- "$PARSEDOPTS"
62 while [ $# -gt 0 ]; do
65 echo >&2 "$helpmessage"
73 ONLY_BUILD="$2"; shift
89 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
97 if [[ "$COMMAND" != "" ]]; then
98 COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
101 STDOUT_IF_DEBUG=/dev/null
102 STDERR_IF_DEBUG=/dev/null
103 DASHQ_UNLESS_DEBUG=-q
104 if [[ "$DEBUG" != 0 ]]; then
105 STDOUT_IF_DEBUG=/dev/stdout
106 STDERR_IF_DEBUG=/dev/stderr
110 declare -a PYTHON3_BACKPORTS
112 PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
114 ## These defaults are suitable for any Debian-based distribution.
115 # You can customize them as needed in distro sections below.
116 PYTHON3_PACKAGE=python$PYTHON3_VERSION
117 PYTHON3_PKG_PREFIX=python3
119 PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
120 ## End Debian Python defaults.
131 PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
132 PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
134 PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
135 export PYCURL_SSL_LIBRARY=nss
138 echo -e "$0: Unknown target '$TARGET'.\n" >&2
144 if ! [[ -n "$WORKSPACE" ]]; then
145 echo >&2 "$helpmessage"
147 echo >&2 "Error: WORKSPACE environment variable not set"
153 fpm --version >/dev/null 2>&1
155 if [[ "$?" != 0 ]]; then
156 echo >&2 "$helpmessage"
158 echo >&2 "Error: fpm not found"
163 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
164 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
165 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
166 # error; for some reason, the path is not accessible
167 # to the script (e.g. permissions re-evaled after suid)
171 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
172 debug_echo "Workspace is $WORKSPACE"
174 if [[ -f /etc/profile.d/rvm.sh ]]; then
175 source /etc/profile.d/rvm.sh
176 GEM="rvm-exec default gem"
181 # Make all files world-readable -- jenkins runs with umask 027, and has checked
182 # out our git tree here
183 chmod o+r "$WORKSPACE" -R
185 # More cleanup - make sure all executables that we'll package are 755
187 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
189 # Now fix our umask to something better suited to building and publishing
193 debug_echo "umask is" `umask`
195 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
196 mkdir -p $WORKSPACE/packages/$TARGET
197 chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
201 debug_echo -e "\nPerl packages\n"
203 if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
204 cd "$WORKSPACE/sdk/perl"
205 libarvados_perl_version="$(version_from_git)"
207 cd $WORKSPACE/packages/$TARGET
208 test_package_presence libarvados-perl "$libarvados_perl_version"
210 if [[ "$?" == "0" ]]; then
211 cd "$WORKSPACE/sdk/perl"
213 if [[ -e Makefile ]]; then
214 make realclean >"$STDOUT_IF_DEBUG"
216 find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
220 perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
221 make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
222 fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
223 dir "$(version_from_git)" install/man/=/usr/share/man \
224 "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
225 mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
230 debug_echo -e "\nRuby gems\n"
232 FPM_GEM_PREFIX=$($GEM environment gemdir)
234 cd "$WORKSPACE/sdk/ruby"
235 handle_ruby_gem arvados
237 cd "$WORKSPACE/sdk/cli"
238 handle_ruby_gem arvados-cli
240 cd "$WORKSPACE/services/login-sync"
241 handle_ruby_gem arvados-login-sync
244 debug_echo -e "\nPython packages\n"
249 COMMIT_HASH=$(format_last_commit_here "%H")
250 arvados_src_version="$(version_from_git)"
252 cd $WORKSPACE/packages/$TARGET
253 test_package_presence arvados-src $arvados_src_version src ""
255 if [[ "$?" == "0" ]]; then
257 SRC_BUILD_DIR=$(mktemp -d)
258 # mktemp creates the directory with 0700 permissions by default
259 chmod 755 $SRC_BUILD_DIR
260 git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
263 # go into detached-head state
264 git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
265 echo "$COMMIT_HASH" >git-commit.version
267 cd $WORKSPACE/packages/$TARGET
268 fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
270 rm -rf "$SRC_BUILD_DIR"
275 cd $WORKSPACE/packages/$TARGET
276 export GOPATH=$(mktemp -d)
277 package_go_binary cmd/arvados-client arvados-client \
278 "Arvados command line tool (beta)"
279 package_go_binary cmd/arvados-server arvados-server \
280 "Arvados server daemons"
281 package_go_binary cmd/arvados-server arvados-controller \
282 "Arvados cluster controller daemon"
283 package_go_binary cmd/arvados-server arvados-dispatch-cloud \
284 "Arvados cluster cloud dispatch"
285 package_go_binary cmd/arvados-server arvados-dispatch-lsf \
286 "Dispatch Arvados containers to an LSF cluster"
287 package_go_binary services/arv-git-httpd arvados-git-httpd \
288 "Provide authenticated http access to Arvados-hosted git repositories"
289 package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
290 "Dispatch Crunch containers on the local system"
291 package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
292 "Dispatch Crunch containers to a SLURM cluster"
293 package_go_binary cmd/arvados-server crunch-run \
294 "Supervise a single Crunch container"
295 package_go_binary services/crunchstat crunchstat \
296 "Gather cpu/memory/network statistics of running Crunch jobs"
297 package_go_binary services/health arvados-health \
298 "Check health of all Arvados cluster services"
299 package_go_binary services/keep-balance keep-balance \
300 "Rebalance and garbage-collect data blocks stored in Arvados Keep"
301 package_go_binary services/keepproxy keepproxy \
302 "Make a Keep cluster accessible to clients that are not on the LAN"
303 package_go_binary cmd/arvados-server keepstore \
304 "Keep storage daemon, accessible to clients on the LAN"
305 package_go_binary services/keep-web keep-web \
306 "Static web hosting service for user data stored in Arvados Keep"
307 package_go_binary cmd/arvados-server arvados-ws \
308 "Arvados Websocket server"
309 package_go_binary tools/sync-groups arvados-sync-groups \
310 "Synchronize remote groups into Arvados from an external source"
311 package_go_binary tools/keep-block-check keep-block-check \
312 "Verify that all data from one set of Keep servers to another was copied"
313 package_go_binary tools/keep-rsync keep-rsync \
314 "Copy all data from one set of Keep servers to another"
315 package_go_binary tools/keep-exercise keep-exercise \
316 "Performance testing tool for Arvados Keep"
317 package_go_so lib/pam pam_arvados.so libpam-arvados-go \
318 "Arvados PAM authentication module"
320 # The Python SDK - Python3 package
321 fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
323 # Arvados cwl runner - Python3 package
324 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
326 # The FUSE driver - Python3 package
327 fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
329 # The Arvados crunchstat-summary tool
330 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
332 # The Docker image cleaner
333 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
335 # The Arvados user activity tool
336 fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
338 # The python->python3 metapackages
339 build_metapackage "arvados-fuse" "services/fuse"
340 build_metapackage "arvados-python-client" "services/fuse"
341 build_metapackage "arvados-cwl-runner" "sdk/cwl"
342 build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
343 build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
344 build_metapackage "arvados-user-activity" "tools/user-activity"
346 # The cwltest package, which lives out of tree
348 if [[ -e "$WORKSPACE/cwltest" ]]; then
349 rm -rf "$WORKSPACE/cwltest"
351 git clone https://github.com/common-workflow-language/cwltest.git
352 # signal to our build script that we want a cwltest executable installed in /usr/bin/
353 mkdir cwltest/bin && touch cwltest/bin/cwltest
354 fpm_build_virtualenv "cwltest" "cwltest" "python3"
355 # The python->python3 metapackage
356 build_metapackage "cwltest" "cwltest"
358 rm -rf "$WORKSPACE/cwltest"
360 calculate_go_package_version arvados_server_version cmd/arvados-server
361 arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
363 # Build the API server package
364 test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
365 if [[ "$?" == "0" ]]; then
366 handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
367 "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
368 --description="Arvados API server - Arvados is a free and open source platform for big data science." \
369 --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
372 # Build the workbench server package
373 test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
374 if [[ "$?" == "0" ]] ; then
378 # The workbench package has a build-time dependency on the arvados-server
379 # package for config manipulation, so install it first.
380 cd $WORKSPACE/cmd/arvados-server
381 get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
383 arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
384 if [[ ! -e ${arvados_server_pkg_path} ]]; then
385 arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
387 if [[ "$FORMAT" == "deb" ]]; then
388 dpkg -i ${arvados_server_pkg_path}
390 rpm -i ${arvados_server_pkg_path}
393 cd "$WORKSPACE/apps/workbench"
395 # We need to bundle to be ready even when we build a package without vendor directory
396 # because asset compilation requires it.
397 bundle install --system >"$STDOUT_IF_DEBUG"
399 # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
400 # and we want that in the package, so it's easier to not exclude the tmp directory
401 # from the package - empty it instead.
405 # Set up an appropriate config.yml
406 arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
407 mkdir -p /etc/arvados/
408 mv /tmp/x /etc/arvados/config.yml
409 perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
411 ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
412 ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
414 # Remove generated configuration files so they don't go in the package.
418 if [[ "$?" != "0" ]]; then
419 echo "ERROR: Asset precompilation failed"
422 handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
423 "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
424 --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
425 --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
429 # clean up temporary GOPATH