2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 read -rd "\000" helpmessage <<EOF
7 $(basename $0): Orchestrate run-build-packages.sh for one target
10 WORKSPACE=/path/to/arvados $(basename $0) [options]
13 Distribution to build packages for (default: debian10)
15 Build command to execute (default: use built-in Docker image command)
17 Run package install test script "test-packages-[target].sh"
19 Output debug information (default: false)
20 --only-build <package>
21 Build only a specific package
23 Test only a specific package
25 Build a specific architecture (amd64 or arm64, defaults to native architecture)
27 Build even if the package exists upstream or if it has already been
30 Test even if there is no new untested package
31 --build-version <string>
32 Version to build (default:
33 \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
34 0.1.timestamp.commithash)
36 WORKSPACE=path Path to the Arvados source tree to build packages from
42 if ! [[ -n "$WORKSPACE" ]]; then
43 echo >&2 "$helpmessage"
45 echo >&2 "Error: WORKSPACE environment variable not set"
50 if ! [[ -d "$WORKSPACE" ]]; then
51 echo >&2 "$helpmessage"
53 echo >&2 "Error: $WORKSPACE is not a directory"
58 PARSEDOPTS=$(getopt --name "$0" --longoptions \
59 help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version: \
70 eval set -- "$PARSEDOPTS"
71 while [ $# -gt 0 ]; do
74 echo >&2 "$helpmessage"
93 ONLY_BUILD="$2"; shift
109 if [[ -z "$2" ]]; then
111 elif ! [[ "$2" =~ (.*)-(.*) ]]; then
112 echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
114 elif ! [[ "$2" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$ ]]; then
115 echo >&2 "FATAL: --build-version '$2' is invalid, must match pattern ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$"
118 [[ "$2" =~ (.*)-(.*) ]]
119 ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
120 ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
125 if [ $# -gt 1 ]; then
126 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
136 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
137 echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
140 if [[ -n "$test_packages" ]]; then
141 if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
142 CREATEREPO="$(command -v createrepo createrepo_c | tail -n1)"
143 if [[ -z "$CREATEREPO" ]]; then
145 echo >&2 "Error: please install createrepo. E.g. sudo apt install createrepo-c"
149 "$CREATEREPO" $WORKSPACE/packages/$TARGET
152 if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
154 /usr/bin/which dpkg-scanpackages >/dev/null
155 if [[ "$?" != "0" ]]; then
157 echo >&2 "Error: please install dpkg-dev. E.g. sudo apt-get install dpkg-dev"
161 /usr/bin/which apt-ftparchive >/dev/null
162 if [[ "$?" != "0" ]]; then
164 echo >&2 "Error: please install apt-utils. E.g. sudo apt-get install apt-utils"
169 (cd $WORKSPACE/packages/$TARGET
170 dpkg-scanpackages --multiversion . 2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
171 apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
175 COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
176 IMAGE="arvados/package-test:$TARGET"
178 IMAGE="arvados/build:$TARGET"
179 if [[ "$COMMAND" != "" ]]; then
180 COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
184 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
186 if [[ -n "$test_packages" ]]; then
187 pushd "$JENKINS_DIR/package-test-dockerfiles"
188 make "$TARGET/generated"
190 pushd "$JENKINS_DIR/package-build-dockerfiles"
191 make "$TARGET/generated"
194 GOVERSION=$(grep 'const goversion =' $WORKSPACE/lib/install/deps.go |awk -F'"' '{print $2}')
198 time docker build --tag "$IMAGE" \
199 --build-arg HOSTTYPE=$HOSTTYPE \
200 --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) \
201 --build-arg GOVERSION=$GOVERSION --no-cache .
204 if test -z "$packages" ; then
205 packages="arvados-api-server
208 arvados-dispatch-cloud
210 arvados-docker-cleaner
220 crunch-dispatch-local
221 crunch-dispatch-slurm
236 python3-arvados-python-client
237 python3-arvados-cwl-runner
238 python3-crunchstat-summary
239 python3-arvados-user-activity"
246 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
247 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
250 -v "$JENKINS_DIR:/jenkins"
251 -v "$WORKSPACE:/arvados"
252 -v /arvados/services/api/vendor/bundle
253 -v /arvados/apps/workbench/vendor/bundle
254 -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
255 -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
258 if [[ -n "$test_packages" ]]; then
259 for p in $packages ; do
260 if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then
263 if [[ -e "${WORKSPACE}/packages/.last_test_${TARGET}" ]] && [[ -z "$FORCE_TEST" ]]; then
264 MATCH=`find ${WORKSPACE}/packages/ -newer ${WORKSPACE}/packages/.last_test_${TARGET} -regex .*${TARGET}/$p.*`
265 if [[ "$MATCH" == "" ]]; then
266 # No new package has been built that needs testing
267 echo "Skipping $p test because no new package was built since the last test."
271 # If we're testing all packages, we should not error out on packages that don't exist.
272 # If we are testing one specific package only (i.e. --only-test was given), we should
273 # error out if that package does not exist.
274 if [[ -z "$testing_one_package" ]]; then
275 MATCH=`find ${WORKSPACE}/packages/ -regextype posix-extended -regex .*${TARGET}/$p.*\\(deb\\|rpm\\)`
276 if [[ "$MATCH" == "" ]]; then
277 # No new package has been built that needs testing
278 echo "Skipping $p test because no package file is available to test."
283 echo "START: $p test on $IMAGE" >&2
286 "${docker_volume_args[@]}" \
287 --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
288 --env "TARGET=$TARGET" \
289 --env "WORKSPACE=/arvados" \
292 echo "OK: $p test on $IMAGE succeeded" >&2
295 package_fails="$package_fails $p"
296 echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
300 if [[ "$FINAL_EXITCODE" == "0" ]]; then
301 touch ${WORKSPACE}/packages/.last_test_${TARGET}
305 echo "START: build packages on $IMAGE" >&2
306 # Move existing packages and other files into the processed/ subdirectory
307 if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then
308 mkdir -p "${WORKSPACE}/packages/${TARGET}/processed"
311 mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
313 # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
314 # docker run command below.
315 # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
316 tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
317 cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
318 printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
322 "${docker_volume_args[@]}" \
323 -v $tmpfile:/root/.bundle/config \
324 --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
325 --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
326 --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
327 --env "ONLY_BUILD=$ONLY_BUILD" \
328 --env "FORCE_BUILD=$FORCE_BUILD" \
330 --env "MAKE=make --jobs $cores" \
334 echo "OK: build packages on $IMAGE succeeded" >&2
337 echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
339 # Clean up the bundle config file
343 if test -n "$package_fails" ; then
344 echo "Failed package tests:$package_fails" >&2