2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 read -rd "\000" helpmessage <<EOF
7 $(basename $0): Orchestrate run-build-packages.sh for one target
10 WORKSPACE=/path/to/arvados $(basename $0) [options]
13 Distribution to build packages for (default: debian10)
15 Build command to execute (default: use built-in Docker image command)
17 Run package install test script "test-packages-[target].sh"
19 Output debug information (default: false)
20 --only-build <package>
21 Build only a specific package
23 Test only a specific package
25 Build a specific architecture (amd64 or arm64, defaults to native architecture)
27 Build even if the package exists upstream or if it has already been
30 Test even if there is no new untested package
31 --build-version <string>
32 Version to build (default:
33 \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
34 0.1.timestamp.commithash)
36 WORKSPACE=path Path to the Arvados source tree to build packages from
42 if ! [[ -n "$WORKSPACE" ]]; then
43 echo >&2 "$helpmessage"
45 echo >&2 "Error: WORKSPACE environment variable not set"
50 if ! [[ -d "$WORKSPACE" ]]; then
51 echo >&2 "$helpmessage"
53 echo >&2 "Error: $WORKSPACE is not a directory"
58 PARSEDOPTS=$(getopt --name "$0" --longoptions \
59 help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version: \
71 eval set -- "$PARSEDOPTS"
72 while [ $# -gt 0 ]; do
75 echo >&2 "$helpmessage"
94 ONLY_BUILD="$2"; shift
110 if [[ -z "$2" ]]; then
112 elif ! [[ "$2" =~ (.*)-(.*) ]]; then
113 echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
115 elif ! [[ "$2" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$ ]]; then
116 echo >&2 "FATAL: --build-version '$2' is invalid, must match pattern ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$"
119 [[ "$2" =~ (.*)-(.*) ]]
120 ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
121 ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
126 if [ $# -gt 1 ]; then
127 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
137 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
138 echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
141 if [[ -n "$test_packages" ]]; then
142 if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
144 /usr/bin/which createrepo >/dev/null
145 if [[ "$?" != "0" ]]; then
147 echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo"
152 createrepo $WORKSPACE/packages/$TARGET
155 if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
157 /usr/bin/which dpkg-scanpackages >/dev/null
158 if [[ "$?" != "0" ]]; then
160 echo >&2 "Error: please install dpkg-dev. E.g. sudo apt-get install dpkg-dev"
164 /usr/bin/which apt-ftparchive >/dev/null
165 if [[ "$?" != "0" ]]; then
167 echo >&2 "Error: please install apt-utils. E.g. sudo apt-get install apt-utils"
172 (cd $WORKSPACE/packages/$TARGET
173 dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
174 apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
178 COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
179 IMAGE="arvados/package-test:$TARGET"
181 IMAGE="arvados/build:$TARGET"
182 if [[ "$COMMAND" != "" ]]; then
183 COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
187 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
189 if [[ -n "$test_packages" ]]; then
190 pushd "$JENKINS_DIR/package-test-dockerfiles"
191 make "$TARGET/generated"
193 pushd "$JENKINS_DIR/package-build-dockerfiles"
194 make "$TARGET/generated"
199 time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
202 if test -z "$packages" ; then
203 packages="arvados-api-server
206 arvados-dispatch-cloud
208 arvados-docker-cleaner
217 crunch-dispatch-local
218 crunch-dispatch-slurm
234 python3-arvados-python-client
235 python3-arvados-cwl-runner
236 python3-crunchstat-summary
237 python3-arvados-user-activity"
244 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
245 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
248 -v "$JENKINS_DIR:/jenkins"
249 -v "$WORKSPACE:/arvados"
250 -v /arvados/services/api/vendor/bundle
251 -v /arvados/apps/workbench/vendor/bundle
252 -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
253 -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
256 if [[ -n "$test_packages" ]]; then
257 for p in $packages ; do
258 if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then
261 if [[ -e "${WORKSPACE}/packages/.last_test_${TARGET}" ]] && [[ -z "$FORCE_TEST" ]]; then
262 MATCH=`find ${WORKSPACE}/packages/ -newer ${WORKSPACE}/packages/.last_test_${TARGET} -regex .*${TARGET}/$p.*`
263 if [[ "$MATCH" == "" ]]; then
264 # No new package has been built that needs testing
265 echo "Skipping $p test because no new package was built since the last test."
269 # If we're testing all packages, we should not error out on packages that don't exist.
270 # If we are testing one specific package only (i.e. --only-test was given), we should
271 # error out if that package does not exist.
272 if [[ -z "$testing_one_package" ]]; then
273 MATCH=`find ${WORKSPACE}/packages/ -regextype posix-extended -regex .*${TARGET}/$p.*\\(deb\\|rpm\\)`
274 if [[ "$MATCH" == "" ]]; then
275 # No new package has been built that needs testing
276 echo "Skipping $p test because no package file is available to test."
281 echo "START: $p test on $IMAGE" >&2
284 "${docker_volume_args[@]}" \
285 --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
286 --env "TARGET=$TARGET" \
287 --env "WORKSPACE=/arvados" \
290 echo "OK: $p test on $IMAGE succeeded" >&2
293 package_fails="$package_fails $p"
294 echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
298 if [[ "$FINAL_EXITCODE" == "0" ]]; then
299 touch ${WORKSPACE}/packages/.last_test_${TARGET}
303 echo "START: build packages on $IMAGE" >&2
304 # Move existing packages and other files into the processed/ subdirectory
305 if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then
306 mkdir -p "${WORKSPACE}/packages/${TARGET}/processed"
309 mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
314 "${docker_volume_args[@]}" \
315 --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
316 --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
317 --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
318 --env "ONLY_BUILD=$ONLY_BUILD" \
319 --env "FORCE_BUILD=$FORCE_BUILD" \
324 echo "OK: build packages on $IMAGE succeeded" >&2
327 echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
331 if test -n "$package_fails" ; then
332 echo "Failed package tests:$package_fails" >&2