X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bfe7a62407aa0a6bc078659f259468f9b4105581..db09e5f787a1b2d0e02abb627bf25054a047f7db:/build/run-build-packages-one-target.sh diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index 958067c91c..46379e7b9a 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -1,4 +1,7 @@ #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 read -rd "\000" helpmessage < Build only a specific package --only-test Test only a specific package +--force-test + Test even if there is no new untested package +--build-version + Version to build (default: + \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or + 0.1.timestamp.commithash) WORKSPACE=path Path to the Arvados source tree to build packages from @@ -42,7 +51,7 @@ if ! [[ -d "$WORKSPACE" ]]; then fi PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,debug,test-packages,target:,command:,only-test:,only-build: \ + help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,build-version: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -67,11 +76,15 @@ while [ $# -gt 0 ]; do test_packages=1 packages="$2"; shift ;; + --force-test) + FORCE_TEST=true + ;; --only-build) ONLY_BUILD="$2"; shift ;; --debug) DEBUG=" --debug" + ARVADOS_DEBUG="1" ;; --command) COMMAND="$2"; shift @@ -79,6 +92,18 @@ while [ $# -gt 0 ]; do --test-packages) test_packages=1 ;; + --build-version) + if [[ -z "$2" ]]; then + : + elif ! [[ "$2" =~ (.*)-(.*) ]]; then + echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?" + exit 1 + else + ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}" + ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}" + fi + shift + ;; --) if [ $# -gt 1 ]; then echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help" @@ -91,14 +116,28 @@ done set -e +if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then + echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'" +fi + if [[ -n "$test_packages" ]]; then if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then + set +e + /usr/bin/which createrepo >/dev/null + if [[ "$?" != "0" ]]; then + echo >&2 + echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo" + echo >&2 + exit 1 + fi + set -e createrepo $WORKSPACE/packages/$TARGET fi if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then (cd $WORKSPACE/packages/$TARGET - dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz + dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz + apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release ) fi @@ -115,6 +154,7 @@ JENKINS_DIR=$(dirname "$(readlink -e "$0")") if [[ -n "$test_packages" ]]; then pushd "$JENKINS_DIR/package-test-dockerfiles" + make "$TARGET/generated" else pushd "$JENKINS_DIR/package-build-dockerfiles" make "$TARGET/generated" @@ -127,6 +167,7 @@ popd if test -z "$packages" ; then packages="arvados-api-server + arvados-client arvados-docker-cleaner arvados-git-httpd arvados-node-manager @@ -142,18 +183,10 @@ if test -z "$packages" ; then keep-rsync keepstore keep-web - libarvados-perl" - - case "$TARGET" in - centos6) - packages="$packages python27-python-arvados-fuse - python27-python-arvados-python-client python27-python-arvados-cwl-runner" - ;; - *) - packages="$packages python-arvados-fuse - python-arvados-python-client python-arvados-cwl-runner" - ;; - esac + libarvados-perl + python-arvados-fuse + python-arvados-python-client + python-arvados-cwl-runner" fi FINAL_EXITCODE=0 @@ -177,11 +210,21 @@ if [[ -n "$test_packages" ]]; then if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then continue fi + if [[ -e "${WORKSPACE}/packages/.last_test_${TARGET}" ]] && [[ -z "$FORCE_TEST" ]]; then + MATCH=`find ${WORKSPACE}/packages/ -newer ${WORKSPACE}/packages/.last_test_${TARGET} -regex .*${TARGET}/$p.*` + if [[ "$MATCH" == "" ]]; then + # No new package has been built that needs testing + echo "Skipping $p test because no new package was built since the last test." + continue + fi + fi echo echo "START: $p test on $IMAGE" >&2 - if docker run --rm \ + # ulimit option can be removed when debian8 and ubuntu1404 are retired + if docker run --ulimit nofile=4096:4096 \ + --rm \ "${docker_volume_args[@]}" \ - --env ARVADOS_DEBUG=1 \ + --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "TARGET=$TARGET" \ --env "WORKSPACE=/arvados" \ "$IMAGE" $COMMAND $p @@ -193,12 +236,25 @@ if [[ -n "$test_packages" ]]; then echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2 fi done + + touch ${WORKSPACE}/packages/.last_test_${TARGET} else echo echo "START: build packages on $IMAGE" >&2 - if docker run --rm \ + # Move existing packages and other files into the processed/ subdirectory + if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then + mkdir -p "${WORKSPACE}/packages/${TARGET}/processed" + fi + set +e + mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null + set -e + # Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired + if docker run --ulimit nofile=4096:4096 \ + --rm \ "${docker_volume_args[@]}" \ - --env ARVADOS_DEBUG=1 \ + --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \ + --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \ + --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "ONLY_BUILD=$ONLY_BUILD" \ "$IMAGE" $COMMAND then