build/run-build-packages-one-target.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 read -rd "\000" helpmessage <<EOF
   7 $(basename $0): Orchestrate run-build-packages.sh for one target
   8
   9 Syntax:
  10         WORKSPACE=/path/to/arvados $(basename $0) --target <target> [options]
  11
  12 --target <target>
  13     Distribution to build packages for
  14 --command
  15     Build command to execute (default: use built-in Docker image command)
  16 --test-packages
  17     Run package install test script "test-packages-[target].sh"
  18 --debug
  19     Output debug information (default: false)
  20 --only-build <package>
  21     Build only a specific package
  22 --only-test <package>
  23     Test only a specific package
  24 --force-build
  25     Build even if the package exists upstream or if it has already been
  26     built locally
  27 --force-test
  28     Test even if there is no new untested package
  29 --build-version <string>
  30     Version to build (default:
  31     \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
  32     0.1.timestamp.commithash)
  33 --skip-docker-build
  34     Don't try to build Docker images
  35
  36 WORKSPACE=path         Path to the Arvados source tree to build packages from
  37
  38 EOF
  39
  40 set -e
  41
  42 if ! [[ -n "$WORKSPACE" ]]; then
  43   echo >&2 "$helpmessage"
  44   echo >&2
  45   echo >&2 "Error: WORKSPACE environment variable not set"
  46   echo >&2
  47   exit 1
  48 fi
  49
  50 if ! [[ -d "$WORKSPACE" ]]; then
  51   echo >&2 "$helpmessage"
  52   echo >&2
  53   echo >&2 "Error: $WORKSPACE is not a directory"
  54   echo >&2
  55   exit 1
  56 fi
  57
  58 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  59     help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version:,skip-docker-build \
  60     -- "" "$@")
  61 if [ $? -ne 0 ]; then
  62     exit 1
  63 fi
  64
  65 FORCE_BUILD=0
  66 COMMAND=run-build-packages.sh
  67 DEBUG=
  68 TARGET=
  69
  70 eval set -- "$PARSEDOPTS"
  71 while [ $# -gt 0 ]; do
  72     case "$1" in
  73         --help)
  74             echo >&2 "$helpmessage"
  75             echo >&2
  76             exit 1
  77             ;;
  78         --target)
  79             TARGET="$2"; shift
  80             ;;
  81         --only-test)
  82             test_packages=1
  83             testing_one_package=1
  84             packages="$2"; shift
  85             ;;
  86         --force-test)
  87             FORCE_TEST=true
  88             ;;
  89         --force-build)
  90             FORCE_BUILD=1
  91             ;;
  92         --only-build)
  93             ONLY_BUILD="$2"; shift
  94             ;;
  95         --arch)
  96             case "$2" in
  97                 amd64) ;;
  98                 *)
  99                     printf "FATAL: --arch '%s' is not supported" "$2" >&2
 100                     exit 2
 101                     ;;
 102             esac
 103             ARCH="$2"; shift
 104             ;;
 105         --debug)
 106             DEBUG=" --debug"
 107             ARVADOS_DEBUG="1"
 108             ;;
 109         --command)
 110             COMMAND="$2"; shift
 111             ;;
 112         --test-packages)
 113             test_packages=1
 114             ;;
 115         --build-version)
 116             if [[ -z "$2" ]]; then
 117                 :
 118             elif ! [[ "$2" =~ (.*)-(.*) ]]; then
 119                 echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
 120                 exit 1
 121             elif ! [[ "$2" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$ ]]; then
 122                 echo >&2 "FATAL: --build-version '$2' is invalid, must match pattern ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$"
 123                 exit 1
 124             else
 125                 [[ "$2" =~ (.*)-(.*) ]]
 126                 ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
 127                 ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
 128             fi
 129             shift
 130             ;;
 131         --skip-docker-build)
 132             SKIP_DOCKER_BUILD=1
 133             ;;
 134         --)
 135             if [ $# -gt 1 ]; then
 136                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
 137                 exit 1
 138             fi
 139             ;;
 140     esac
 141     shift
 142 done
 143
 144 set -e
 145 orig_umask="$(umask)"
 146
 147 if [[ -z "$TARGET" ]]; then
 148     echo "FATAL: --target must be specified" >&2
 149     exit 2
 150 elif [[ ! -e "$WORKSPACE/build/package-testing/test-packages-$TARGET.sh" ]]; then
 151     echo "FATAL: unknown build target '$TARGET'" >&2
 152     exit 2
 153 fi
 154
 155 if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
 156     echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
 157 fi
 158
 159 if [[ -n "$test_packages" ]]; then
 160   # Packages are built world-readable, so package indexes should be too,
 161   # especially because since 2022 apt uses an unprivileged user `_apt` to
 162   # retrieve everything.  Ensure it has permissions to read the packages
 163   # when mounted as a volume inside the Docker container.
 164   chmod a+rx "$WORKSPACE" "$WORKSPACE/packages" "$WORKSPACE/packages/$TARGET"
 165   umask 022
 166   if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
 167     CREATEREPO="$(command -v createrepo createrepo_c | tail -n1)"
 168     if [[ -z "$CREATEREPO" ]]; then
 169       echo >&2
 170       echo >&2 "Error: please install createrepo. E.g. sudo apt install createrepo-c"
 171       echo >&2
 172       exit 1
 173     fi
 174     "$CREATEREPO" $WORKSPACE/packages/$TARGET
 175   fi
 176
 177   if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
 178     set +e
 179     /usr/bin/which dpkg-scanpackages >/dev/null
 180     if [[ "$?" != "0" ]]; then
 181       echo >&2
 182       echo >&2 "Error: please install dpkg-dev. E.g. sudo apt-get install dpkg-dev"
 183       echo >&2
 184       exit 1
 185     fi
 186     /usr/bin/which apt-ftparchive >/dev/null
 187     if [[ "$?" != "0" ]]; then
 188       echo >&2
 189       echo >&2 "Error: please install apt-utils. E.g. sudo apt-get install apt-utils"
 190       echo >&2
 191       exit 1
 192     fi
 193     set -e
 194     (cd $WORKSPACE/packages/$TARGET
 195       dpkg-scanpackages --multiversion .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
 196       apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
 197     )
 198   fi
 199
 200   COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
 201   IMAGE="arvados/package-test:$TARGET"
 202   umask "$orig_umask"
 203 else
 204   IMAGE="arvados/build:$TARGET"
 205   COMMAND="bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 206 fi
 207
 208 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 209
 210 if [[ "$SKIP_DOCKER_BUILD" != 1 ]] ; then
 211   env -C "$WORKSPACE/tools/ansible" ansible-galaxy install -r requirements.yml
 212   declare -a ansible_opts=()
 213   if [[ -n "$test_packages" ]]; then
 214       ansible_opts+=(
 215           --extra-vars=arvados_build_playbook=setup-package-tests.yml
 216           --limit="arvados_pkgtest_$TARGET"
 217       )
 218   else
 219       ansible_opts+=(
 220           --extra-vars=arvados_build_playbook=install-dev-tools.yml
 221           --limit="arvados_pkgbuild_$TARGET"
 222       )
 223   fi
 224   env -C "$WORKSPACE/tools/ansible" ansible-playbook \
 225       --inventory=files/development-docker-images.yml \
 226       "${ansible_opts[@]}" build-docker-image.yml
 227   unset ansible_opts
 228 fi
 229
 230 if test -z "$packages" ; then
 231     packages="arvados-api-server
 232         arvados-client
 233         arvados-controller
 234         arvados-dispatch-cloud
 235         arvados-dispatch-lsf
 236         arvados-docker-cleaner
 237         arvados-health
 238         arvados-server
 239         arvados-src
 240         arvados-sync-groups
 241         arvados-sync-users
 242         arvados-workbench2
 243         arvados-ws
 244         crunch-dispatch-local
 245         crunch-dispatch-slurm
 246         crunch-run
 247         keep-balance
 248         keep-block-check
 249         keep-exercise
 250         keep-rsync
 251         keep-web
 252         keepproxy
 253         keepstore
 254         libpam-arvados-go
 255         python3-arvados-cwl-runner
 256         python3-arvados-fuse
 257         python3-arvados-python-client
 258         python3-arvados-user-activity
 259         python3-arvados-cluster-activity
 260         python3-crunchstat-summary"
 261 fi
 262
 263 FINAL_EXITCODE=0
 264
 265 package_fails=""
 266
 267 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
 268
 269 docker_volume_args=(
 270     --mount "type=bind,src=$JENKINS_DIR,dst=/jenkins"
 271     --mount "type=bind,src=$WORKSPACE,dst=/arvados"
 272     --tmpfs /arvados/services/api/.bundle:rw,noexec,nosuid,size=1m
 273     --tmpfs /arvados/services/api/vendor:rw,exec,nosuid,size=1g
 274     --mount "type=bind,src=$WORKSPACE/services/api/vendor/cache-$TARGET,dst=/arvados/services/api/vendor/cache"
 275 )
 276
 277 if [[ -n "$test_packages" ]]; then
 278     for p in $packages ; do
 279         if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then
 280             continue
 281         fi
 282         if [[ -e "${WORKSPACE}/packages/.last_test_${TARGET}" ]] && [[ -z "$FORCE_TEST" ]]; then
 283           MATCH=`find ${WORKSPACE}/packages/ -newer ${WORKSPACE}/packages/.last_test_${TARGET} -regex .*${TARGET}/$p.*`
 284           if [[ "$MATCH" == "" ]]; then
 285             # No new package has been built that needs testing
 286             echo "Skipping $p test because no new package was built since the last test."
 287             continue
 288           fi
 289         fi
 290         # If we're testing all packages, we should not error out on packages that don't exist.
 291         # If we are testing one specific package only (i.e. --only-test was given), we should
 292         # error out if that package does not exist.
 293         if [[ -z "$testing_one_package" ]]; then
 294           MATCH=`find ${WORKSPACE}/packages/ -regextype posix-extended -regex .*${TARGET}/$p.*\\(deb\\|rpm\\)`
 295           if [[ "$MATCH" == "" ]]; then
 296             # No new package has been built that needs testing
 297             echo "Skipping $p test because no package file is available to test."
 298             continue
 299           fi
 300         fi
 301         echo
 302         echo "START: $p test on $IMAGE" >&2
 303         if docker run \
 304             --rm \
 305             "${docker_volume_args[@]}" \
 306             --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 307             --env "TARGET=$TARGET" \
 308             --env "WORKSPACE=/arvados" \
 309             "$IMAGE" $COMMAND $p
 310         then
 311             echo "OK: $p test on $IMAGE succeeded" >&2
 312         else
 313             FINAL_EXITCODE=$?
 314             package_fails="$package_fails $p"
 315             echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 316         fi
 317     done
 318
 319     if [[ "$FINAL_EXITCODE" == "0" ]]; then
 320       touch ${WORKSPACE}/packages/.last_test_${TARGET}
 321     fi
 322 else
 323     echo
 324     echo "START: build packages on $IMAGE" >&2
 325     # Move existing packages and other files into the processed/ subdirectory
 326     if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then
 327       mkdir -p "${WORKSPACE}/packages/${TARGET}/processed"
 328     fi
 329     set +e
 330     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
 331     set -e
 332     # give bundle (almost) all the cores. See also the MAKE env var that is passed into the
 333     # docker run command below.
 334     # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
 335     tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX)
 336     cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
 337     printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile
 338     # Build packages.
 339     if docker run \
 340         --rm \
 341         "${docker_volume_args[@]}" \
 342         --mount "type=bind,src=$tmpfile,dst=/root/.bundle/config" \
 343         --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
 344         --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
 345         --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 346         --env "ONLY_BUILD=$ONLY_BUILD" \
 347         --env "FORCE_BUILD=$FORCE_BUILD" \
 348         --env "ARCH=$ARCH" \
 349         --env "MAKE=make --jobs $cores" \
 350         "$IMAGE" $COMMAND
 351     then
 352         echo
 353         echo "OK: build packages on $IMAGE succeeded" >&2
 354     else
 355         FINAL_EXITCODE=$?
 356         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 357     fi
 358     # Clean up the bundle config file
 359     rm -f $tmpfile
 360 fi
 361
 362 if test -n "$package_fails" ; then
 363     echo "Failed package tests:$package_fails" >&2
 364 fi
 365
 366 exit $FINAL_EXITCODE