build/run-build-packages-one-target.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 read -rd "\000" helpmessage <<EOF
   7 $(basename $0): Orchestrate run-build-packages.sh for one target
   8
   9 Syntax:
  10         WORKSPACE=/path/to/arvados $(basename $0) [options]
  11
  12 --target <target>
  13     Distribution to build packages for (default: debian8)
  14 --command
  15     Build command to execute (default: use built-in Docker image command)
  16 --test-packages
  17     Run package install test script "test-packages-[target].sh"
  18 --debug
  19     Output debug information (default: false)
  20 --only-build <package>
  21     Build only a specific package
  22 --only-test <package>
  23     Test only a specific package
  24
  25 WORKSPACE=path         Path to the Arvados source tree to build packages from
  26
  27 EOF
  28
  29 set -e
  30
  31 if ! [[ -n "$WORKSPACE" ]]; then
  32   echo >&2 "$helpmessage"
  33   echo >&2
  34   echo >&2 "Error: WORKSPACE environment variable not set"
  35   echo >&2
  36   exit 1
  37 fi
  38
  39 if ! [[ -d "$WORKSPACE" ]]; then
  40   echo >&2 "$helpmessage"
  41   echo >&2
  42   echo >&2 "Error: $WORKSPACE is not a directory"
  43   echo >&2
  44   exit 1
  45 fi
  46
  47 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  48     help,debug,test-packages,target:,command:,only-test:,only-build: \
  49     -- "" "$@")
  50 if [ $? -ne 0 ]; then
  51     exit 1
  52 fi
  53
  54 TARGET=debian8
  55 COMMAND=
  56 DEBUG=
  57
  58 eval set -- "$PARSEDOPTS"
  59 while [ $# -gt 0 ]; do
  60     case "$1" in
  61         --help)
  62             echo >&2 "$helpmessage"
  63             echo >&2
  64             exit 1
  65             ;;
  66         --target)
  67             TARGET="$2"; shift
  68             ;;
  69         --only-test)
  70             test_packages=1
  71             packages="$2"; shift
  72             ;;
  73         --only-build)
  74             ONLY_BUILD="$2"; shift
  75             ;;
  76         --debug)
  77             DEBUG=" --debug"
  78             ARVADOS_DEBUG="1"
  79             ;;
  80         --command)
  81             COMMAND="$2"; shift
  82             ;;
  83         --test-packages)
  84             test_packages=1
  85             ;;
  86         --)
  87             if [ $# -gt 1 ]; then
  88                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  89                 exit 1
  90             fi
  91             ;;
  92     esac
  93     shift
  94 done
  95
  96 set -e
  97
  98 if [[ -n "$test_packages" ]]; then
  99     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
 100         createrepo $WORKSPACE/packages/$TARGET
 101     fi
 102
 103     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
 104         (cd $WORKSPACE/packages/$TARGET
 105           dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
 106           apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
 107         )
 108     fi
 109
 110     COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
 111     IMAGE="arvados/package-test:$TARGET"
 112 else
 113     IMAGE="arvados/build:$TARGET"
 114     if [[ "$COMMAND" != "" ]]; then
 115         COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 116     fi
 117 fi
 118
 119 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 120
 121 if [[ -n "$test_packages" ]]; then
 122     pushd "$JENKINS_DIR/package-test-dockerfiles"
 123 else
 124     pushd "$JENKINS_DIR/package-build-dockerfiles"
 125     make "$TARGET/generated"
 126 fi
 127
 128 echo $TARGET
 129 cd $TARGET
 130 time docker build --tag=$IMAGE .
 131 popd
 132
 133 if test -z "$packages" ; then
 134     packages="arvados-api-server
 135         arvados-docker-cleaner
 136         arvados-git-httpd
 137         arvados-node-manager
 138         arvados-src
 139         arvados-workbench
 140         crunch-dispatch-local
 141         crunch-dispatch-slurm
 142         crunch-run
 143         crunchstat
 144         keep-balance
 145         keep-block-check
 146         keepproxy
 147         keep-rsync
 148         keepstore
 149         keep-web
 150         libarvados-perl"
 151
 152     case "$TARGET" in
 153         *)
 154             packages="$packages python-arvados-fuse
 155                   python-arvados-python-client python-arvados-cwl-runner"
 156             ;;
 157     esac
 158 fi
 159
 160 FINAL_EXITCODE=0
 161
 162 package_fails=""
 163
 164 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
 165 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
 166
 167 docker_volume_args=(
 168     -v "$JENKINS_DIR:/jenkins"
 169     -v "$WORKSPACE:/arvados"
 170     -v /arvados/services/api/vendor/bundle
 171     -v /arvados/apps/workbench/vendor/bundle
 172     -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
 173     -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
 174 )
 175
 176 if [[ -n "$test_packages" ]]; then
 177     for p in $packages ; do
 178         if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then
 179             continue
 180         fi
 181         echo
 182         echo "START: $p test on $IMAGE" >&2
 183         if docker run --rm \
 184             "${docker_volume_args[@]}" \
 185             --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 186             --env "TARGET=$TARGET" \
 187             --env "WORKSPACE=/arvados" \
 188             "$IMAGE" $COMMAND $p
 189         then
 190             echo "OK: $p test on $IMAGE succeeded" >&2
 191         else
 192             FINAL_EXITCODE=$?
 193             package_fails="$package_fails $p"
 194             echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 195         fi
 196     done
 197 else
 198     echo
 199     echo "START: build packages on $IMAGE" >&2
 200     # Move existing packages and other files into the processed/ subdirectory
 201     if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then
 202       mkdir -p "${WORKSPACE}/packages/${TARGET}/processed"
 203     fi
 204     set +e
 205     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
 206     set -e
 207     # Build packages
 208     if docker run --rm \
 209         "${docker_volume_args[@]}" \
 210         --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 211         --env "ONLY_BUILD=$ONLY_BUILD" \
 212         "$IMAGE" $COMMAND
 213     then
 214         echo
 215         echo "OK: build packages on $IMAGE succeeded" >&2
 216     else
 217         FINAL_EXITCODE=$?
 218         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 219     fi
 220 fi
 221
 222 if test -n "$package_fails" ; then
 223     echo "Failed package tests:$package_fails" >&2
 224 fi
 225
 226 exit $FINAL_EXITCODE