build/run-build-packages-one-target.sh

   1 #!/bin/bash
   2
   3 read -rd "\000" helpmessage <<EOF
   4 $(basename $0): Orchestrate run-build-packages.sh for one target
   5
   6 Syntax:
   7         WORKSPACE=/path/to/arvados $(basename $0) [options]
   8
   9 --target <target>
  10     Distribution to build packages for (default: debian8)
  11 --command
  12     Build command to execute (default: use built-in Docker image command)
  13 --test-packages
  14     Run package install test script "test-packages-[target].sh"
  15 --debug
  16     Output debug information (default: false)
  17 --only-build <package>
  18     Build only a specific package
  19 --only-test <package>
  20     Test only a specific package
  21
  22 WORKSPACE=path         Path to the Arvados source tree to build packages from
  23
  24 EOF
  25
  26 set -e
  27
  28 if ! [[ -n "$WORKSPACE" ]]; then
  29   echo >&2 "$helpmessage"
  30   echo >&2
  31   echo >&2 "Error: WORKSPACE environment variable not set"
  32   echo >&2
  33   exit 1
  34 fi
  35
  36 if ! [[ -d "$WORKSPACE" ]]; then
  37   echo >&2 "$helpmessage"
  38   echo >&2
  39   echo >&2 "Error: $WORKSPACE is not a directory"
  40   echo >&2
  41   exit 1
  42 fi
  43
  44 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  45     help,debug,test-packages,target:,command:,only-test:,only-build: \
  46     -- "" "$@")
  47 if [ $? -ne 0 ]; then
  48     exit 1
  49 fi
  50
  51 TARGET=debian8
  52 COMMAND=
  53 DEBUG=
  54
  55 eval set -- "$PARSEDOPTS"
  56 while [ $# -gt 0 ]; do
  57     case "$1" in
  58         --help)
  59             echo >&2 "$helpmessage"
  60             echo >&2
  61             exit 1
  62             ;;
  63         --target)
  64             TARGET="$2"; shift
  65             ;;
  66         --only-test)
  67             test_packages=1
  68             packages="$2"; shift
  69             ;;
  70         --only-build)
  71             ONLY_BUILD="$2"; shift
  72             ;;
  73         --debug)
  74             DEBUG=" --debug"
  75             ARVADOS_DEBUG="1"
  76             ;;
  77         --command)
  78             COMMAND="$2"; shift
  79             ;;
  80         --test-packages)
  81             test_packages=1
  82             ;;
  83         --)
  84             if [ $# -gt 1 ]; then
  85                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  86                 exit 1
  87             fi
  88             ;;
  89     esac
  90     shift
  91 done
  92
  93 set -e
  94
  95 if [[ -n "$test_packages" ]]; then
  96     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
  97         createrepo $WORKSPACE/packages/$TARGET
  98     fi
  99
 100     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
 101         (cd $WORKSPACE/packages/$TARGET
 102           dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
 103           apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
 104         )
 105     fi
 106
 107     COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
 108     IMAGE="arvados/package-test:$TARGET"
 109 else
 110     IMAGE="arvados/build:$TARGET"
 111     if [[ "$COMMAND" != "" ]]; then
 112         COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 113     fi
 114 fi
 115
 116 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 117
 118 if [[ -n "$test_packages" ]]; then
 119     pushd "$JENKINS_DIR/package-test-dockerfiles"
 120 else
 121     pushd "$JENKINS_DIR/package-build-dockerfiles"
 122     make "$TARGET/generated"
 123 fi
 124
 125 echo $TARGET
 126 cd $TARGET
 127 time docker build --tag=$IMAGE .
 128 popd
 129
 130 if test -z "$packages" ; then
 131     packages="arvados-api-server
 132         arvados-docker-cleaner
 133         arvados-git-httpd
 134         arvados-node-manager
 135         arvados-src
 136         arvados-workbench
 137         crunch-dispatch-local
 138         crunch-dispatch-slurm
 139         crunch-run
 140         crunchstat
 141         keep-balance
 142         keep-block-check
 143         keepproxy
 144         keep-rsync
 145         keepstore
 146         keep-web
 147         libarvados-perl"
 148
 149     case "$TARGET" in
 150         *)
 151             packages="$packages python-arvados-fuse
 152                   python-arvados-python-client python-arvados-cwl-runner"
 153             ;;
 154     esac
 155 fi
 156
 157 FINAL_EXITCODE=0
 158
 159 package_fails=""
 160
 161 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
 162 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
 163
 164 docker_volume_args=(
 165     -v "$JENKINS_DIR:/jenkins"
 166     -v "$WORKSPACE:/arvados"
 167     -v /arvados/services/api/vendor/bundle
 168     -v /arvados/apps/workbench/vendor/bundle
 169     -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
 170     -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
 171 )
 172
 173 if [[ -n "$test_packages" ]]; then
 174     for p in $packages ; do
 175         if [[ -n "$ONLY_BUILD" ]] && [[ "$p" != "$ONLY_BUILD" ]]; then
 176             continue
 177         fi
 178         echo
 179         echo "START: $p test on $IMAGE" >&2
 180         if docker run --rm \
 181             "${docker_volume_args[@]}" \
 182             --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 183             --env "TARGET=$TARGET" \
 184             --env "WORKSPACE=/arvados" \
 185             "$IMAGE" $COMMAND $p
 186         then
 187             echo "OK: $p test on $IMAGE succeeded" >&2
 188         else
 189             FINAL_EXITCODE=$?
 190             package_fails="$package_fails $p"
 191             echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 192         fi
 193     done
 194 else
 195     echo
 196     echo "START: build packages on $IMAGE" >&2
 197     # Move existing packages and other files into the processed/ subdirectory
 198     if [[ ! -e "${WORKSPACE}/packages/${TARGET}/processed" ]]; then
 199       mkdir -p "${WORKSPACE}/packages/${TARGET}/processed"
 200     fi
 201     set +e
 202     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
 203     set -e
 204     # Build packages
 205     if docker run --rm \
 206         "${docker_volume_args[@]}" \
 207         --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
 208         --env "ONLY_BUILD=$ONLY_BUILD" \
 209         "$IMAGE" $COMMAND
 210     then
 211         echo
 212         echo "OK: build packages on $IMAGE succeeded" >&2
 213     else
 214         FINAL_EXITCODE=$?
 215         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 216     fi
 217 fi
 218
 219 if test -n "$package_fails" ; then
 220     echo "Failed package tests:$package_fails" >&2
 221 fi
 222
 223 exit $FINAL_EXITCODE