build/run-build-packages-one-target.sh

   1 #!/bin/bash
   2
   3 read -rd "\000" helpmessage <<EOF
   4 $(basename $0): Orchestrate run-build-packages.sh for one target
   5
   6 Syntax:
   7         WORKSPACE=/path/to/arvados $(basename $0) [options]
   8
   9 --target <target>
  10     Distribution to build packages for (default: debian7)
  11 --command
  12     Build command to execute (default: use built-in Docker image command)
  13 --test-packages
  14     Run package install test script "test-packages-$target.sh"
  15 --debug
  16     Output debug information (default: false)
  17 --only-test
  18     Test only a specific package
  19
  20 WORKSPACE=path         Path to the Arvados source tree to build packages from
  21
  22 EOF
  23
  24 set -e
  25
  26 if ! [[ -n "$WORKSPACE" ]]; then
  27   echo >&2 "$helpmessage"
  28   echo >&2
  29   echo >&2 "Error: WORKSPACE environment variable not set"
  30   echo >&2
  31   exit 1
  32 fi
  33
  34 if ! [[ -d "$WORKSPACE" ]]; then
  35   echo >&2 "$helpmessage"
  36   echo >&2
  37   echo >&2 "Error: $WORKSPACE is not a directory"
  38   echo >&2
  39   exit 1
  40 fi
  41
  42 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  43     help,debug,test-packages,target:,command:,only-test: \
  44     -- "" "$@")
  45 if [ $? -ne 0 ]; then
  46     exit 1
  47 fi
  48
  49 TARGET=debian7
  50 COMMAND=
  51 DEBUG=
  52
  53 eval set -- "$PARSEDOPTS"
  54 while [ $# -gt 0 ]; do
  55     case "$1" in
  56         --help)
  57             echo >&2 "$helpmessage"
  58             echo >&2
  59             exit 1
  60             ;;
  61         --target)
  62             TARGET="$2"; shift
  63             ;;
  64         --only-test)
  65             packages="$2"; shift
  66             ;;
  67         --debug)
  68             DEBUG=" --debug"
  69             ;;
  70         --command)
  71             COMMAND="$2"; shift
  72             ;;
  73         --test-packages)
  74             test_packages=1
  75             ;;
  76         --)
  77             if [ $# -gt 1 ]; then
  78                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  79                 exit 1
  80             fi
  81             ;;
  82     esac
  83     shift
  84 done
  85
  86 set -e
  87
  88 if [[ -n "$test_packages" ]]; then
  89     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
  90         createrepo $WORKSPACE/packages/$TARGET
  91     fi
  92
  93     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
  94         (cd $WORKSPACE/packages/$TARGET
  95          dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz
  96         )
  97     fi
  98
  99     COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
 100     IMAGE="arvados/package-test:$TARGET"
 101 else
 102     IMAGE="arvados/build:$TARGET"
 103     if [[ "$COMMAND" != "" ]]; then
 104         COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 105     fi
 106 fi
 107
 108 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 109
 110 if [[ -n "$test_packages" ]]; then
 111     pushd "$JENKINS_DIR/package-test-dockerfiles"
 112 else
 113     pushd "$JENKINS_DIR/package-build-dockerfiles"
 114     make "$TARGET/generated"
 115 fi
 116
 117 echo $TARGET
 118 cd $TARGET
 119 time docker build --tag=$IMAGE .
 120 popd
 121
 122 if test -z "$packages" ; then
 123     packages="arvados-api-server
 124         arvados-data-manager
 125         arvados-docker-cleaner
 126         arvados-git-httpd
 127         arvados-node-manager
 128         arvados-src
 129         arvados-workbench
 130         crunch-dispatch-local
 131         crunch-dispatch-slurm
 132         crunch-run
 133         crunchstat
 134         keep-balance
 135         keep-block-check
 136         keepproxy
 137         keep-rsync
 138         keepstore
 139         keep-web
 140         libarvados-perl"
 141
 142     case "$TARGET" in
 143         centos6)
 144             packages="$packages python27-python-arvados-fuse
 145                   python27-python-arvados-python-client python27-python-arvados-cwl-runner"
 146             ;;
 147         *)
 148             packages="$packages python-arvados-fuse
 149                   python-arvados-python-client python-arvados-cwl-runner"
 150             ;;
 151     esac
 152 fi
 153
 154 FINAL_EXITCODE=0
 155
 156 package_fails=""
 157
 158 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
 159 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
 160
 161 docker_volume_args=(
 162     -v "$JENKINS_DIR:/jenkins"
 163     -v "$WORKSPACE:/arvados"
 164     -v /arvados/services/api/vendor/bundle
 165     -v /arvados/apps/workbench/vendor/bundle
 166     -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
 167     -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
 168 )
 169
 170 if [[ -n "$test_packages" ]]; then
 171     for p in $packages ; do
 172         echo
 173         echo "START: $p test on $IMAGE" >&2
 174         if docker run --rm \
 175             "${docker_volume_args[@]}" \
 176             --env ARVADOS_DEBUG=1 \
 177             --env "TARGET=$TARGET" \
 178             --env "WORKSPACE=/arvados" \
 179             "$IMAGE" $COMMAND $p
 180         then
 181             echo "OK: $p test on $IMAGE succeeded" >&2
 182         else
 183             FINAL_EXITCODE=$?
 184             package_fails="$package_fails $p"
 185             echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 186         fi
 187     done
 188 else
 189     echo
 190     echo "START: build packages on $IMAGE" >&2
 191     if docker run --rm \
 192         "${docker_volume_args[@]}" \
 193         --env ARVADOS_DEBUG=1 \
 194         "$IMAGE" $COMMAND
 195     then
 196         echo
 197         echo "OK: build packages on $IMAGE succeeded" >&2
 198     else
 199         FINAL_EXITCODE=$?
 200         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 201     fi
 202 fi
 203
 204 if test -n "$package_fails" ; then
 205     echo "Failed package tests:$package_fails" >&2
 206 fi
 207
 208 exit $FINAL_EXITCODE