build/run-build-packages-one-target.sh

   1 #!/bin/bash
   2
   3 read -rd "\000" helpmessage <<EOF
   4 $(basename $0): Orchestrate run-build-packages.sh for one target
   5
   6 Syntax:
   7         WORKSPACE=/path/to/arvados $(basename $0) [options]
   8
   9 --target <target>
  10     Distribution to build packages for (default: debian7)
  11 --command
  12     Build command to execute (default: use built-in Docker image command)
  13 --test-packages
  14     Run package install test script "test-packages-$target.sh"
  15 --debug
  16     Output debug information (default: false)
  17 --only-test
  18     Test only a specific package
  19
  20 WORKSPACE=path         Path to the Arvados source tree to build packages from
  21
  22 EOF
  23
  24 set -e
  25
  26 if ! [[ -n "$WORKSPACE" ]]; then
  27   echo >&2 "$helpmessage"
  28   echo >&2
  29   echo >&2 "Error: WORKSPACE environment variable not set"
  30   echo >&2
  31   exit 1
  32 fi
  33
  34 if ! [[ -d "$WORKSPACE" ]]; then
  35   echo >&2 "$helpmessage"
  36   echo >&2
  37   echo >&2 "Error: $WORKSPACE is not a directory"
  38   echo >&2
  39   exit 1
  40 fi
  41
  42 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  43     help,debug,test-packages,target:,command:,only-test: \
  44     -- "" "$@")
  45 if [ $? -ne 0 ]; then
  46     exit 1
  47 fi
  48
  49 TARGET=debian7
  50 COMMAND=
  51 DEBUG=
  52
  53 eval set -- "$PARSEDOPTS"
  54 while [ $# -gt 0 ]; do
  55     case "$1" in
  56         --help)
  57             echo >&2 "$helpmessage"
  58             echo >&2
  59             exit 1
  60             ;;
  61         --target)
  62             TARGET="$2"; shift
  63             ;;
  64         --only-test)
  65             packages="$2"; shift
  66             ;;
  67         --debug)
  68             DEBUG=" --debug"
  69             ;;
  70         --command)
  71             COMMAND="$2"; shift
  72             ;;
  73         --test-packages)
  74             test_packages=1
  75             ;;
  76         --)
  77             if [ $# -gt 1 ]; then
  78                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  79                 exit 1
  80             fi
  81             ;;
  82     esac
  83     shift
  84 done
  85
  86 set -e
  87
  88 if [[ -n "$test_packages" ]]; then
  89     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
  90         createrepo $WORKSPACE/packages/$TARGET
  91     fi
  92
  93     if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
  94         (cd $WORKSPACE/packages/$TARGET
  95          dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz
  96         )
  97     fi
  98
  99     COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
 100     IMAGE="arvados/package-test:$TARGET"
 101 else
 102     IMAGE="arvados/build:$TARGET"
 103     if [[ "$COMMAND" != "" ]]; then
 104         COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
 105     fi
 106 fi
 107
 108 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 109
 110 if [[ -n "$test_packages" ]]; then
 111     pushd "$JENKINS_DIR/package-test-dockerfiles"
 112 else
 113     pushd "$JENKINS_DIR/package-build-dockerfiles"
 114     make "$TARGET/generated"
 115 fi
 116
 117 echo $TARGET
 118 cd $TARGET
 119 time docker build --tag=$IMAGE .
 120 popd
 121
 122 if test -z "$packages" ; then
 123     packages="arvados-api-server
 124         arvados-data-manager
 125         arvados-docker-cleaner
 126         arvados-git-httpd
 127         arvados-node-manager
 128         arvados-src
 129         arvados-workbench
 130         crunchstat
 131         keepproxy
 132         keep-rsync
 133         keep-block-check
 134         keepstore
 135         keep-web
 136         libarvados-perl"
 137
 138     case "$TARGET" in
 139         centos6)
 140             packages="$packages python27-python-arvados-fuse
 141                   python27-python-arvados-python-client python27-python-arvados-cwl-runner"
 142             ;;
 143         *)
 144             packages="$packages python-arvados-fuse
 145                   python-arvados-python-client python-arvados-cwl-runner"
 146             ;;
 147     esac
 148 fi
 149
 150 FINAL_EXITCODE=0
 151
 152 package_fails=""
 153
 154 mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
 155 mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
 156
 157 docker_volume_args=(
 158     -v "$JENKINS_DIR:/jenkins"
 159     -v "$WORKSPACE:/arvados"
 160     -v /arvados/services/api/vendor/bundle
 161     -v /arvados/apps/workbench/vendor/bundle
 162     -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
 163     -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
 164 )
 165
 166 if [[ -n "$test_packages" ]]; then
 167     for p in $packages ; do
 168         echo
 169         echo "START: $p test on $IMAGE" >&2
 170         if docker run --rm \
 171             "${docker_volume_args[@]}" \
 172             --env ARVADOS_DEBUG=1 \
 173             --env "TARGET=$TARGET" \
 174             --env "WORKSPACE=/arvados" \
 175             "$IMAGE" $COMMAND $p
 176         then
 177             echo "OK: $p test on $IMAGE succeeded" >&2
 178         else
 179             FINAL_EXITCODE=$?
 180             package_fails="$package_fails $p"
 181             echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 182         fi
 183     done
 184 else
 185     echo
 186     echo "START: build packages on $IMAGE" >&2
 187     if docker run --rm \
 188         "${docker_volume_args[@]}" \
 189         --env ARVADOS_DEBUG=1 \
 190         "$IMAGE" $COMMAND
 191     then
 192         echo
 193         echo "OK: build packages on $IMAGE succeeded" >&2
 194     else
 195         FINAL_EXITCODE=$?
 196         echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
 197     fi
 198 fi
 199
 200 if test -n "$package_fails" ; then
 201     echo "Failed package tests:$package_fails" >&2
 202 fi
 203
 204 exit $FINAL_EXITCODE