Merge branch '7591-httplib2-perms-fix-wip'

[arvados-dev.git] / jenkins / run-build-packages.sh

diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 31bf7084f1cee8e138963a257b340b913afe533a..fad3f776d0c0ff02c36823de5e32bc3dd41e26af 100755 (executable)
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
@@ -95,7 +96,7 @@ case "$TARGET" in
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            pykka requests six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -108,7 +109,7 @@ case "$TARGET" in
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            pykka requests six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -121,7 +122,7 @@ case "$TARGET" in
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            pykka requests six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -147,8 +148,8 @@ case "$TARGET" in
             rsa uritemplate httplib2 ws4py \
             pykka requests six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl
-            python-daemon lockfile llfuse)
-        PYTHON3_BACKPORTS=(docker-py six requests)
+            python-daemon lockfile llfuse 'pbr<1.0')
+        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -270,6 +271,8 @@ handle_python_package
     COMMIT_HASH=$(format_last_commit_here "%H")
 
     SRC_BUILD_DIR=$(mktemp -d)
+    # mktemp creates the directory with 0700 permissions by default
+    chmod 755 $SRC_BUILD_DIR
     git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
     cd "$SRC_BUILD_DIR"
 
@@ -317,6 +320,21 @@ go get "git.curoverse.com/arvados.git/services/keepproxy"
 cd $WORKSPACE/packages/$TARGET
 fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
 
+# keep-web
+cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keep-web"
+KEEP_WEB_VERSION=$(version_from_git)
+KEEP_WEB_TIMESTAMP=$(timestamp_from_git)
+
+if [[ "$GO_SDK_TIMESTAMP" -gt "$KEEP_WEB_TIMESTAMP" ]]; then
+  PKG_VERSION=$GO_SDK_VERSION
+else
+  PKG_VERSION=$KEEP_WEB_VERSION
+fi
+
+go get "git.curoverse.com/arvados.git/services/keep-web"
+cd $WORKSPACE/packages/$TARGET
+fpm_build $GOPATH/bin/keep-web=/usr/bin/keep-web keep-web 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Static web hosting service for user data stored in Arvados Keep"
+
 # datamanager
 cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager"
 DATAMANAGER_VERSION=$(version_from_git)
@@ -394,16 +412,52 @@ LIBCLOUD_DIR=$(mktemp -d)
 (
     cd $LIBCLOUD_DIR
     git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
-    git checkout apache-libcloud-0.18.1.dev2
+    git checkout apache-libcloud-$LIBCLOUD_PIN
+    # libcloud is absurdly noisy without -q, so force -q here
+    OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+    DASHQ_UNLESS_DEBUG=-q
     handle_python_package
+    DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
 )
 fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
 rm -rf $LIBCLOUD_DIR
 
-# A few dependencies
+# Python 2 dependencies
 for deppkg in "${PYTHON_BACKPORTS[@]}"; do
     outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
-    fpm_build "$deppkg" "$outname"
+    case "$deppkg" in
+        httplib2)
+        # Work around 0640 permissions on some package files on Debian 8
+        # and Ubuntu 14.04.  See #7591.
+            httplib2_workdir=$(mktemp -d httplib2-XXXXXX) && (
+                set -e
+                cd "$httplib2_workdir"
+                pip install --download . httplib2
+                tar -xf httplib2-*.tar*
+                cd httplib2-*/
+                "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
+                chmod -R go+rX .
+                set +e
+                # --iteration 2 provides an upgrade for previously built
+                # buggy packages.  Arguments past $outname can be removed
+                # once we're building httplib2 > 0.9.2.
+                fpm_build . "$outname" "" python "" --iteration 2
+                # The upload step uses the package timestamp to determine
+                # whether it's new.  --no-clobber plays nice with that.
+                mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
+            )
+            if [ 0 != "$?" ]; then
+                echo "ERROR: httplib2 build process failed"
+                EXITCODE=1
+            fi
+            if [ -n "$httplib2_workdir" ]; then
+                rm -rf "$httplib2_workdir"
+            fi
+            ;;
+        *)
+            fpm_build "$deppkg" "$outname"
+            ;;
+    esac
 done
 
 # Python 3 dependencies