X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d5f9f022f9414ffb8bc1b7b2f04096468574e2e5..bebf531a2cb9cf08843d6ac4d4ae2238d58efc2b:/jenkins/run-build-packages.sh

diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh
index 31bf7084f1..84ada48bc6 100755
--- a/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
@@ -94,8 +95,7 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=python3
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -107,8 +107,7 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=python3
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -120,8 +119,7 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=python3
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py \
-            virtualenv pykka requests six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -144,11 +142,10 @@ case "$TARGET" in
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
         PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
             oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py \
-            pykka requests six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname pycurl
-            python-daemon lockfile llfuse)
-        PYTHON3_BACKPORTS=(docker-py six requests)
+            python-daemon lockfile llfuse 'pbr<1.0')
+        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -256,6 +253,9 @@ handle_python_package
 cd "$WORKSPACE/sdk/python"
 handle_python_package
 
+cd "$WORKSPACE/sdk/cwl"
+handle_python_package
+
 cd "$WORKSPACE/services/fuse"
 handle_python_package
 
@@ -270,6 +270,8 @@ handle_python_package
     COMMIT_HASH=$(format_last_commit_here "%H")
 
     SRC_BUILD_DIR=$(mktemp -d)
+    # mktemp creates the directory with 0700 permissions by default
+    chmod 755 $SRC_BUILD_DIR
     git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
     cd "$SRC_BUILD_DIR"
 
@@ -282,77 +284,25 @@ handle_python_package
     cd $WORKSPACE/packages/$TARGET
     fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
 
-    rm -r "$SRC_BUILD_DIR"
+    rm -rf "$SRC_BUILD_DIR"
 )
 
-# Keep
+# Go binaries
 export GOPATH=$(mktemp -d)
-mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
-
-# keepstore
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore"
-PKG_VERSION=$(version_from_git)
-go get "git.curoverse.com/arvados.git/services/keepstore"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN"
-
-# Get GO SDK version
-cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
-GO_SDK_VERSION=$(version_from_git)
-GO_SDK_TIMESTAMP=$(timestamp_from_git)
-
-# keepproxy
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepproxy"
-KEEPPROXY_VERSION=$(version_from_git)
-KEEPPROXY_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$KEEPPROXY_TIMESTAMP" ]]; then
-  PKG_VERSION=$GO_SDK_VERSION
-else
-  PKG_VERSION=$KEEPPROXY_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/keepproxy"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
-
-# datamanager
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager"
-DATAMANAGER_VERSION=$(version_from_git)
-DATAMANAGER_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$DATAMANAGER_TIMESTAMP" ]]; then
-  PKG_VERSION=$GO_SDK_VERSION
-else
-  PKG_VERSION=$DATAMANAGER_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/datamanager"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
-
-# arv-git-httpd
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd"
-ARVGITHTTPD_VERSION=$(version_from_git)
-ARVGITHTTPD_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$ARVGITHTTPD_TIMESTAMP" ]]; then
-  PKG_VERSION=$GO_SDK_VERSION
-else
-  PKG_VERSION=$ARVGITHTTPD_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
-
-# crunchstat
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
-PKG_VERSION=$(version_from_git)
-go get "git.curoverse.com/arvados.git/services/crunchstat"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs"
+package_go_binary services/keepstore keepstore \
+    "Keep storage daemon, accessible to clients on the LAN"
+package_go_binary services/keepproxy keepproxy \
+    "Make a Keep cluster accessible to clients that are not on the LAN"
+package_go_binary services/keep-web keep-web \
+    "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary services/datamanager arvados-data-manager \
+    "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary services/arv-git-httpd arvados-git-httpd \
+    "Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/crunchstat crunchstat \
+    "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary tools/keep-rsync keep-rsync \
+    "Copy all data from one set of Keep servers to another"
 
 # The Python SDK
 # Please resist the temptation to add --no-python-fix-name to the fpm call here
@@ -394,16 +344,52 @@ LIBCLOUD_DIR=$(mktemp -d)
 (
     cd $LIBCLOUD_DIR
     git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
-    git checkout apache-libcloud-0.18.1.dev2
+    git checkout apache-libcloud-$LIBCLOUD_PIN
+    # libcloud is absurdly noisy without -q, so force -q here
+    OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+    DASHQ_UNLESS_DEBUG=-q
     handle_python_package
+    DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
 )
 fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
 rm -rf $LIBCLOUD_DIR
 
-# A few dependencies
+# Python 2 dependencies
 for deppkg in "${PYTHON_BACKPORTS[@]}"; do
     outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
-    fpm_build "$deppkg" "$outname"
+    case "$deppkg" in
+        httplib2)
+        # Work around 0640 permissions on some package files on Debian 8
+        # and Ubuntu 14.04.  See #7591.
+            httplib2_workdir=$(mktemp -d httplib2-XXXXXX) && (
+                set -e
+                cd "$httplib2_workdir"
+                pip install --download . httplib2
+                tar -xf httplib2-*.tar*
+                cd httplib2-*/
+                "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
+                chmod -R go+rX .
+                set +e
+                # --iteration 2 provides an upgrade for previously built
+                # buggy packages.  Arguments past $outname can be removed
+                # once we're building httplib2 > 0.9.2.
+                fpm_build . "$outname" "" python "" --iteration 2
+                # The upload step uses the package timestamp to determine
+                # whether it's new.  --no-clobber plays nice with that.
+                mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
+            )
+            if [ 0 != "$?" ]; then
+                echo "ERROR: httplib2 build process failed"
+                EXITCODE=1
+            fi
+            if [ -n "$httplib2_workdir" ]; then
+                rm -rf "$httplib2_workdir"
+            fi
+            ;;
+        *)
+            fpm_build "$deppkg" "$outname"
+            ;;
+    esac
 done
 
 # Python 3 dependencies