Fix backports dir search for PACKAGE="llfuse==0.41.1". refs #8345
[arvados-dev.git] / jenkins / run-build-packages.sh
index 97251c5cc0d9b3264ac92b385d5ffe486e459c76..697092966bda6575d0ddaaafac635841555de650 100755 (executable)
@@ -89,11 +89,11 @@ case "$TARGET" in
         PYTHON2_PKG_PREFIX=python
         PYTHON3_PACKAGE=python$PYTHON3_VERSION
         PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
-            oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5')
+            'pycurl<7.21.5' contextlib2)
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
     debian8)
@@ -102,8 +102,8 @@ case "$TARGET" in
         PYTHON2_PKG_PREFIX=python
         PYTHON3_PACKAGE=python$PYTHON3_VERSION
         PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
-            oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
             'pycurl<7.21.5')
@@ -115,10 +115,11 @@ case "$TARGET" in
         PYTHON2_PKG_PREFIX=python
         PYTHON3_PACKAGE=python$PYTHON3_VERSION
         PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
-            oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+            contextlib2 \
             'pycurl<7.21.5')
         PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
         ;;
@@ -129,7 +130,7 @@ case "$TARGET" in
         PYTHON3_PACKAGE=python$PYTHON3_VERSION
         PYTHON3_PKG_PREFIX=python3
         PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
-            google-api-python-client six uritemplate oauth2client httplib2 \
+            google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
             rsa 'pycurl<7.21.5' backports.ssl_match_hostname)
         PYTHON3_BACKPORTS=(docker-py requests websocket-client)
         ;;
@@ -139,8 +140,8 @@ case "$TARGET" in
         PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
-            oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
             rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
             ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
             python-daemon lockfile llfuse==0.41.1 'pbr<1.0')
@@ -287,7 +288,89 @@ handle_python_package
     rm -rf "$SRC_BUILD_DIR"
 )
 
+# On older platforms we need to publish a backport of libfuse >=2.9.2,
+# and we need to build and install it here in order to even build an
+# llfuse package.
+cd $WORKSPACE/packages/$TARGET
+if [[ $TARGET =~ ubuntu1204 ]]; then
+    # port libfuse 2.9.2 to Ubuntu 12.04
+    LIBFUSE_DIR=$(mktemp -d)
+    (
+        cd $LIBFUSE_DIR
+        # download fuse 2.9.2 ubuntu 14.04 source package
+        file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+        file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+        file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+
+        # install dpkg-source and dpkg-buildpackage commands
+        apt-get install -y --no-install-recommends dpkg-dev
+
+        # extract source and apply patches
+        dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
+        rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
+
+        # add new version to changelog
+        cd fuse-2.9.2
+        (
+            echo "fuse (2.9.2-5) precise; urgency=low"
+            echo
+            echo "  * Backported from trusty-security to precise"
+            echo
+            echo " -- Joshua Randall <jcrandall@alum.mit.edu>  Thu, 4 Feb 2016 11:31:00 -0000"
+            echo
+            cat debian/changelog
+        ) > debian/changelog.new
+        mv debian/changelog.new debian/changelog
+
+        # install build-deps and build
+        apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
+        dpkg-buildpackage -rfakeroot -b
+    )
+    fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
+    fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
+    fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
+    dpkg -i \
+        "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
+        "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
+        "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
+    apt-get -y --no-install-recommends -f install
+    rm -rf $LIBFUSE_DIR
+elif [[ $TARGET =~ centos6 ]]; then
+    # port fuse 2.9.2 to centos 6
+    # install tools to build rpm from source
+    yum install -y rpm-build redhat-rpm-config
+    LIBFUSE_DIR=$(mktemp -d)
+    (
+        cd "$LIBFUSE_DIR"
+        # download fuse 2.9.2 centos 7 source rpm
+        file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
+        (
+            # modify source rpm spec to remove conflict on filesystem version
+            mkdir -p /root/rpmbuild/SOURCES
+            cd /root/rpmbuild/SOURCES
+            rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
+            perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
+        )
+        # build rpms from source 
+        rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
+        rm -f fuse-2.9.2-6.el7.src.rpm
+        # move built RPMs to LIBFUSE_DIR
+        mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+        mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+        mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+        rm -rf /root/rpmbuild
+    )
+    fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
+    fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+    fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+    yum install -y \
+        "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
+        "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
+        "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
+fi
+
 # Go binaries
+cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
 package_go_binary services/keepstore keepstore \
     "Keep storage daemon, accessible to clients on the LAN"
@@ -315,6 +398,11 @@ cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/python/build"
 fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
 
+# cwl-runner
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/sdk/cwl/build"
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
+
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
     cd $WORKSPACE/packages/$TARGET