Merge branch '8345-llfuse-unpin' refs #8345

diff --combined jenkins/run-build-packages.sh
index c89c418af63f531f8994a2e1033c91f796e12fcd,061bc99a627d50dde03d04b6f4f02d7a6d1a4a64..697092966bda6575d0ddaaafac635841555de650
--- 1/jenkins/run-build-packages.sh
--- 2/jenkins/run-build-packages.sh
+++ b/jenkins/run-build-packages.sh
@@@ -93,7 -93,7 +93,7 @@@ case "$TARGET" i
              oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
              rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
              ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-             'pycurl<7.21.5')
+             'pycurl<7.21.5' contextlib2)
          PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
          ;;
      debian8)
@@@ -119,6 -119,7 +119,7 @@@
              oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
              rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
              ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+             contextlib2 \
              'pycurl<7.21.5')
          PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
          ;;
@@@ -287,7 -288,89 +288,89 @@@ handle_python_packag
      rm -rf "$SRC_BUILD_DIR"
  )
  
+ # On older platforms we need to publish a backport of libfuse >=2.9.2,
+ # and we need to build and install it here in order to even build an
+ # llfuse package.
+ cd $WORKSPACE/packages/$TARGET
+ if [[ $TARGET =~ ubuntu1204 ]]; then
+     # port libfuse 2.9.2 to Ubuntu 12.04
+     LIBFUSE_DIR=$(mktemp -d)
+     (
+         cd $LIBFUSE_DIR
+         # download fuse 2.9.2 ubuntu 14.04 source package
+         file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+         file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+         file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+ 
+         # install dpkg-source and dpkg-buildpackage commands
+         apt-get install -y --no-install-recommends dpkg-dev
+ 
+         # extract source and apply patches
+         dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
+         rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
+ 
+         # add new version to changelog
+         cd fuse-2.9.2
+         (
+             echo "fuse (2.9.2-5) precise; urgency=low"
+             echo
+             echo "  * Backported from trusty-security to precise"
+             echo
+             echo " -- Joshua Randall <jcrandall@alum.mit.edu>  Thu, 4 Feb 2016 11:31:00 -0000"
+             echo
+             cat debian/changelog
+         ) > debian/changelog.new
+         mv debian/changelog.new debian/changelog
+ 
+         # install build-deps and build
+         apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
+         dpkg-buildpackage -rfakeroot -b
+     )
+     fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
+     fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
+     fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
+     dpkg -i \
+         "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
+         "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
+         "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
+     apt-get -y --no-install-recommends -f install
+     rm -rf $LIBFUSE_DIR
+ elif [[ $TARGET =~ centos6 ]]; then
+     # port fuse 2.9.2 to centos 6
+     # install tools to build rpm from source
+     yum install -y rpm-build redhat-rpm-config
+     LIBFUSE_DIR=$(mktemp -d)
+     (
+         cd "$LIBFUSE_DIR"
+         # download fuse 2.9.2 centos 7 source rpm
+         file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
+         (
+             # modify source rpm spec to remove conflict on filesystem version
+             mkdir -p /root/rpmbuild/SOURCES
+             cd /root/rpmbuild/SOURCES
+             rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
+             perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
+         )
+         # build rpms from source 
+         rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
+         rm -f fuse-2.9.2-6.el7.src.rpm
+         # move built RPMs to LIBFUSE_DIR
+         mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+         mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+         mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+         rm -rf /root/rpmbuild
+     )
+     fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
+     fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+     fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+     yum install -y \
+         "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
+         "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
+         "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
+ fi
+ 
  # Go binaries
+ cd $WORKSPACE/packages/$TARGET
  export GOPATH=$(mktemp -d)
  package_go_binary services/keepstore keepstore \
      "Keep storage daemon, accessible to clients on the LAN"
@@@ -315,11 -398,6 +398,11 @@@ cd $WORKSPACE/packages/$TARGE
  rm -rf "$WORKSPACE/sdk/python/build"
  fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
  
 +# cwl-runner
 +cd $WORKSPACE/packages/$TARGET
 +rm -rf "$WORKSPACE/sdk/cwl/build"
 +fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
 +
  # The PAM module
  if [[ $TARGET =~ debian|ubuntu ]]; then
      cd $WORKSPACE/packages/$TARGET

diff --combined jenkins/run-tests.sh
index 3b3ddff053dd8011a77513ab621d76a26cb54f10,07a4bf48980823310db3bda767418f8a6c09191c..41b237bc7f8c88f4d31686ac87afee3be46bcfdd
--- 1/jenkins/run-tests.sh
--- 2/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@@ -79,7 -79,6 +79,7 @@@ sdk/go/manifes
  sdk/go/blockdigest
  sdk/go/streamer
  sdk/go/crunchrunner
 +sdk/cwl
  tools/crunchstat-summary
  tools/keep-rsync
  
@@@ -426,6 -425,15 +426,15 @@@ pip freeze 2>/dev/null | egrep ^apache-
      || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
      || fatal "pip install apache-libcloud failed"
  
+ # This will help people who reuse --temp dirs when we upgrade to llfuse 0.42
+ if egrep -q 'llfuse.*>= *0\.42' "$WORKSPACE/services/fuse/setup.py"; then
+     # Uninstall old llfuse, because services/fuse "pip install" won't
+     # upgrade it by default.
+     if pip freeze | egrep '^llfuse==0\.41\.'; then
+         yes | pip uninstall 'llfuse<0.42'
+     fi
+ fi
+ 
  # Deactivate Python 2 virtualenv
  deactivate
  
@@@ -643,7 -651,6 +652,7 @@@ declare -a pythonstuf
  pythonstuff=(
      sdk/pam
      sdk/python
 +    sdk/cwl
      services/fuse
      services/nodemanager
      tools/crunchstat-summary