X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/f46095f12c8470a4eac64997bef622e5f26cafb9..23391a8d819238b3b9f934c93331a0d96a8dec85:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index 56a67f1..6970929 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -89,11 +89,11 @@ case "$TARGET" in PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION PYTHON3_PKG_PREFIX=python3 - PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \ + oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse \ - 'pycurl<7.21.5') + ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ + 'pycurl<7.21.5' contextlib2) PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; debian8) @@ -102,10 +102,10 @@ case "$TARGET" in PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION PYTHON3_PKG_PREFIX=python3 - PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \ + oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse \ + ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ 'pycurl<7.21.5') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; @@ -115,10 +115,10 @@ case "$TARGET" in PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION PYTHON3_PKG_PREFIX=python3 - PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \ + oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse \ + ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ contextlib2 \ 'pycurl<7.21.5') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) @@ -129,8 +129,8 @@ case "$TARGET" in PYTHON2_PKG_PREFIX=python PYTHON3_PACKAGE=python$PYTHON3_VERSION PYTHON3_PKG_PREFIX=python3 - PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse ciso8601 \ - google-api-python-client six uritemplate oauth2client httplib2 \ + PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \ + google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \ rsa 'pycurl<7.21.5' backports.ssl_match_hostname) PYTHON3_BACKPORTS=(docker-py requests websocket-client) ;; @@ -140,11 +140,11 @@ case "$TARGET" in PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE - PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ - oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \ + oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \ - python-daemon lockfile llfuse 'pbr<1.0') + python-daemon lockfile llfuse==0.41.1 'pbr<1.0') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) export PYCURL_SSL_LIBRARY=nss ;; @@ -288,7 +288,89 @@ handle_python_package rm -rf "$SRC_BUILD_DIR" ) +# On older platforms we need to publish a backport of libfuse >=2.9.2, +# and we need to build and install it here in order to even build an +# llfuse package. +cd $WORKSPACE/packages/$TARGET +if [[ $TARGET =~ ubuntu1204 ]]; then + # port libfuse 2.9.2 to Ubuntu 12.04 + LIBFUSE_DIR=$(mktemp -d) + ( + cd $LIBFUSE_DIR + # download fuse 2.9.2 ubuntu 14.04 source package + file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" + file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" + file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" + + # install dpkg-source and dpkg-buildpackage commands + apt-get install -y --no-install-recommends dpkg-dev + + # extract source and apply patches + dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc + rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc + + # add new version to changelog + cd fuse-2.9.2 + ( + echo "fuse (2.9.2-5) precise; urgency=low" + echo + echo " * Backported from trusty-security to precise" + echo + echo " -- Joshua Randall Thu, 4 Feb 2016 11:31:00 -0000" + echo + cat debian/changelog + ) > debian/changelog.new + mv debian/changelog.new debian/changelog + + # install build-deps and build + apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev + dpkg-buildpackage -rfakeroot -b + ) + fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5 + fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5 + fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5 + dpkg -i \ + "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \ + "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \ + "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb" + apt-get -y --no-install-recommends -f install + rm -rf $LIBFUSE_DIR +elif [[ $TARGET =~ centos6 ]]; then + # port fuse 2.9.2 to centos 6 + # install tools to build rpm from source + yum install -y rpm-build redhat-rpm-config + LIBFUSE_DIR=$(mktemp -d) + ( + cd "$LIBFUSE_DIR" + # download fuse 2.9.2 centos 7 source rpm + file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}" + ( + # modify source rpm spec to remove conflict on filesystem version + mkdir -p /root/rpmbuild/SOURCES + cd /root/rpmbuild/SOURCES + rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i + perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec + ) + # build rpms from source + rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec + rm -f fuse-2.9.2-6.el7.src.rpm + # move built RPMs to LIBFUSE_DIR + mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ + mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ + mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ + rm -rf /root/rpmbuild + ) + fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5 + fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends + fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends + yum install -y \ + "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \ + "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \ + "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm" +fi + # Go binaries +cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) package_go_binary services/keepstore keepstore \ "Keep storage daemon, accessible to clients on the LAN" @@ -316,6 +398,11 @@ cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/sdk/python/build" fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git +# cwl-runner +cd $WORKSPACE/packages/$TARGET +rm -rf "$WORKSPACE/sdk/cwl/build" +fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" + # The PAM module if [[ $TARGET =~ debian|ubuntu ]]; then cd $WORKSPACE/packages/$TARGET @@ -355,73 +442,6 @@ LIBCLOUD_DIR=$(mktemp -d) fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud rm -rf $LIBCLOUD_DIR -if [[ $TARGET =~ ubuntu1204 ]]; then - # port libfuse 2.9.2 to Ubuntu 12.04 - LIBFUSE_DIR=$(mktemp -d) - ( - cd $LIBFUSE_DIR - # download fuse 2.9.2 ubuntu 14.04 source package - curl -o fuse_2.9.2.orig.tar.xz http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/fuse_2.9.2.orig.tar.xz - curl -o fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz - curl -o fuse_2.9.2-4ubuntu4.14.04.1.dsc http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/fuse_2.9.2-4ubuntu4.14.04.1.dsc - - # install dpkg-source and dpkg-buildpackage commands - apt-get install -y dpkg-dev - - # extract source and apply patches - dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc - rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc - - # add new version to changelog - cd fuse-2.9.2 - mv debian/changelog debian/changelog.old - echo "fuse (2.9.2-5) precise; urgency=low" > debian/changelog - echo "" >> debian/changelog - echo " * Backported from trusty-security to precise" >> debian/changelog - echo "" >> debian/changelog - echo " -- Joshua Randall Thu, 4 Feb 2016 11:31:00 -0000" >> debian/changelog - echo "" >> debian/changelog - cat debian/changelog.old >> debian/changelog - rm -f debian/changelog.old - - # install build-deps and build - apt-get install -y $(awk 'BEGIN {FS=":"} $1=="Build-Depends" {n=split($2,deps,","); for (i=0; i