From: Brett Smith Date: Fri, 27 May 2016 22:34:56 +0000 (-0400) Subject: 9309: Add packages and tests for CentOS 7. X-Git-Tag: 1.1.0~910^2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/7961ba6a5a14ab60fbbe181f9d5fcb7df9c4991f 9309: Add packages and tests for CentOS 7. --- diff --git a/backports/python-apache-libcloud/fpm-info.sh b/backports/python-apache-libcloud/fpm-info.sh new file mode 100644 index 0000000000..0333bdf479 --- /dev/null +++ b/backports/python-apache-libcloud/fpm-info.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +case $TARGET in + centos7) + # fpm incorrectly transforms the dependency name in this case. + fpm_depends+=(python-backports-ssl_match_hostname) + fpm_args+=(--python-disable-dependency backports.ssl-match-hostname) + ;; +esac diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile index 3482886112..2180b871f7 100644 --- a/build/package-build-dockerfiles/Makefile +++ b/build/package-build-dockerfiles/Makefile @@ -1,9 +1,13 @@ -all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated +all: centos6/generated centos7/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated centos6/generated: common-generated-all test -d centos6/generated || mkdir centos6/generated cp -rlt centos6/generated common-generated/* +centos7/generated: common-generated-all + test -d centos7/generated || mkdir centos7/generated + cp -rlt centos7/generated common-generated/* + debian7/generated: common-generated-all test -d debian7/generated || mkdir debian7/generated cp -rlt debian7/generated common-generated/* diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile new file mode 100644 index 0000000000..311aaa2663 --- /dev/null +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -0,0 +1,25 @@ +FROM centos:7 +MAINTAINER Brett Smith + +# Install build dependencies provided in base distribution +RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git + +# Install golang binary +ADD generated/go1.6.2.linux-amd64.tar.gz /usr/local/ +RUN ln -s /usr/local/go/bin/go /usr/local/bin/ + +# Install RVM +RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.1 && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler && \ + /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b + +# Need to "touch" RPM database to workaround bug in interaction between +# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602) +RUN touch /var/lib/rpm/* && yum -q -y install python33 +RUN scl enable python33 "easy_install-3.3 pip" && easy_install-2.7 pip + +ENV WORKSPACE /arvados +CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"] diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile new file mode 100644 index 0000000000..6bc40bffa5 --- /dev/null +++ b/build/package-test-dockerfiles/centos7/Dockerfile @@ -0,0 +1,15 @@ +FROM centos:7 +MAINTAINER Brett Smith + +RUN yum -q -y install scl-utils centos-release-scl which tar + +# Install RVM +RUN touch /var/lib/rpm/* && \ + gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.1 && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundle && \ + /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b + +COPY localrepo.repo /etc/yum.repos.d/localrepo.repo diff --git a/build/package-test-dockerfiles/centos7/localrepo.repo b/build/package-test-dockerfiles/centos7/localrepo.repo new file mode 100644 index 0000000000..ebb8765940 --- /dev/null +++ b/build/package-test-dockerfiles/centos7/localrepo.repo @@ -0,0 +1,5 @@ +[localrepo] +name=Arvados Test +baseurl=file:///arvados/packages/centos7 +gpgcheck=0 +enabled=1 diff --git a/build/package-testing/rpm-common-test-packages.sh b/build/package-testing/rpm-common-test-packages.sh new file mode 100755 index 0000000000..4d0c32bc9c --- /dev/null +++ b/build/package-testing/rpm-common-test-packages.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -eu + +target=$(basename "$0" | grep -Eo '\bcentos[[:digit:]]+\b') + +yum -q clean all +touch /var/lib/rpm/* + +export ARV_PACKAGES_DIR="/arvados/packages/$target" + +rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before" + +yum install --assumeyes $1 + +rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after" + +diff "$ARV_PACKAGES_DIR/$1".{before,after} >"$ARV_PACKAGES_DIR/$1.diff" || true + +# Enable any Software Collections that the package depended on. +if [[ -d /opt/rh ]]; then + # We have to stage the list to a file, because `ls | while read` would + # make a subshell, causing the `source` lines to have no effect. + scl_list=$(mktemp) + ls /opt/rh >"$scl_list" + + # SCL scripts aren't designed to run with -eu. + set +eu + while read scl; do + source scl_source enable "$scl" + done <"$scl_list" + set -eu + rm "$scl_list" +fi + +mkdir -p /tmp/opts +cd /tmp/opts + +rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null + +find -name '*.so' | while read so; do + echo -e "\n== Packages dependencies for $so ==" + ldd "$so" \ + | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u +done + +exec /jenkins/package-testing/common-test-packages.sh "$1" diff --git a/build/package-testing/test-package-arvados-api-server.sh b/build/package-testing/test-package-arvados-api-server.sh index e97544828c..66d5969428 100755 --- a/build/package-testing/test-package-arvados-api-server.sh +++ b/build/package-testing/test-package-arvados-api-server.sh @@ -7,7 +7,7 @@ case "$TARGET" in apt-get install -y nginx dpkg-reconfigure arvados-api-server ;; - centos6) + centos*) yum install --assumeyes httpd yum reinstall --assumeyes arvados-api-server ;; diff --git a/build/package-testing/test-package-arvados-sso-server.sh b/build/package-testing/test-package-arvados-sso-server.sh index c1a377e419..ab0b225ed5 100755 --- a/build/package-testing/test-package-arvados-sso-server.sh +++ b/build/package-testing/test-package-arvados-sso-server.sh @@ -18,7 +18,7 @@ case "$TARGET" in debian*|ubuntu*) FORMAT=deb ;; - centos6) + centos*) FORMAT=rpm ;; *) @@ -75,12 +75,9 @@ fi if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then # We need to set up our database configuration now. if [[ "$FORMAT" == "rpm" ]]; then - # postgres packaging on CentOS6 is kind of primitive, needs an initdb service postgresql initdb - if [ "$TARGET" = "centos6" ]; then - sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf - sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf - fi + sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf + sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf fi service postgresql start diff --git a/build/package-testing/test-package-arvados-workbench.sh b/build/package-testing/test-package-arvados-workbench.sh index 1be4dea3ed..5deb1a0aa1 100755 --- a/build/package-testing/test-package-arvados-workbench.sh +++ b/build/package-testing/test-package-arvados-workbench.sh @@ -7,7 +7,7 @@ case "$TARGET" in apt-get install -y nginx dpkg-reconfigure arvados-workbench ;; - centos6) + centos*) yum install --assumeyes httpd yum reinstall --assumeyes arvados-workbench ;; diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh deleted file mode 100755 index 4e05364f01..0000000000 --- a/build/package-testing/test-packages-centos6.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -eu - -yum -q clean all -touch /var/lib/rpm/* - -export ARV_PACKAGES_DIR=/arvados/packages/centos6 - -rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before" - -yum install --assumeyes $1 - -rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after" - -set +e -diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff" -set -e - -SCL="" -if scl enable python27 true 2>/dev/null ; then - SCL="scl enable python27" -fi - -mkdir -p /tmp/opts -cd /tmp/opts - -rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null - -shared=$(find -name '*.so') -if test -n "$shared" ; then - for so in $shared ; do - echo - echo "== Packages dependencies for $so ==" - $SCL ldd "$so" \ - | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u - done -fi - -if test -n "$SCL" ; then - exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'" -else - exec /jenkins/package-testing/common-test-packages.sh "$1" -fi diff --git a/build/package-testing/test-packages-centos6.sh b/build/package-testing/test-packages-centos6.sh new file mode 120000 index 0000000000..64ef6040c1 --- /dev/null +++ b/build/package-testing/test-packages-centos6.sh @@ -0,0 +1 @@ +rpm-common-test-packages.sh \ No newline at end of file diff --git a/build/package-testing/test-packages-centos7.sh b/build/package-testing/test-packages-centos7.sh new file mode 120000 index 0000000000..64ef6040c1 --- /dev/null +++ b/build/package-testing/test-packages-centos7.sh @@ -0,0 +1 @@ +rpm-common-test-packages.sh \ No newline at end of file diff --git a/build/run-build-packages-sso.sh b/build/run-build-packages-sso.sh index cc673a6d33..b5dcdfce53 100755 --- a/build/run-build-packages-sso.sh +++ b/build/run-build-packages-sso.sh @@ -80,7 +80,7 @@ case "$TARGET" in ubuntu1404) FORMAT=deb ;; - centos6) + centos6|centos7) FORMAT=rpm ;; *) diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index bd49c4dedc..e7934a6484 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -155,6 +155,26 @@ case "$TARGET" in PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) export PYCURL_SSL_LIBRARY=nss ;; + centos7) + FORMAT=rpm + PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') + PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE + PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages + PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') + PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE + PYTHON3_PREFIX=/opt/rh/python33/root/usr + PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages + PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ + oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ + rsa uritemplate httplib2 ws4py pykka pyexecjs jsonschema \ + ciso8601 pycrypto 'pycurl<7.21.5' \ + python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \ + 'rdflib>=4.2.0' shellescape mistune typing avro \ + isodate pyparsing sparqlwrapper html5lib keepalive \ + ruamel.ordereddict) + PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) + export PYCURL_SSL_LIBRARY=nss + ;; *) echo -e "$0: Unknown target '$TARGET'.\n" >&2 exit 1