7370: Refactoring way the package test is set up and executed.

diff --git a/jenkins/deb-common-test-packages.sh b/jenkins/deb-common-test-packages.sh
index f40ba7bf49eec0cdf3aa708873f1932c9be8451f..d9d0301647d63572b6233a2fb5480128a00ebdfe 100755 (executable)
--- a/jenkins/deb-common-test-packages.sh
+++ b/jenkins/deb-common-test-packages.sh
@@ -1,29 +1,9 @@
-#!/bin/sh
+#!/bin/bash
 
-if test "$1" = --run-test ; then
+# Multiple .deb based distros symlink to this script, so extract the target
+# from the invocation path.
+target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
 
-    if test -z "$WORKSPACE" ; then
-        echo "Must set WORKSPACE"
-        exit 1
-    fi
-
-    self=$(readlink -f $0)
-    base=$(dirname $self)
-
-    cd $WORKSPACE/packages/$2
-    dpkg-scanpackages . /dev/null | gzip -c9 > Packages.gz
-
-    exec docker run \
-         --rm \
-         --volume=$WORKSPACE/packages/$2:/mnt \
-         --volume=$self:/root/run-test.sh \
-         --volume=$base/common-test-packages.sh:/root/common-test.sh \
-         --workdir=/mnt \
-         $3 \
-         /root/run-test.sh
-fi
-
-echo "deb file:///mnt /" >>/etc/apt/sources.list
 apt-get -qq update
 if ! apt-get -qq --assume-yes --force-yes install \
      python-arvados-python-client python-arvados-fuse arvados-node-manager
@@ -34,7 +14,7 @@ fi
 mkdir -p /tmp/opts
 cd /tmp/opts
 
-for r in /mnt/python-*amd64.deb ; do
+for r in /arvados/packages/$target/python-*amd64.deb ; do
     dpkg-deb -x $r .
 done
 
@@ -44,4 +24,4 @@ for so in $(find . -name "*.so") ; do
     ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
 done
 
-exec /root/common-test.sh
+exec /jenkins/common-test-packages.sh

diff --git a/jenkins/dockerfiles/centos6/Dockerfile b/jenkins/dockerfiles/centos6/Dockerfile
index ca974df34f0384200517e1d0af2adaea0987d0eb..d0daef64338e8faffd4895615750aac19b53b34d 100644 (file)
--- a/jenkins/dockerfiles/centos6/Dockerfile
+++ b/jenkins/dockerfiles/centos6/Dockerfile
@@ -19,7 +19,10 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
 ADD https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm /opt/
 
 RUN yum -q -y install /opt/rhscl-*.rpm
-RUN yum -q -y install python27 rh-python34
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python27 rh-python34
 
 ENV WORKSPACE /arvados
 CMD ["scl", "enable", "rh-python34", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]

diff --git a/jenkins/package-test-dockerfiles/centos6/Dockerfile b/jenkins/package-test-dockerfiles/centos6/Dockerfile
new file mode 100644 (file)
index 0000000..5e354e2
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/Dockerfile
@@ -0,0 +1,9 @@
+FROM centos:6
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN yum -q install --assumeyes scl-utils && \
+    curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm && \
+    yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm && \
+    yum -q install --assumeyes python27
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file

diff --git a/jenkins/package-test-dockerfiles/centos6/localrepo.repo b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
new file mode 100644 (file)
index 0000000..ac6b898
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/centos6/localrepo.repo
@@ -0,0 +1,5 @@
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos6
+gpgcheck=0
+enabled=1

diff --git a/jenkins/package-test-dockerfiles/debian7/Dockerfile b/jenkins/package-test-dockerfiles/debian7/Dockerfile
new file mode 100644 (file)
index 0000000..fb789c6
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian7/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:7
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list

diff --git a/jenkins/package-test-dockerfiles/debian8/Dockerfile b/jenkins/package-test-dockerfiles/debian8/Dockerfile
new file mode 100644 (file)
index 0000000..fbcd8f4
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/debian8/Dockerfile
@@ -0,0 +1,4 @@
+FROM debian:8
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
\ No newline at end of file

diff --git a/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
new file mode 100644 (file)
index 0000000..4b6e936
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1204/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:precise
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file

diff --git a/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
new file mode 100644 (file)
index 0000000..4f49d92
--- /dev/null
+++ b/jenkins/package-test-dockerfiles/ubuntu1404/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:trusty
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file

diff --git a/jenkins/run-build-packages-one-target.sh b/jenkins/run-build-packages-one-target.sh
index 5451a4ece3a2bf0bcf5860e2d04bad093c9b2066..3e0e13670fc419cffd89af87ecc90c966668bc22 100755 (executable)
--- a/jenkins/run-build-packages-one-target.sh
+++ b/jenkins/run-build-packages-one-target.sh
@@ -10,6 +10,8 @@ Syntax:
     Distribution to build packages for (default: debian7)
 --command
     Build command to execute (default: use built-in Docker image command)
+--test-packages
+    Run package install test script "test-packages-$target.sh"
 
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 
@@ -32,7 +34,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
 fi
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,target:,command: \
+    help,test-packages,target:,command: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -55,6 +57,9 @@ while [ $# -gt 0 ]; do
         --command)
             COMMAND="$2"; shift
             ;;
+        --test-packages)
+            test_packages=1
+            ;;
         --)
             if [ $# -gt 1 ]; then
                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
@@ -67,34 +72,48 @@ done
 
 set -e
 
-if [[ "$COMMAND" != "" ]]; then
-  COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+if [[ -n "$test_packages" ]]; then
+    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+        createrepo $WORKSPACE/packages/$TARGET
+    fi
+
+    if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+        (cd $WORKSPACE/packages/$TARGET
+         dpkg-scanpackages . /dev/null | gzip -c > Packages.gz
+        )
+    fi
+
+    COMMAND="/jenkins/test-packages-$TARGET.sh"
+    IMAGE="arvados/package-test:$TARGET"
+else
+    IMAGE="arvados/build:$TARGET"
+    if [[ "$COMMAND" != "" ]]; then
+        COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+    fi
 fi
 
-FINAL_EXITCODE=0
 JENKINS_DIR=$(dirname "$(readlink -e "$0")")
 
-run_docker() {
-    local tag=$1; shift
-    if docker run -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
-          --env ARVADOS_DEBUG=1 "arvados/build:$tag" $COMMAND; then
-        # Success - nothing more to do.
-        true
-    else
-        FINAL_EXITCODE=$?
-        echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
-    fi
-}
+if [[ -n "$test_packages" ]]; then
+    pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+    pushd "$JENKINS_DIR/dockerfiles"
+fi
 
-# In case it's needed, build the container. This costs just a few
-# seconds when the container already exist, so it's not a big deal to
-# do it on each run.
-cd "$JENKINS_DIR/dockerfiles"
 echo $TARGET
 cd $TARGET
-time docker build -t arvados/build:$TARGET .
-cd ..
+time docker build --tag=$IMAGE .
+popd
 
-run_docker $TARGET
+FINAL_EXITCODE=0
+
+if docker run --rm -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" \
+          --env ARVADOS_DEBUG=1 "$IMAGE" $COMMAND; then
+    # Success - nothing more to do.
+    true
+else
+    FINAL_EXITCODE=$?
+    echo "ERROR: $tag build failed with exit status $FINAL_EXITCODE." >&2
+fi
 
 exit $FINAL_EXITCODE

diff --git a/jenkins/test-packages-centos6.sh b/jenkins/test-packages-centos6.sh
index 555b9f1058a6f442a7fbea8286144be29de435ba..c8a62b515372df47c8c7ce3de4f94bbd64ca0199 100755 (executable)
--- a/jenkins/test-packages-centos6.sh
+++ b/jenkins/test-packages-centos6.sh
@@ -1,62 +1,29 @@
-#!/bin/sh
+#!/bin/bash
 
-if test "$1" = --run-test ; then
-
-    if test -z "$WORKSPACE" ; then
-        echo "Must set WORKSPACE"
+if ! python --version |& grep "Python 2.7" >/dev/null ; then
+    # Re-execute in the software collections Python 2.7 environment.
+    exec scl enable python27 $0
+else
+    yum -q clean all
+    yum -q update
+    if ! yum -q install --assumeyes \
+         python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
+    then
         exit 1
     fi
 
-    self=$(readlink -f $0)
-    base=$(dirname $self)
+    mkdir -p /tmp/opts
+    cd /tmp/opts
 
-    createrepo $WORKSPACE/packages/centos6
+    for r in /arvados/packages/centos6/python27-python-*x86_64.rpm ; do
+        rpm2cpio $r | cpio -idm
+    done
 
-    exec docker run \
-         --rm \
-         --volume=$WORKSPACE/packages/centos6:/mnt \
-         --volume=$(readlink -f $0):/root/run-test.sh \
-         --volume=$base/common-test-packages.sh:/root/common-test.sh \
-         --workdir=/mnt \
-         centos:6 \
-         /root/run-test.sh --install-scl
-fi
+    for so in $(find . -name "*.so") ; do
+        echo
+        echo "== Packages dependencies for $so =="
+        ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+    done
 
-if test "$1" = --install-scl ; then
-    yum -q install --assumeyes scl-utils
-    curl -L -O https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum -q install --assumeyes rhscl-python27-epel-6-x86_64.noarch.rpm
-    yum -q install --assumeyes python27
-    exec scl enable python27 $0
+    exec /jenkins/common-test-packages.sh
 fi
-
-cat >/etc/yum.repos.d/localrepo.repo <<EOF
-[localrepo]
-name=Arvados Test
-baseurl=file:///mnt
-gpgcheck=0
-enabled=1
-EOF
-
-yum -q clean all
-yum -q update
-if ! yum -q install --assumeyes \
-     python27-python-arvados-python-client python27-python-arvados-fuse arvados-node-manager
-then
-    exit 1
-fi
-
-mkdir -p /tmp/opts
-cd /tmp/opts
-
-for r in /mnt/python27-python-*x86_64.rpm ; do
-    rpm2cpio $r | cpio -idm
-done
-
-for so in $(find . -name "*.so") ; do
-    echo
-    echo "== Packages dependencies for $so =="
-    ldd $so | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
-done
-
-exec /root/common-test.sh

diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
deleted file mode 100755 (executable)
index c5cbfe6148ad635f3dbf29e2e38e8807439c7bbb..0000000000000000000000000000000000000000
--- a/jenkins/test-packages-debian7.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian7 debian:wheezy

diff --git a/jenkins/test-packages-debian7.sh b/jenkins/test-packages-debian7.sh
new file mode 120000 (symlink)
index 0000000000000000000000000000000000000000..54ce94c357439331502e2926fc95dfed001b9edd
--- /dev/null
+++ b/jenkins/test-packages-debian7.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file

diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
deleted file mode 100755 (executable)
index 5502f46c0c2497e8ffb77d81b7b0f33b0f17a097..0000000000000000000000000000000000000000
--- a/jenkins/test-packages-debian8.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test debian8 debian:jessie

diff --git a/jenkins/test-packages-debian8.sh b/jenkins/test-packages-debian8.sh
new file mode 120000 (symlink)
index 0000000000000000000000000000000000000000..54ce94c357439331502e2926fc95dfed001b9edd
--- /dev/null
+++ b/jenkins/test-packages-debian8.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file

diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
deleted file mode 100755 (executable)
index 00727a0caef4ff6e758cb5fc2d7beee85cf04fe5..0000000000000000000000000000000000000000
--- a/jenkins/test-packages-ubuntu1204.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1204 ubuntu:precise

diff --git a/jenkins/test-packages-ubuntu1204.sh b/jenkins/test-packages-ubuntu1204.sh
new file mode 120000 (symlink)
index 0000000000000000000000000000000000000000..54ce94c357439331502e2926fc95dfed001b9edd
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1204.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file

diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
deleted file mode 100755 (executable)
index bfc2c852186010641471cfb667b41ce90a43712d..0000000000000000000000000000000000000000
--- a/jenkins/test-packages-ubuntu1404.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-exec ./deb-common-test-packages.sh --run-test ubuntu1404 ubuntu:trusty

diff --git a/jenkins/test-packages-ubuntu1404.sh b/jenkins/test-packages-ubuntu1404.sh
new file mode 120000 (symlink)
index 0000000000000000000000000000000000000000..54ce94c357439331502e2926fc95dfed001b9edd
--- /dev/null
+++ b/jenkins/test-packages-ubuntu1404.sh
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file