projects / arvados.git / commitdiff
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 28209cb)
14853: we need a newer version of setuptools to build on centos7, debian8 and ubuntu1...
authorWard Vandewege <wvandewege@veritasgenetics.com>
Wed, 20 Feb 2019 22:04:42 +0000 (17:04 -0500)
committerWard Vandewege <wvandewege@veritasgenetics.com>
Wed, 20 Feb 2019 22:42:11 +0000 (17:42 -0500)
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege@veritasgenetics.com>

build/package-build-dockerfiles/centos7/Dockerfile patch | blob | history
build/package-build-dockerfiles/debian8/Dockerfile patch | blob | history
build/package-build-dockerfiles/debian9/Dockerfile patch | blob | history
build/package-build-dockerfiles/ubuntu1404/Dockerfile patch | blob | history
build/package-build-dockerfiles/ubuntu1604/Dockerfile patch | blob | history
build/package-build-dockerfiles/ubuntu1804/Dockerfile patch | blob | history
build/run-build-packages.sh patch | blob | history
build/run-library.sh patch | blob | history

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 522189a20cd92625652a4ee38d9c2f02222135b4..ad6f4e1e8f5051c8bb90e449eb89873ca13ea107 100644 (file)
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -32,14 +32,14 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 RUN touch /var/lib/rpm/* && yum -q -y install rh-python35
 RUN scl enable rh-python35 "easy_install-3.5 pip" && easy_install-2.7 pip
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 # Add epel, we need it for the python-pam dependency
 RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 RUN rpm -ivh epel-release-latest-7.noarch.rpm
 
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
+# The version of setuptools that comes with CentOS is way too old
+RUN pip install --upgrade setuptools
+
 ENV WORKSPACE /arvados
 CMD ["scl", "enable", "rh-python35", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 1e6c62ce7ea72fba1380260bb2c7c07dbc19c87c..3f591cdfa14aceab1ff1b4be1c650192d396131f 100644 (file)
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
 
 # Install virtualenv
 RUN /usr/bin/pip install virtualenv
@@ -32,9 +32,6 @@ RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
 RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index fcaf0bc0a1bb74cb134847eed3cbde9fc14f0c80..6f7f3faafe8c4e18f580be1f8e341a978cf38c84 100644 (file)
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -9,7 +9,7 @@ MAINTAINER Nico Cesar <nico@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
 
 # Install virtualenv
 RUN /usr/bin/pip install virtualenv
@@ -33,9 +33,6 @@ RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
 RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index a88435a5b4bb0e98fd646f7ede5e03a45c5490bd..4c01c9e8180fefb268ff80e9d930f339144eb879 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3.4-venv python3.4-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3.4-venv python3.4-dev
 
 # Install virtualenv
 RUN /usr/bin/pip install virtualenv
@@ -32,9 +32,6 @@ RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
 RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index be3911eff3e94ee6907efc883a8da047a4622ae4..a83fc77132315651dfd432877c7fecb5b4090f4e 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev
 
 # Install virtualenv
 RUN /usr/bin/pip install virtualenv
@@ -32,9 +32,6 @@ RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
 RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index 8995d14be0bdc55cbdeb47c005c2323f648b6bc4..d0a099911ce91ac9c0e7892d75072310c91e710e 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -32,9 +32,6 @@ RUN ln -s /usr/local/go/bin/go /usr/local/bin/
 ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
 RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
-# Old versions of setuptools cannot build a schema-salad package.
-RUN pip install --upgrade setuptools
-
 RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
 
 ENV WORKSPACE /arvados
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ef1df03d5a97498e0ba15b02dd16cb012a8ba3c8..6264e93f0f1e9b3d2d6634de35a76a0a55fd2588 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -248,21 +248,6 @@ handle_ruby_gem arvados-login-sync
 # Python packages
 debug_echo -e "\nPython packages\n"
 
-cd "$WORKSPACE/sdk/pam"
-handle_python_package
-
-cd "$WORKSPACE/sdk/python"
-handle_python_package
-
-cd "$WORKSPACE/sdk/cwl"
-handle_python_package
-
-cd "$WORKSPACE/services/fuse"
-handle_python_package
-
-cd "$WORKSPACE/services/nodemanager"
-handle_python_package
-
 # arvados-src
 (
     cd "$WORKSPACE"
diff --git a/build/run-library.sh b/build/run-library.sh
index 0442abc4ac7f38b81592440b8bd0d479479dd649..8bf2166d5e2a3c29e46494cba956f1cec47eeeb4 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -381,16 +381,23 @@ fpm_build_virtualenv () {
 
   local python=""
   case "$PACKAGE_TYPE" in
+    python3)
+        python=python3
+        if [[ "$FORMAT" != "rpm" ]]; then
+          pip=pip3
+        else
+          # In CentOS, we use a different mechanism to get the right version of pip
+          pip=pip
+        fi
+        PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
+        ;;
     python)
         # All Arvados Python2 packages depend on Python 2.7.
         # Make sure we build with that for consistency.
         python=python2.7
+        pip=pip
         PACKAGE_PREFIX=$PYTHON2_PKG_PREFIX
         ;;
-    python3)
-        PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
-        python=python3
-        ;;
   esac
 
   if [[ "$PKG" != "libpam-arvados" ]] &&
@@ -410,8 +417,14 @@ fpm_build_virtualenv () {
 
   rm -rf dist/*
 
+  # Get the latest setuptools
+  if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+    echo "Error, unable to upgrade setuptools with"
+    echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+    exit 1
+  fi
   if ! $python setup.py $DASHQ_UNLESS_DEBUG sdist; then
-    echo "Error, unable to run python setup.py sdist for $PKG"
+    echo "Error, unable to run $python setup.py sdist for $PKG"
     exit 1
   fi
 
@@ -446,33 +459,33 @@ fpm_build_virtualenv () {
     exit 1
   fi
 
-  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip; then
     echo "Error, unable to upgrade pip with"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U pip"
     exit 1
   fi
-  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
-    echo "Error, unable to upgrade wheel with"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+    echo "Error, unable to upgrade setuptools with"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
     exit 1
   fi
-  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
-    echo "Error, unable to upgrade setuptools with"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
+  if ! build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel; then
+    echo "Error, unable to upgrade wheel with"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U wheel"
     exit 1
   fi
 
   if [[ "$TARGET" != "centos7" ]] || [[ "$PYTHON_PKG" != "python-arvados-fuse" ]]; then
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
   else
     # centos7 needs these special tweaks to install python-arvados-fuse
-    build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
-    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
+    build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG docutils
+    PYCURL_SSL_LIBRARY=nss build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH
   fi
 
   if [[ "$?" != "0" ]]; then
     echo "Error, unable to run"
-    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
+    echo "  build/usr/share/$python/dist/$PYTHON_PKG/bin/$pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG $PACKAGE_PATH"
     exit 1
   fi