9945: first set of changes.

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege@veritasgenetics.com>

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index ba616eef1aeefcf36c67edfaa6cc62dc6226f008..20a8871279823e3be199311f258d5a59d79ee928 100644 (file)
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -6,7 +6,7 @@ FROM centos:7
 MAINTAINER Ward Vandewege <ward@curoverse.com>
 
 # Install dependencies.
 MAINTAINER Ward Vandewege <ward@curoverse.com>
 
 # Install dependencies.

-RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git python-virtualenv

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 1244e3f55d4ac244742f9a3bfb803dde9b7aca2d..134cece799f271601186801714b0e8cc448908e6 100644 (file)
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.

-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
index 42094d53bb8a4d982afce90b5a65bd28dd3fd763..9674c0cf2641680f09f6d40433569300b429034e 100644 (file)
--- a/build/package-build-dockerfiles/debian9/Dockerfile
+++ b/build/package-build-dockerfiles/debian9/Dockerfile
@@ -9,7 +9,10 @@ MAINTAINER Nico Cesar <nico@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.

-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev
+
+# Install virtualenv
+RUN /usr/bin/pip install virtualenv virtualenv-tools

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index a6bf65bc11a84c12b32c3231c33257ff28be9c65..798680e48d6309fe281227ccf6e1fc21d767ef2d 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.

-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip 
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
index 17bf89f3b64a253a0421846974801f2ebc6ffbf0..b9e966c7457ee904be73b216737d4c9c54db9794 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.

-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
index c66de46908daef3de6700c9b790917fde6c08e85..3891df3935261754ceb9daf4b636f772e237ffd9 100644 (file)
--- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install dependencies.

-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev

 
 # Install RVM
 ADD generated/rvm.asc /tmp/
 
 # Install RVM
 ADD generated/rvm.asc /tmp/

diff --git a/build/python-package-scripts/after-install.sh b/build/python-package-scripts/after-install.sh
new file mode 100755 (executable)
index 0000000..8851f28
--- /dev/null
+++ b/build/python-package-scripts/after-install.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+# Detect rpm-based systems: the exit code of the following command is zero
+# on rpm-based systems
+if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then
+    # Red Hat ("%{...}" is interpolated at package build time)
+    pkg="%{name}"
+    pkgtype=rpm
+    prefix="${RPM_INSTALL_PREFIX}"
+else
+    # Debian
+    script="$(basename "${0}")"
+    pkg="${script%.postinst}"
+    pkgtype=deb
+    prefix=/usr
+fi
+
+# populated from the build script
+# dash only supports one array, $@
+if [ "%FPM_BINARIES" != "" ]; then
+  set %FPM_BINARIES
+fi
+
+# Install symlinks to the binary/binaries
+if [ "$pkg" != "" ]; then
+
+  if [ "%FPM_BINARIES" != "" ]; then
+    # read from $@
+    for binary do
+      if [ -e /usr/bin/$binary ]; then
+        rm -f /usr/bin/$binary
+      fi
+       ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/$binary
+    done
+  fi
+
+  # special case for arvados-cwl-runner
+  if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then
+    if [ -e /usr/bin/cwl-runner ]; then
+      rm -f /usr/bin/cwl-runner
+    fi
+    ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/cwl-runner
+  fi
+fi
+

diff --git a/build/python-package-scripts/before-remove.sh b/build/python-package-scripts/before-remove.sh
new file mode 100755 (executable)
index 0000000..2ea15ee
--- /dev/null
+++ b/build/python-package-scripts/before-remove.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+# Detect rpm-based systems: the exit code of the following command is zero
+# on rpm-based systems
+if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then
+    # Red Hat ("%{...}" is interpolated at package build time)
+    pkg="%{name}"
+    pkgtype=rpm
+    prefix="${RPM_INSTALL_PREFIX}"
+else
+    # Debian
+    script="$(basename "${0}")"
+    pkg="${script%.prerm}"
+    pkgtype=deb
+    prefix=/usr
+fi
+
+# populated from the build script
+# dash only supports one array, $@
+if [ "%FPM_BINARIES" != "" ]; then
+  set %FPM_BINARIES
+fi
+
+if [ "$pkg" != "" ]; then
+  # Remove the binary python files so the package manager doesn't throw warnings
+  # on removing the package.
+  find /usr/share/%PYTHON/dist/$pkg -iname *.pyc -exec rm {} \; || true
+  find /usr/share/%PYTHON/dist/$pkg -iname *.pyo -exec rm {} \; || true
+
+  if [ "%FPM_BINARIES" != "" ]; then
+    # read from $@
+    for binary do
+      if [ -L /usr/bin/$binary ]; then
+        # Remove the symlinks we installed
+        rm -f /usr/bin/$binary
+      fi
+    done
+  fi
+
+  if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then
+    if [ -L /usr/bin/cwl-runner ]; then
+      rm -f /usr/bin/cwl-runner
+    fi
+  fi
+fi

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 73e2e628631bbc71ea84388b37e6ae4d60917734..efc20b234df4ec25b0f1eb5483eb4a6317624672 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -348,38 +348,12 @@ for i in ${deps[@]}; do
   fi
 done
 
   fi
 done
 

-

 # The Python SDK
 # The Python SDK

-# Please resist the temptation to add --no-python-fix-name to the fpm call here
-# (which would remove the python- prefix from the package name), because this
-# package is a dependency of arvados-fuse, and fpm can not omit the python-
-# prefix from only one of the dependencies of a package...  Maybe I could
-# whip up a patch and send it upstream, but that will be for another day. Ward,
-# 2014-05-15
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/python/build"
-arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)}
-test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
-if [[ "$?" == "0" ]]; then
+fpm_build_virtualenv "arvados-python-client" "sdk/python"
+fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"

 
 

-  fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git  --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
-fi
-
-# cwl-runner
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/sdk/cwl/build"
-arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)}
-declare -a iterargs=()
-if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
-    arvados_cwl_runner_iteration=4
-    iterargs+=(--iteration $arvados_cwl_runner_iteration)
-else
-    arvados_cwl_runner_iteration=
-fi
-test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
-if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" "${iterargs[@]}"
-fi
+# Arvadow cwl runner
+fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"

 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
 # but we can't use the usual PYTHONPACKAGES way to build this package due to the


@@ -421,15 +395,7 @@ if [[ $TARGET =~ debian|ubuntu ]]; then
 fi
 
 # The FUSE driver
 fi
 
 # The FUSE driver

-# Please see comment about --no-python-fix-name above; we stay consistent and do
-# not omit the python- prefix first.
-cd $WORKSPACE/packages/$TARGET
-rm -rf "$WORKSPACE/services/fuse/build"
-arvados_fuse_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)}
-test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python
-if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
-fi
+fpm_build_virtualenv "arvados-fuse" "services/fuse"

 
 # The node manager
 cd $WORKSPACE/packages/$TARGET
 
 # The node manager
 cd $WORKSPACE/packages/$TARGET

diff --git a/services/dockercleaner/README.rst b/services/dockercleaner/README.rst
new file mode 100644 (file)
index 0000000..dd2b7e9
--- /dev/null
+++ b/services/dockercleaner/README.rst
@@ -0,0 +1,5 @@
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: Apache-2.0
+
+Arvados Docker Cleaner.

diff --git a/services/dockercleaner/bin/arvados-docker-cleaner b/services/dockercleaner/bin/arvados-docker-cleaner
new file mode 100755 (executable)
index 0000000..c00593f
--- /dev/null
+++ b/services/dockercleaner/bin/arvados-docker-cleaner
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+from __future__ import absolute_import, print_function
+
+from arvados_docker.cleaner import main
+main()

diff --git a/tools/crunchstat-summary/README.rst b/tools/crunchstat-summary/README.rst
new file mode 100644 (file)
index 0000000..fa8e1bd
--- /dev/null
+++ b/tools/crunchstat-summary/README.rst
@@ -0,0 +1,5 @@
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: Apache-2.0
+
+Arvados Crunchstat Summary.

diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 04adba8494e20e5151b972ac7b7a3f547e09dd44..642428181c685858fa1b6ee4d4861c6836e3e211 100755 (executable)
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -23,7 +23,7 @@ if '--short-tests-only' in sys.argv:
 
 setup(name='crunchstat_summary',
       version=version,
 
 setup(name='crunchstat_summary',
       version=version,

-      description='read crunch log files and summarize resource usage',
+      description='Arvados crunchstat-summary reads crunch log files and summarizes resource usage',

       author='Arvados',
       author_email='info@arvados.org',
       url="https://arvados.org",
       author='Arvados',
       author_email='info@arvados.org',
       url="https://arvados.org",