From 0ddb70d7350801a56c1754f02975c95ee2687c1a Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Thu, 20 Dec 2018 11:51:39 -0500 Subject: [PATCH] 9945: first set of changes. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- .../centos7/Dockerfile | 2 +- .../debian8/Dockerfile | 2 +- .../debian9/Dockerfile | 5 +- .../ubuntu1404/Dockerfile | 2 +- .../ubuntu1604/Dockerfile | 2 +- .../ubuntu1804/Dockerfile | 2 +- build/python-package-scripts/after-install.sh | 50 +++++++++++++++++++ build/python-package-scripts/before-remove.sh | 50 +++++++++++++++++++ build/run-build-packages.sh | 44 ++-------------- services/dockercleaner/README.rst | 5 ++ .../dockercleaner/bin/arvados-docker-cleaner | 9 ++++ tools/crunchstat-summary/README.rst | 5 ++ tools/crunchstat-summary/setup.py | 2 +- 13 files changed, 134 insertions(+), 46 deletions(-) create mode 100755 build/python-package-scripts/after-install.sh create mode 100755 build/python-package-scripts/before-remove.sh create mode 100644 services/dockercleaner/README.rst create mode 100755 services/dockercleaner/bin/arvados-docker-cleaner create mode 100644 tools/crunchstat-summary/README.rst diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile index ba616eef1a..20a8871279 100644 --- a/build/package-build-dockerfiles/centos7/Dockerfile +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -6,7 +6,7 @@ FROM centos:7 MAINTAINER Ward Vandewege # Install dependencies. -RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git +RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git python-virtualenv # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile index 1244e3f55d..134cece799 100644 --- a/build/package-build-dockerfiles/debian8/Dockerfile +++ b/build/package-build-dockerfiles/debian8/Dockerfile @@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive # Install dependencies. -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile index 42094d53bb..9674c0cf26 100644 --- a/build/package-build-dockerfiles/debian9/Dockerfile +++ b/build/package-build-dockerfiles/debian9/Dockerfile @@ -9,7 +9,10 @@ MAINTAINER Nico Cesar ENV DEBIAN_FRONTEND noninteractive # Install dependencies. -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev + +# Install virtualenv +RUN /usr/bin/pip install virtualenv virtualenv-tools # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile index a6bf65bc11..798680e48d 100644 --- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile @@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive # Install dependencies. -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip python3-venv python3-dev # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile index 17bf89f3b6..b9e966c745 100644 --- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile @@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive # Install dependencies. -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile index c66de46908..3891df3935 100644 --- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile @@ -8,7 +8,7 @@ MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive # Install dependencies. -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata python3-venv python3-dev # Install RVM ADD generated/rvm.asc /tmp/ diff --git a/build/python-package-scripts/after-install.sh b/build/python-package-scripts/after-install.sh new file mode 100755 index 0000000000..8851f28ec4 --- /dev/null +++ b/build/python-package-scripts/after-install.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +set -e + +# Detect rpm-based systems: the exit code of the following command is zero +# on rpm-based systems +if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then + # Red Hat ("%{...}" is interpolated at package build time) + pkg="%{name}" + pkgtype=rpm + prefix="${RPM_INSTALL_PREFIX}" +else + # Debian + script="$(basename "${0}")" + pkg="${script%.postinst}" + pkgtype=deb + prefix=/usr +fi + +# populated from the build script +# dash only supports one array, $@ +if [ "%FPM_BINARIES" != "" ]; then + set %FPM_BINARIES +fi + +# Install symlinks to the binary/binaries +if [ "$pkg" != "" ]; then + + if [ "%FPM_BINARIES" != "" ]; then + # read from $@ + for binary do + if [ -e /usr/bin/$binary ]; then + rm -f /usr/bin/$binary + fi + ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/$binary + done + fi + + # special case for arvados-cwl-runner + if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then + if [ -e /usr/bin/cwl-runner ]; then + rm -f /usr/bin/cwl-runner + fi + ln -s /usr/share/%PYTHON/dist/$pkg/bin/$binary /usr/bin/cwl-runner + fi +fi + diff --git a/build/python-package-scripts/before-remove.sh b/build/python-package-scripts/before-remove.sh new file mode 100755 index 0000000000..2ea15ee705 --- /dev/null +++ b/build/python-package-scripts/before-remove.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +set -e + +# Detect rpm-based systems: the exit code of the following command is zero +# on rpm-based systems +if /usr/bin/rpm -q -f /usr/bin/rpm >/dev/null 2>&1; then + # Red Hat ("%{...}" is interpolated at package build time) + pkg="%{name}" + pkgtype=rpm + prefix="${RPM_INSTALL_PREFIX}" +else + # Debian + script="$(basename "${0}")" + pkg="${script%.prerm}" + pkgtype=deb + prefix=/usr +fi + +# populated from the build script +# dash only supports one array, $@ +if [ "%FPM_BINARIES" != "" ]; then + set %FPM_BINARIES +fi + +if [ "$pkg" != "" ]; then + # Remove the binary python files so the package manager doesn't throw warnings + # on removing the package. + find /usr/share/%PYTHON/dist/$pkg -iname *.pyc -exec rm {} \; || true + find /usr/share/%PYTHON/dist/$pkg -iname *.pyo -exec rm {} \; || true + + if [ "%FPM_BINARIES" != "" ]; then + # read from $@ + for binary do + if [ -L /usr/bin/$binary ]; then + # Remove the symlinks we installed + rm -f /usr/bin/$binary + fi + done + fi + + if [ "${pkg#python-}" = "arvados-cwl-runner" ]; then + if [ -L /usr/bin/cwl-runner ]; then + rm -f /usr/bin/cwl-runner + fi + fi +fi diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 73e2e62863..efc20b234d 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -348,38 +348,12 @@ for i in ${deps[@]}; do fi done - # The Python SDK -# Please resist the temptation to add --no-python-fix-name to the fpm call here -# (which would remove the python- prefix from the package name), because this -# package is a dependency of arvados-fuse, and fpm can not omit the python- -# prefix from only one of the dependencies of a package... Maybe I could -# whip up a patch and send it upstream, but that will be for another day. Ward, -# 2014-05-15 -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/sdk/python/build" -arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)} -test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python -if [[ "$?" == "0" ]]; then +fpm_build_virtualenv "arvados-python-client" "sdk/python" +fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" - fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" -fi - -# cwl-runner -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/sdk/cwl/build" -arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)} -declare -a iterargs=() -if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then - arvados_cwl_runner_iteration=4 - iterargs+=(--iteration $arvados_cwl_runner_iteration) -else - arvados_cwl_runner_iteration= -fi -test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration" -if [[ "$?" == "0" ]]; then - fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" "${iterargs[@]}" -fi +# Arvadow cwl runner +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" # schema_salad. This is a python dependency of arvados-cwl-runner, # but we can't use the usual PYTHONPACKAGES way to build this package due to the @@ -421,15 +395,7 @@ if [[ $TARGET =~ debian|ubuntu ]]; then fi # The FUSE driver -# Please see comment about --no-python-fix-name above; we stay consistent and do -# not omit the python- prefix first. -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/services/fuse/build" -arvados_fuse_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)} -test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python -if [[ "$?" == "0" ]]; then - fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools" -fi +fpm_build_virtualenv "arvados-fuse" "services/fuse" # The node manager cd $WORKSPACE/packages/$TARGET diff --git a/services/dockercleaner/README.rst b/services/dockercleaner/README.rst new file mode 100644 index 0000000000..dd2b7e9cf3 --- /dev/null +++ b/services/dockercleaner/README.rst @@ -0,0 +1,5 @@ +.. Copyright (C) The Arvados Authors. All rights reserved. +.. +.. SPDX-License-Identifier: Apache-2.0 + +Arvados Docker Cleaner. diff --git a/services/dockercleaner/bin/arvados-docker-cleaner b/services/dockercleaner/bin/arvados-docker-cleaner new file mode 100755 index 0000000000..c00593fbf1 --- /dev/null +++ b/services/dockercleaner/bin/arvados-docker-cleaner @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +from __future__ import absolute_import, print_function + +from arvados_docker.cleaner import main +main() diff --git a/tools/crunchstat-summary/README.rst b/tools/crunchstat-summary/README.rst new file mode 100644 index 0000000000..fa8e1bd3db --- /dev/null +++ b/tools/crunchstat-summary/README.rst @@ -0,0 +1,5 @@ +.. Copyright (C) The Arvados Authors. All rights reserved. +.. +.. SPDX-License-Identifier: Apache-2.0 + +Arvados Crunchstat Summary. diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py index 04adba8494..642428181c 100755 --- a/tools/crunchstat-summary/setup.py +++ b/tools/crunchstat-summary/setup.py @@ -23,7 +23,7 @@ if '--short-tests-only' in sys.argv: setup(name='crunchstat_summary', version=version, - description='read crunch log files and summarize resource usage', + description='Arvados crunchstat-summary reads crunch log files and summarizes resource usage', author='Arvados', author_email='info@arvados.org', url="https://arvados.org", -- 2.30.2