From: Ward Vandewege Date: Tue, 19 Nov 2019 21:27:57 +0000 (-0500) Subject: 15839: add Debian Buster packages. X-Git-Tag: 2.0.0~122^2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/5ad819e0aa69a8ea453abe18a8e00aa93daa90a2 15839: add Debian Buster packages. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile new file mode 100644 index 0000000000..bab447f538 --- /dev/null +++ b/build/package-build-dockerfiles/debian10/Dockerfile @@ -0,0 +1,38 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries +FROM debian:buster +MAINTAINER Ward Vandewege + +ENV DEBIAN_FRONTEND noninteractive + +# Install dependencies. +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip python3-venv python3-dev + +# Install virtualenv +RUN /usr/bin/pip install virtualenv + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.5 && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + +# Install golang binary +ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/ +RUN ln -s /usr/local/go/bin/go /usr/local/bin/ + +# Install nodejs and npm +ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/ +RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/ + +RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle + +ENV WORKSPACE /arvados +CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian10"] diff --git a/build/package-test-dockerfiles/debian10/Dockerfile b/build/package-test-dockerfiles/debian10/Dockerfile new file mode 100644 index 0000000000..3aa6fdcce1 --- /dev/null +++ b/build/package-test-dockerfiles/debian10/Dockerfile @@ -0,0 +1,26 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +FROM debian:buster +MAINTAINER Ward Vandewege + +ENV DEBIAN_FRONTEND noninteractive + +# Install dependencies +RUN apt-get update && \ + apt-get -y install --no-install-recommends curl ca-certificates gpg procps gpg-agent + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.5 && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.5 + +# udev daemon can't start in a container, so don't try. +RUN mkdir -p /etc/udev/disabled + +RUN echo "deb file:///arvados/packages/debian10/ /" >>/etc/apt/sources.list diff --git a/build/package-testing/deb-common-test-packages.sh b/build/package-testing/deb-common-test-packages.sh index 77017ba970..32fb2009e1 100755 --- a/build/package-testing/deb-common-test-packages.sh +++ b/build/package-testing/deb-common-test-packages.sh @@ -23,7 +23,7 @@ export ARV_PACKAGES_DIR="/arvados/packages/$target" dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before" -apt-get $DASHQQ_UNLESS_DEBUG update +apt-get $DASHQQ_UNLESS_DEBUG --allow-insecure-repositories update apt-get $DASHQQ_UNLESS_DEBUG -y --allow-unauthenticated install "$1" >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG" diff --git a/build/package-testing/test-packages-debian10.sh b/build/package-testing/test-packages-debian10.sh new file mode 120000 index 0000000000..54ce94c357 --- /dev/null +++ b/build/package-testing/test-packages-debian10.sh @@ -0,0 +1 @@ +deb-common-test-packages.sh \ No newline at end of file diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index 378c9bbfa3..7f3ca3242b 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -10,7 +10,7 @@ Syntax: WORKSPACE=/path/to/arvados $(basename $0) [options] --target - Distribution to build packages for (default: debian9) + Distribution to build packages for (default: debian10) --command Build command to execute (default: use built-in Docker image command) --test-packages @@ -21,6 +21,9 @@ Syntax: Build only a specific package --only-test Test only a specific package +--force-build + Build even if the package exists upstream or if it has already been + built locally --force-test Test even if there is no new untested package --build-version @@ -51,15 +54,16 @@ if ! [[ -d "$WORKSPACE" ]]; then fi PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,build-version: \ + help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,build-version: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 fi -TARGET=debian9 +TARGET=debian10 +FORCE_BUILD=0 COMMAND= -DEBUG= +DEBUG=${ARVADOS_DEBUG:-0} eval set -- "$PARSEDOPTS" while [ $# -gt 0 ]; do @@ -80,6 +84,9 @@ while [ $# -gt 0 ]; do --force-test) FORCE_TEST=true ;; + --force-build) + FORCE_BUILD=1 + ;; --only-build) ONLY_BUILD="$2"; shift ;; @@ -269,6 +276,7 @@ else --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \ --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "ONLY_BUILD=$ONLY_BUILD" \ + --env "FORCE_BUILD=$FORCE_BUILD" \ "$IMAGE" $COMMAND then echo diff --git a/build/run-build-packages-sso.sh b/build/run-build-packages-sso.sh index e7a3aacda3..d8d9b984a0 100755 --- a/build/run-build-packages-sso.sh +++ b/build/run-build-packages-sso.sh @@ -17,7 +17,7 @@ Options: --debug Output debug information (default: false) --target - Distribution to build packages for (default: debian9) + Distribution to build packages for (default: debian10) WORKSPACE=path Path to the Arvados SSO source tree to build packages from @@ -25,7 +25,7 @@ EOF EXITCODE=0 DEBUG=${ARVADOS_DEBUG:-0} -TARGET=debian9 +TARGET=debian10 PARSEDOPTS=$(getopt --name "$0" --longoptions \ help,build-bundle-packages,debug,target: \ diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index a07b308179..d6c8f5ac64 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -19,9 +19,12 @@ Options: --debug Output debug information (default: false) --target - Distribution to build packages for (default: debian9) + Distribution to build packages for (default: debian10) --only-build Build only a specific package (or $ONLY_BUILD from environment) +--force-build + Build even if the package exists upstream or if it has already been + built locally --command Build command to execute (defaults to the run command defined in the Docker image) @@ -41,12 +44,13 @@ VENDOR="Veritas Genetics, Inc." # End of user configuration DEBUG=${ARVADOS_DEBUG:-0} +FORCE_BUILD=${FORCE_BUILD:-0} EXITCODE=0 -TARGET=debian9 +TARGET=debian10 COMMAND= PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,build-bundle-packages,debug,target:,only-build: \ + help,build-bundle-packages,debug,target:,only-build:,force-build \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -66,6 +70,9 @@ while [ $# -gt 0 ]; do --only-build) ONLY_BUILD="$2"; shift ;; + --force-build) + FORCE_BUILD=1 + ;; --debug) DEBUG=1 ;; @@ -157,14 +164,6 @@ if [[ "$?" != 0 ]]; then exit 1 fi -PYTHON2_FPM_INSTALLER=(--python-easyinstall "$(find_python_program easy_install-$PYTHON2_VERSION easy_install)") -install3=$(find_python_program easy_install-$PYTHON3_VERSION easy_install3 pip-$PYTHON3_VERSION pip3) -if [[ $install3 =~ easy_ ]]; then - PYTHON3_FPM_INSTALLER=(--python-easyinstall "$install3") -else - PYTHON3_FPM_INSTALLER=(--python-pip "$install3") -fi - RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`" RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh index d75e2785ec..8539ec4f0a 100755 --- a/build/run-build-test-packages-one-target.sh +++ b/build/run-build-test-packages-one-target.sh @@ -10,7 +10,7 @@ Syntax: WORKSPACE=/path/to/arvados $(basename $0) [options] --target - Distribution to build packages for (default: debian9) + Distribution to build packages for (default: debian10) --upload If the build and test steps are successful, upload the packages to a remote apt repository (default: false) @@ -50,7 +50,7 @@ if [ $? -ne 0 ]; then exit 1 fi -TARGET=debian9 +TARGET=debian10 UPLOAD=0 RC=0 DEBUG= diff --git a/build/run-library.sh b/build/run-library.sh index a4cebbc8a7..f173504c58 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -316,7 +316,9 @@ test_package_presence() { # sure it gets picked up by the test and/or upload steps. # Get the list of packages from the repos - if [[ "$FORMAT" == "deb" ]]; then + if [[ "$FORCE_BUILD" == "1" ]]; then + echo "Package $full_pkgname build forced with --force-build, building" + elif [[ "$FORMAT" == "deb" ]]; then declare -A dd dd[debian9]=stretch dd[debian10]=buster