From c3944344c170a2b1014347cce3ea5388f2b8b79c Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Fri, 18 Sep 2020 17:06:41 -0400 Subject: [PATCH] 16831: arvbox: bugfix: fix bundler version issue in documentation. Upgrade images to Debian Buster. Clean up whitespace in the arvbox command. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- tools/arvbox/bin/arvbox | 216 +++++++++--------- .../arvbox/lib/arvbox/docker/Dockerfile.base | 27 +-- tools/arvbox/lib/arvbox/docker/common.sh | 11 +- tools/arvbox/lib/arvbox/docker/createusers.sh | 1 - .../docker/service/postgres/run-service | 2 +- .../lib/arvbox/docker/service/sdk/run-service | 14 -- 6 files changed, 128 insertions(+), 143 deletions(-) diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox index 279d46c08b..122e2bec7c 100755 --- a/tools/arvbox/bin/arvbox +++ b/tools/arvbox/bin/arvbox @@ -107,11 +107,11 @@ wait_for_arvbox() { while read line ; do if [[ $line =~ "ok: down: ready:" ]] ; then kill $LOGPID - set +e - wait $LOGPID 2>/dev/null - set -e - else - echo $line + set +e + wait $LOGPID 2>/dev/null + set -e + else + echo $line fi done < $FF rm $FF @@ -125,7 +125,7 @@ wait_for_arvbox() { docker_run_dev() { docker run \ - "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ + "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ "--volume=$PG_DATA:/var/lib/postgresql:rw" \ @@ -136,8 +136,8 @@ docker_run_dev() { "--volume=$NPMCACHE:/var/lib/npm:rw" \ "--volume=$GOSTUFF:/var/lib/gopath:rw" \ "--volume=$RLIBS:/var/lib/Rlibs:rw" \ - --label "org.arvados.arvbox_config=$CONFIG" \ - "$@" + --label "org.arvados.arvbox_config=$CONFIG" \ + "$@" } running_config() { @@ -153,10 +153,10 @@ run() { need_setup=1 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - if [[ $(running_config) != "$CONFIG" ]] ; then - echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" - return 1 - fi + if [[ $(running_config) != "$CONFIG" ]] ; then + echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" + return 1 + fi if test "$CONFIG" = test -o "$CONFIG" = devenv ; then need_setup=0 else @@ -175,12 +175,12 @@ run() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else - if [[ $TAG = '-' ]] ; then - shift - fi + if [[ $TAG = '-' ]] ; then + shift + fi unset TAG fi fi @@ -192,7 +192,7 @@ run() { defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }') localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/') fi - echo "Public arvbox will use address $localip" + echo "Public arvbox will use address $localip" iptemp=$(mktemp) echo $localip > $iptemp chmod og+r $iptemp @@ -207,7 +207,7 @@ run() { --publish=8001:8001 --publish=8002:8002 --publish=4202:4202 - --publish=45000-45020:45000-45020" + --publish=45000-45020:45000-45020" else PUBLIC="" fi @@ -228,7 +228,7 @@ run() { --name=$ARVBOX_CONTAINER \ --privileged \ --volumes-from $ARVBOX_CONTAINER-data \ - --label "org.arvados.arvbox_config=$CONFIG" \ + --label "org.arvados.arvbox_config=$CONFIG" \ $PUBLIC \ arvados/arvbox-demo$TAG updateconf @@ -257,7 +257,7 @@ run() { --detach \ --name=$ARVBOX_CONTAINER \ --privileged \ - "--env=SVDIR=/etc/test-service" \ + "--env=SVDIR=/etc/test-service" \ arvados/arvbox-dev$TAG docker exec -ti \ @@ -271,10 +271,10 @@ run() { /var/lib/arvbox/service/api/run-service --only-setup fi - interactive="" - if [[ -z "$@" ]] ; then - interactive=--interactive - fi + interactive="" + if [[ -z "$@" ]] ; then + interactive=--interactive + fi docker exec -ti \ -e LINES=$(tput lines) \ @@ -282,32 +282,32 @@ run() { -e TERM=$TERM \ -e WORKSPACE=/usr/src/arvados \ -e GEM_HOME=/var/lib/gems \ - -e CONFIGSRC=/var/lib/arvados/run_tests \ + -e CONFIGSRC=/var/lib/arvados/run_tests \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /usr/src/arvados/build/run-tests.sh \ --temp /var/lib/arvados/test \ - $interactive \ + $interactive \ "$@" elif [[ "$CONFIG" = devenv ]] ; then - if [[ $need_setup = 1 ]] ; then - docker_run_dev \ + if [[ $need_setup = 1 ]] ; then + docker_run_dev \ --detach \ - --name=${ARVBOX_CONTAINER} \ - "--env=SVDIR=/etc/devenv-service" \ - "--volume=$HOME:$HOME:rw" \ - --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ - arvados/arvbox-dev$TAG - fi - exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e "ARVBOX_HOME=$HOME" \ - -e "DISPLAY=$DISPLAY" \ - --workdir=$PWD \ - ${ARVBOX_CONTAINER} \ - /usr/local/lib/arvbox/devenv.sh "$@" + --name=${ARVBOX_CONTAINER} \ + "--env=SVDIR=/etc/devenv-service" \ + "--volume=$HOME:$HOME:rw" \ + --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ + arvados/arvbox-dev$TAG + fi + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e "ARVBOX_HOME=$HOME" \ + -e "DISPLAY=$DISPLAY" \ + --workdir=$PWD \ + ${ARVBOX_CONTAINER} \ + /usr/local/lib/arvbox/devenv.sh "$@" elif [[ "$CONFIG" =~ dev$ ]] ; then docker_run_dev \ --detach \ @@ -318,12 +318,12 @@ run() { updateconf wait_for_arvbox echo "The Arvados source code is checked out at: $ARVADOS_ROOT" - echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" - if [[ "$(listusers)" =~ ^\{\} ]] ; then - echo "No users defined, use 'arvbox adduser' to add user logins" - else - echo "Use 'arvbox listusers' to see user logins" - fi + echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" + if [[ "$(listusers)" =~ ^\{\} ]] ; then + echo "No users defined, use 'arvbox adduser' to add user logins" + else + echo "Use 'arvbox listusers' to see user logins" + fi else echo "Unknown configuration '$CONFIG'" fi @@ -337,7 +337,7 @@ update() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else unset TAG @@ -345,9 +345,9 @@ update() { fi if echo "$CONFIG" | grep 'demo$' ; then - docker pull arvados/arvbox-demo$TAG + docker pull arvados/arvbox-demo$TAG else - docker pull arvados/arvbox-dev$TAG + docker pull arvados/arvbox-dev$TAG fi } @@ -421,22 +421,22 @@ case "$subcmd" in sh*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - $ARVBOX_CONTAINER /bin/bash + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + $ARVBOX_CONTAINER /bin/bash ;; ash*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - -u arvbox \ - -w /usr/src/arvados \ - $ARVBOX_CONTAINER /bin/bash --login + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + -u arvbox \ + -w /usr/src/arvados \ + $ARVBOX_CONTAINER /bin/bash --login ;; pipe) @@ -463,7 +463,7 @@ case "$subcmd" in update) check $@ stop - update $@ + update $@ run $@ ;; @@ -482,7 +482,7 @@ case "$subcmd" in status) echo "Container: $ARVBOX_CONTAINER" if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - echo "Cluster id: $(getclusterid)" + echo "Cluster id: $(getclusterid)" echo "Status: running" echo "Container IP: $(getip)" echo "Published host: $(gethost)" @@ -563,17 +563,17 @@ case "$subcmd" in clone) if test -n "$2" ; then - mkdir -p "$ARVBOX_BASE/$2" + mkdir -p "$ARVBOX_BASE/$2" cp -a "$ARVBOX_BASE/$1/passenger" \ - "$ARVBOX_BASE/$1/gems" \ - "$ARVBOX_BASE/$1/pip" \ - "$ARVBOX_BASE/$1/npm" \ - "$ARVBOX_BASE/$1/gopath" \ - "$ARVBOX_BASE/$1/Rlibs" \ - "$ARVBOX_BASE/$1/arvados" \ - "$ARVBOX_BASE/$1/composer" \ - "$ARVBOX_BASE/$1/workbench2" \ - "$ARVBOX_BASE/$2" + "$ARVBOX_BASE/$1/gems" \ + "$ARVBOX_BASE/$1/pip" \ + "$ARVBOX_BASE/$1/npm" \ + "$ARVBOX_BASE/$1/gopath" \ + "$ARVBOX_BASE/$1/Rlibs" \ + "$ARVBOX_BASE/$1/arvados" \ + "$ARVBOX_BASE/$1/composer" \ + "$ARVBOX_BASE/$1/workbench2" \ + "$ARVBOX_BASE/$2" echo "Created new arvbox $2" echo "export ARVBOX_CONTAINER=$2" else @@ -583,28 +583,28 @@ case "$subcmd" in ;; root-cert) - CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt - if test -n "$1" ; then - CERT="$1" - fi - docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" - echo "Certificate copied to $CERT" - ;; + CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt + if test -n "$1" ; then + CERT="$1" + fi + docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" + echo "Certificate copied to $CERT" + ;; psql) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' + ;; checkpoint) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql' + ;; restore) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql' + ;; hotreset) - exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - < build arvbox Docker image" echo "reboot stop, build arvbox Docker image, run" echo "rebuild build arvbox Docker image, no layer cache" - echo "checkpoint create database backup" - echo "restore restore checkpoint" - echo "hotreset reset database and restart API without restarting container" + echo "checkpoint create database backup" + echo "restore restore checkpoint" + echo "hotreset reset database and restart API without restarting container" echo "reset delete arvbox arvados data (be careful!)" echo "destroy delete all arvbox code and data (be careful!)" echo "log tail log of specified service" @@ -671,12 +671,12 @@ EOF echo "cat get contents of files inside arvbox" echo "pipe run a bash script piped in from stdin" echo "sv " - echo " change state of service inside arvbox" + echo " change state of service inside arvbox" echo "clone clone dev arvbox" - echo "adduser " - echo " add a user login" - echo "removeuser " - echo " remove user login" - echo "listusers list user logins" + echo "adduser " + echo " add a user login" + echo "removeuser " + echo " remove user login" + echo "listusers list user logins" ;; esac diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base index 815db22b4c..eb52ca5a78 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base @@ -2,17 +2,17 @@ # # SPDX-License-Identifier: AGPL-3.0 -FROM debian:9 +FROM debian:10 ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ - postgresql-9.6 postgresql-contrib-9.6 git build-essential runit curl libpq-dev \ - libcurl4-openssl-dev libssl1.0-dev zlib1g-dev libpcre3-dev libpam-dev \ + postgresql postgresql-contrib git build-essential runit curl libpq-dev \ + libcurl4-openssl-dev libssl-dev zlib1g-dev libpcre3-dev libpam-dev \ openssh-server netcat-traditional \ graphviz bzip2 less sudo virtualenv \ - libpython-dev fuse libfuse-dev \ + fuse libfuse-dev \ pkg-config libattr1-dev \ libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \ libjson-perl nginx gitolite3 lsof libreadline-dev \ @@ -20,14 +20,12 @@ RUN apt-get update && \ linkchecker python3-virtualenv python3-venv xvfb iceweasel \ libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \ libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \ - python3-setuptools python3-pip openjdk-8-jdk bsdmainutils net-tools \ - ruby2.3 ruby-dev bundler shellinabox && \ - apt-get remove -yq libpython-dev libpython-stdlib libpython2.7 libpython2.7-dev \ - libpython2.7-minimal libpython2.7-stdlib python2.7-minimal python2.7 && \ + python3-setuptools python3-pip default-jdk-headless bsdmainutils net-tools \ + ruby ruby-dev bundler shellinabox && \ apt-get clean -ENV RUBYVERSION_MINOR 2.3 -ENV RUBYVERSION 2.3.5 +ENV RUBYVERSION_MINOR 2.5 +ENV RUBYVERSION 2.5.1 # Install Ruby from source # RUN cd /tmp && \ @@ -40,10 +38,9 @@ ENV RUBYVERSION 2.3.5 # rm -rf ruby-${RUBYVERSION} ENV GEM_HOME /var/lib/gems -ENV GEM_PATH /var/lib/gems ENV PATH $PATH:/var/lib/gems/bin -ENV GOVERSION 1.13.6 +ENV GOVERSION 1.15.2 # Install golang binary RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \ @@ -60,9 +57,9 @@ RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \ rm -f /tmp/8D81803C0EBFCD88.asc RUN mkdir -p /etc/apt/sources.list.d && \ - echo deb https://download.docker.com/linux/debian/ stretch stable > /etc/apt/sources.list.d/docker.list && \ + echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \ apt-get update && \ - apt-get -yq --no-install-recommends install docker-ce=17.06.0~ce-0~debian && \ + apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \ apt-get clean RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql @@ -117,4 +114,4 @@ ADD runit /etc/runit # Start the supervisor. ENV SVDIR /etc/service STOPSIGNAL SIGINT -CMD ["/sbin/runit"] +CMD ["/etc/runit/2"] diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh index e81e8108e2..185467cf7d 100644 --- a/tools/arvbox/lib/arvbox/docker/common.sh +++ b/tools/arvbox/lib/arvbox/docker/common.sh @@ -2,10 +2,9 @@ # # SPDX-License-Identifier: AGPL-3.0 - +export DEBIAN_FRONTEND=noninteractive export PATH=${PATH}:/usr/local/go/bin:/var/lib/gems/bin export GEM_HOME=/var/lib/gems -export GEM_PATH=/var/lib/gems export npm_config_cache=/var/lib/npm export npm_config_cache_min=Infinity export R_LIBS=/var/lib/Rlibs @@ -60,6 +59,10 @@ fi run_bundler() { if test -f Gemfile.lock ; then + # The 'gem install bundler line below' is cf. + # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html, + # until we get bundler 2.7.10/3.0.0 or higher + gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')" frozen=--frozen else frozen="" @@ -73,8 +76,8 @@ run_bundler() { # flock /var/lib/gems/gems.lock gem install --verbose --no-document bundler --version ${bundleversion} # fi # fi - if ! flock /var/lib/gems/gems.lock bundler install --verbose --path $GEM_HOME --local --no-deployment $frozen "$@" ; then - flock /var/lib/gems/gems.lock bundler install --verbose --path $GEM_HOME --no-deployment $frozen "$@" + if ! flock /var/lib/gems/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then + flock /var/lib/gems/gems.lock bundler install --verbose --no-deployment $frozen "$@" fi } diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh index 58fb413582..de1e7bba96 100755 --- a/tools/arvbox/lib/arvbox/docker/createusers.sh +++ b/tools/arvbox/lib/arvbox/docker/createusers.sh @@ -46,7 +46,6 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then cat < /etc/profile.d/paths.sh export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/go/bin:/var/lib/gems/bin:$(ls -d /usr/local/node-*)/bin export GEM_HOME=/var/lib/gems -export GEM_PATH=/var/lib/gems export npm_config_cache=/var/lib/npm export npm_config_cache_min=Infinity export R_LIBS=/var/lib/Rlibs diff --git a/tools/arvbox/lib/arvbox/docker/service/postgres/run-service b/tools/arvbox/lib/arvbox/docker/service/postgres/run-service index a0771aa6a0..f2377a0c2d 100755 --- a/tools/arvbox/lib/arvbox/docker/service/postgres/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/postgres/run-service @@ -6,7 +6,7 @@ exec 2>&1 set -eux -o pipefail -PGVERSION=9.6 +PGVERSION=11 if ! test -d /var/lib/postgresql/$PGVERSION/main ; then /usr/lib/postgresql/$PGVERSION/bin/initdb --locale=en_US.UTF-8 -D /var/lib/postgresql/$PGVERSION/main diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service index 1ec225ca12..d66bf315b1 100755 --- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service @@ -20,20 +20,6 @@ ln -sf /usr/src/arvados/sdk/cli/binstubs/arv /usr/local/bin/arv export PYCMD=python3 -# Need to install the upstream version of pip because the python-pip package -# shipped with Debian 9 is patched to change behavior in a way that breaks our -# use case. -# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=876145 -# When a non-root user attempts to install system packages, it makes the -# --ignore-installed flag the default (and there is no way to turn it off), -# this has the effect of making it very hard to share dependencies shared among -# multiple packages, because it will blindly install the latest version of each -# dependency requested by each package, even if a compatible package version is -# already installed. -if ! pip3 install --no-index --find-links /var/lib/pip pip==9.0.3 ; then - pip3 install pip==9.0.3 -fi - pip_install wheel cd /usr/src/arvados/sdk/python -- 2.30.2