Arvbox updates, no issue #

* Bump Go version, add openjdk.

* Bump node version.  Make sure node is in path.

* Fix $HOME in setup scripts.

* Add "arvbox devenv" command which creates and enters a container
  environment suitable for interactively running development tools

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 66aebf80d4236d6b575a178641bd5eae8c14a1bc..8c443fd71afd3ddb2aee089df9bcb745b1c3315d 100755 (executable)
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -118,6 +118,23 @@ wait_for_arvbox() {
     fi
 }
 
+docker_run_dev() {
+    docker run \
+          "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
+           "--volume=$SSO_ROOT:/usr/src/sso:rw" \
+           "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
+           "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
+           "--volume=$PG_DATA:/var/lib/postgresql:rw" \
+           "--volume=$VAR_DATA:/var/lib/arvados:rw" \
+           "--volume=$PASSENGER:/var/lib/passenger:rw" \
+           "--volume=$GEMS:/var/lib/gems:rw" \
+           "--volume=$PIPCACHE:/var/lib/pip:rw" \
+           "--volume=$NPMCACHE:/var/lib/npm:rw" \
+           "--volume=$GOSTUFF:/var/lib/gopath:rw" \
+           "--volume=$RLIBS:/var/lib/Rlibs:rw" \
+          "$@"
+}
+
 run() {
     CONFIG=$1
     TAG=$2
@@ -220,22 +237,10 @@ run() {
             mkdir -p $VAR_DATA/test
 
             if test "$need_setup" = 1 ; then
-                docker run \
+                docker_run_dev \
                        --detach \
                        --name=$ARVBOX_CONTAINER \
                        --privileged \
-                       "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
-                       "--volume=$SSO_ROOT:/usr/src/sso:rw" \
-                       "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
-                       "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
-                       "--volume=$PG_DATA:/var/lib/postgresql:rw" \
-                       "--volume=$VAR_DATA:/var/lib/arvados:rw" \
-                       "--volume=$PASSENGER:/var/lib/passenger:rw" \
-                       "--volume=$GEMS:/var/lib/gems:rw" \
-                       "--volume=$PIPCACHE:/var/lib/pip:rw" \
-                       "--volume=$NPMCACHE:/var/lib/npm:rw" \
-                       "--volume=$GOSTUFF:/var/lib/gopath:rw" \
-                       "--volume=$RLIBS:/var/lib/Rlibs:rw" \
                       "--env=SVDIR=/etc/test-service" \
                        arvados/arvbox-dev$TAG
 
@@ -264,22 +269,10 @@ run() {
                    GEM_HOME=/var/lib/gems \
                    "$@"
         elif echo "$CONFIG" | grep 'dev$' ; then
-            docker run \
+            docker_run_dev \
                    --detach \
                    --name=$ARVBOX_CONTAINER \
                    --privileged \
-                   "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
-                   "--volume=$SSO_ROOT:/usr/src/sso:rw" \
-                   "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
-                   "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
-                   "--volume=$PG_DATA:/var/lib/postgresql:rw" \
-                   "--volume=$VAR_DATA:/var/lib/arvados:rw" \
-                   "--volume=$PASSENGER:/var/lib/passenger:rw" \
-                   "--volume=$GEMS:/var/lib/gems:rw" \
-                   "--volume=$PIPCACHE:/var/lib/pip:rw" \
-                   "--volume=$NPMCACHE:/var/lib/npm:rw" \
-                   "--volume=$GOSTUFF:/var/lib/gopath:rw" \
-                   "--volume=$RLIBS:/var/lib/Rlibs:rw" \
                    $PUBLIC \
                    arvados/arvbox-dev$TAG
             updateconf
@@ -494,6 +487,46 @@ case "$subcmd" in
         fi
         ;;
 
+    install-root-cert)
+       set -x
+       sudo cp $VAR_DATA/root-cert.pem /usr/local/share/ca-certificates/${ARVBOX_CONTAINER}-testing-cert.crt
+       sudo update-ca-certificates
+       ;;
+
+    devenv)
+       set -x
+       if docker ps -a --filter "status=exited" | grep -E "${ARVBOX_CONTAINER}-devenv$" -q ; then
+           docker start ${ARVBOX_CONTAINER}-devenv
+       elif ! (docker ps -a --filter "status=running" | grep -E "${ARVBOX_CONTAINER}-devenv$" -q) ; then
+           docker_run_dev \
+                 --detach \
+                --name=${ARVBOX_CONTAINER}-devenv \
+                "--env=SVDIR=/etc/devenv-service" \
+                "--volume=$HOME:$HOME:rw" \
+                --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \
+                arvados/arvbox-dev$TAG
+       fi
+
+       exec docker exec --interactive --tty \
+            -e LINES=$(tput lines) \
+            -e COLUMNS=$(tput cols) \
+            -e TERM=$TERM \
+            -e "ARVBOX_HOME=$HOME" \
+            -e "DISPLAY=$DISPLAY" \
+            --workdir=$PWD \
+            ${ARVBOX_CONTAINER}-devenv \
+            /usr/local/lib/arvbox/devenv.sh "$@"
+       ;;
+
+    devenv-stop)
+       docker stop ${ARVBOX_CONTAINER}-devenv
+       ;;
+
+    devenv-reset)
+       docker stop ${ARVBOX_CONTAINER}-devenv
+       docker rm ${ARVBOX_CONTAINER}-devenv
+       ;;
+
     *)
         echo "Arvados-in-a-box                      http://arvados.org"
         echo

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 162edc927fe04a566422a53aca7735ee1bd31096..2ee493deba6f0961ceb054bad00466b2e07ad508 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -20,7 +20,7 @@ RUN apt-get update && \
     linkchecker python3-virtualenv python-virtualenv xvfb iceweasel \
     libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \
     libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \
-    python3-setuptools python3-pip && \
+    python3-setuptools python3-pip openjdk-8-jdk && \
     apt-get clean
 
 ENV RUBYVERSION_MINOR 2.3
@@ -40,7 +40,7 @@ ENV GEM_HOME /var/lib/gems
 ENV GEM_PATH /var/lib/gems
 ENV PATH $PATH:/var/lib/gems/bin
 
-ENV GOVERSION 1.10.1
+ENV GOVERSION 1.11.5
 
 # Install golang binary
 RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
@@ -79,7 +79,7 @@ RUN set -e && curl -L -f ${GDURL} | tar -C /usr/local/bin -xzf - geckodriver
 
 RUN pip install -U setuptools
 
-ENV NODEVERSION v6.11.4
+ENV NODEVERSION v10.15.1
 
 # Install nodejs binary
 RUN curl -L -f https://nodejs.org/dist/${NODEVERSION}/node-${NODEVERSION}-linux-x64.tar.xz | tar -C /usr/local -xJf - && \
@@ -100,7 +100,7 @@ ADD crunch-setup.sh gitolite.rc \
     keep-setup.sh common.sh createusers.sh \
     logger runsu.sh waitforpostgres.sh \
     yml_override.py api-setup.sh \
-    go-setup.sh \
+    go-setup.sh devenv.sh \
     /usr/local/lib/arvbox/
 
 ADD runit /etc/runit

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
index e6e0397b99a1a3c7095f1b23e4d571edfcb915dd..bb0ff76fe8f065c1be45338f677cf0e7cd99b8ed 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
@@ -13,3 +13,4 @@ RUN echo "development" > /var/lib/arvados/sso_rails_env
 RUN echo "development" > /var/lib/arvados/workbench_rails_env
 
 RUN mkdir /etc/test-service && ln -sf /var/lib/arvbox/service/postgres /etc/test-service
+RUN mkdir /etc/devenv-service
\ No newline at end of file

diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index bbd11f03416a9783904a48cb6823136ceb5c0686..36ff49db51b3b011dc6dea0346c530cb27cb6dd9 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -9,6 +9,7 @@ export GEM_PATH=/var/lib/gems
 export npm_config_cache=/var/lib/npm
 export npm_config_cache_min=Infinity
 export R_LIBS=/var/lib/Rlibs
+export HOME=$(getent passwd arvbox | cut -d: -f6)
 
 if test -s /var/run/localip_override ; then
     localip=$(cat /var/run/localip_override)

diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh
index a4689f004aeebab31c71f0dbe35833ec99df7b89..e9721fd55d87c1e5a597f3a56b632310321bb8d1 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/createusers.sh
+++ b/tools/arvbox/lib/arvbox/docker/createusers.sh
@@ -13,9 +13,13 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
           /var/lib/passenger /var/lib/gopath \
           /var/lib/pip /var/lib/npm
 
+    if test -z "$ARVBOX_HOME" ; then
+       ARVBOX_HOME=/var/lib/arvados
+    fi
+
     groupadd --gid $HOSTGID --non-unique arvbox
     groupadd --gid $HOSTGID --non-unique git
-    useradd --home-dir /var/lib/arvados \
+    useradd --home-dir $ARVBOX_HOME \
             --uid $HOSTUID --gid $HOSTGID \
             --non-unique \
             --groups docker \
@@ -36,6 +40,17 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
     chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
 
     echo "arvbox    ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
+
+    cat <<EOF > /etc/profile.d/paths.sh
+export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/go/bin:/var/lib/gems/bin:$(ls -d /usr/local/node-*)/bin
+export GEM_HOME=/var/lib/gems
+export GEM_PATH=/var/lib/gems
+export npm_config_cache=/var/lib/npm
+export npm_config_cache_min=Infinity
+export R_LIBS=/var/lib/Rlibs
+export GOPATH=/var/lib/gopath
+EOF
+
 fi
 
 if ! grep "^fuse:" /etc/group >/dev/null 2>/dev/null ; then

diff --git a/tools/arvbox/lib/arvbox/docker/devenv.sh b/tools/arvbox/lib/arvbox/docker/devenv.sh
new file mode 100755 (executable)
index 0000000..9ab3ac4
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/devenv.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh
+
+if [[ -n "$*" ]] ; then
+    exec su --preserve-environment arvbox -c "$*"
+else
+    exec su --login arvbox
+fi