X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/51c1daf863f3e1920f758f73b4e5d70ff2c706d6..4de0821a28d54153c6046655d4a2d8f57da7e005:/tools/arvbox/bin/arvbox diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox index ada53a2009..7d45ba17c1 100755 --- a/tools/arvbox/bin/arvbox +++ b/tools/arvbox/bin/arvbox @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 @@ -19,11 +19,13 @@ if ! which docker >/dev/null 2>/dev/null ; then fi if test -z "$ARVBOX_DOCKER" ; then + set +e if which greadlink >/dev/null 2>/dev/null ; then ARVBOX_DOCKER=$(greadlink -f $(dirname $0)/../lib/arvbox/docker) else ARVBOX_DOCKER=$(readlink -f $(dirname $0)/../lib/arvbox/docker) fi + set -e fi if test -z "$ARVBOX_CONTAINER" ; then @@ -42,10 +44,6 @@ if test -z "$ARVADOS_ROOT" ; then ARVADOS_ROOT="$ARVBOX_DATA/arvados" fi -if test -z "$SSO_ROOT" ; then - SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider" -fi - if test -z "$COMPOSER_ROOT" ; then COMPOSER_ROOT="$ARVBOX_DATA/composer" fi @@ -64,7 +62,7 @@ GOSTUFF="$ARVBOX_DATA/gopath" RLIBS="$ARVBOX_DATA/Rlibs" getip() { - docker inspect $ARVBOX_CONTAINER | grep \"IPAddress\" | head -n1 | tr -d ' ":,\n' | cut -c10- + docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER } gethost() { @@ -103,9 +101,13 @@ wait_for_arvbox() { docker logs -f $ARVBOX_CONTAINER > $FF & LOGPID=$! while read line ; do - echo $line - if echo $line | grep "Workbench2 is running at" >/dev/null ; then + if [[ $line =~ "ok: down: ready:" ]] ; then kill $LOGPID + set +e + wait $LOGPID 2>/dev/null + set -e + else + echo $line fi done < $FF rm $FF @@ -117,6 +119,27 @@ wait_for_arvbox() { fi } +docker_run_dev() { + docker run \ + "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ + "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ + "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ + "--volume=$PG_DATA:/var/lib/postgresql:rw" \ + "--volume=$VAR_DATA:/var/lib/arvados:rw" \ + "--volume=$PASSENGER:/var/lib/passenger:rw" \ + "--volume=$GEMS:/var/lib/gems:rw" \ + "--volume=$PIPCACHE:/var/lib/pip:rw" \ + "--volume=$NPMCACHE:/var/lib/npm:rw" \ + "--volume=$GOSTUFF:/var/lib/gopath:rw" \ + "--volume=$RLIBS:/var/lib/Rlibs:rw" \ + --label "org.arvados.arvbox_config=$CONFIG" \ + "$@" +} + +running_config() { + docker inspect $ARVBOX_CONTAINER -f '{{index .Config.Labels "org.arvados.arvbox_config"}}' +} + run() { CONFIG=$1 TAG=$2 @@ -126,18 +149,22 @@ run() { need_setup=1 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - if test "$CONFIG" = test ; then + if [[ $(running_config) != "$CONFIG" ]] ; then + echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" + return 1 + fi + if test "$CONFIG" = test -o "$CONFIG" = devenv ; then need_setup=0 else echo "Container $ARVBOX_CONTAINER is already running" - exit 0 + return 0 fi fi if test $need_setup = 1 ; then if docker ps -a | grep -E "$ARVBOX_CONTAINER$" -q ; then echo "Container $ARVBOX_CONTAINER already exists but is not running; use restart or reboot" - exit 1 + return 1 fi fi @@ -147,18 +174,22 @@ run() { TAG=":$TAG" shift else + if [[ $TAG = '-' ]] ; then + shift + fi unset TAG fi fi - if echo "$CONFIG" | grep '^public' ; then + if [[ "$CONFIG" =~ ^public ]] ; then if test -n "$ARVBOX_PUBLISH_IP" ; then localip=$ARVBOX_PUBLISH_IP else defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }') localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/') fi - iptemp=$(tempfile) + echo "Public arvbox will use address $localip" + iptemp=$(mktemp) echo $localip > $iptemp chmod og+r $iptemp PUBLIC="--volume=$iptemp:/var/run/localip_override @@ -166,21 +197,19 @@ run() { --publish=3001:3001 --publish=8000:8000 --publish=8900:8900 - --publish=9001:9001 + --publish=9000:9000 --publish=9002:9002 - --publish=25100:25100 - --publish=25107:25107 - --publish=25108:25108 + --publish=25101:25101 --publish=8001:8001 --publish=8002:8002" else PUBLIC="" fi - if echo "$CONFIG" | grep 'demo$' ; then + if [[ "$CONFIG" =~ demo$ ]] ; then if test -d "$ARVBOX_DATA" ; then echo "It looks like you already have a development container named $ARVBOX_CONTAINER." - echo "Set ARVBOX_CONTAINER to set a different name for your demo container" + echo "Set environment variable ARVBOX_CONTAINER to set a different name for your demo container" exit 1 fi @@ -193,6 +222,7 @@ run() { --name=$ARVBOX_CONTAINER \ --privileged \ --volumes-from $ARVBOX_CONTAINER-data \ + --label "org.arvados.arvbox_config=$CONFIG" \ $PUBLIC \ arvados/arvbox-demo$TAG updateconf @@ -200,41 +230,27 @@ run() { else mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF" "$RLIBS" - if ! test -d "$ARVADOS_ROOT" ; then - git clone https://github.com/curoverse/arvados.git "$ARVADOS_ROOT" - fi - if ! test -d "$SSO_ROOT" ; then - git clone https://github.com/curoverse/sso-devise-omniauth-provider.git "$SSO_ROOT" + git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT" fi if ! test -d "$COMPOSER_ROOT" ; then - git clone https://github.com/curoverse/composer.git "$COMPOSER_ROOT" + git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT" + git -C "$COMPOSER_ROOT" checkout arvados-fork + git -C "$COMPOSER_ROOT" pull fi if ! test -d "$WORKBENCH2_ROOT" ; then - git clone https://github.com/curoverse/arvados-workbench2.git "$WORKBENCH2_ROOT" + git clone https://github.com/arvados/arvados-workbench2.git "$WORKBENCH2_ROOT" fi - if test "$CONFIG" = test ; then + if [[ "$CONFIG" = test ]] ; then mkdir -p $VAR_DATA/test if test "$need_setup" = 1 ; then - docker run \ + docker_run_dev \ --detach \ --name=$ARVBOX_CONTAINER \ --privileged \ - "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ - "--volume=$SSO_ROOT:/usr/src/sso:rw" \ - "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ - "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ - "--volume=$PG_DATA:/var/lib/postgresql:rw" \ - "--volume=$VAR_DATA:/var/lib/arvados:rw" \ - "--volume=$PASSENGER:/var/lib/passenger:rw" \ - "--volume=$GEMS:/var/lib/gems:rw" \ - "--volume=$PIPCACHE:/var/lib/pip:rw" \ - "--volume=$NPMCACHE:/var/lib/npm:rw" \ - "--volume=$GOSTUFF:/var/lib/gopath:rw" \ - "--volume=$RLIBS:/var/lib/Rlibs:rw" \ "--env=SVDIR=/etc/test-service" \ arvados/arvbox-dev$TAG @@ -243,42 +259,54 @@ run() { /usr/local/lib/arvbox/runsu.sh \ /usr/local/lib/arvbox/waitforpostgres.sh - docker exec -ti \ - $ARVBOX_CONTAINER \ - /usr/local/lib/arvbox/runsu.sh \ - /var/lib/arvbox/service/sso/run-service --only-setup - docker exec -ti \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /var/lib/arvbox/service/api/run-service --only-setup fi + interactive="" + if [[ -z "$@" ]] ; then + interactive=--interactive + fi + docker exec -ti \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e WORKSPACE=/usr/src/arvados \ + -e GEM_HOME=/var/lib/gems \ + -e CONFIGSRC=/var/lib/arvados/run_tests \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /usr/src/arvados/build/run-tests.sh \ --temp /var/lib/arvados/test \ - WORKSPACE=/usr/src/arvados \ - GEM_HOME=/var/lib/gems \ + $interactive \ "$@" - elif echo "$CONFIG" | grep 'dev$' ; then - docker run \ + elif [[ "$CONFIG" = devenv ]] ; then + if [[ $need_setup = 1 ]] ; then + docker_run_dev \ + --detach \ + --name=${ARVBOX_CONTAINER} \ + "--env=SVDIR=/etc/devenv-service" \ + "--volume=$HOME:$HOME:rw" \ + --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ + arvados/arvbox-dev$TAG + fi + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e "ARVBOX_HOME=$HOME" \ + -e "DISPLAY=$DISPLAY" \ + --workdir=$PWD \ + ${ARVBOX_CONTAINER} \ + /usr/local/lib/arvbox/devenv.sh "$@" + elif [[ "$CONFIG" =~ dev$ ]] ; then + docker_run_dev \ --detach \ --name=$ARVBOX_CONTAINER \ --privileged \ - "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ - "--volume=$SSO_ROOT:/usr/src/sso:rw" \ - "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ - "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ - "--volume=$PG_DATA:/var/lib/postgresql:rw" \ - "--volume=$VAR_DATA:/var/lib/arvados:rw" \ - "--volume=$PASSENGER:/var/lib/passenger:rw" \ - "--volume=$GEMS:/var/lib/gems:rw" \ - "--volume=$PIPCACHE:/var/lib/pip:rw" \ - "--volume=$NPMCACHE:/var/lib/npm:rw" \ - "--volume=$GOSTUFF:/var/lib/gopath:rw" \ - "--volume=$RLIBS:/var/lib/Rlibs:rw" \ $PUBLIC \ arvados/arvbox-dev$TAG updateconf @@ -291,6 +319,27 @@ run() { fi } +update() { + CONFIG=$1 + TAG=$2 + + if test -n "$TAG" + then + if test $(echo $TAG | cut -c1-1) != '-' ; then + TAG=":$TAG" + shift + else + unset TAG + fi + fi + + if echo "$CONFIG" | grep 'demo$' ; then + docker pull arvados/arvbox-demo$TAG + else + docker pull arvados/arvbox-dev$TAG + fi +} + stop() { if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then docker stop $ARVBOX_CONTAINER @@ -329,11 +378,11 @@ build() { check() { case "$1" in - localdemo|publicdemo|dev|publicdev|test) + localdemo|publicdemo|dev|publicdev|test|devenv) true ;; *) - echo "Argument to $subcmd must be one of localdemo, publicdemo, dev, publicdev, test" + echo "Argument to $subcmd must be one of localdemo, publicdemo, dev, publicdev, test, devenv" exit 1 ;; esac @@ -360,7 +409,23 @@ case "$subcmd" in ;; sh*) - exec docker exec -ti -e LINES=$(tput lines) -e COLUMNS=$(tput cols) -e TERM=$TERM -e GEM_HOME=/var/lib/gems $ARVBOX_CONTAINER /bin/bash + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + $ARVBOX_CONTAINER /bin/bash + ;; + + ash*) + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + -u arvbox \ + -w /usr/src/arvados \ + $ARVBOX_CONTAINER /bin/bash --login ;; pipe) @@ -384,6 +449,13 @@ case "$subcmd" in run $@ ;; + update) + check $@ + stop + update $@ + run $@ + ;; + ip) getip ;; @@ -425,6 +497,7 @@ case "$subcmd" in exit 1 fi set -x + chmod -R u+w "$ARVBOX_DATA" rm -rf "$ARVBOX_DATA" else if test "$1" != -f ; then @@ -479,7 +552,17 @@ case "$subcmd" in clone) if test -n "$2" ; then - cp -r "$ARVBOX_BASE/$1" "$ARVBOX_BASE/$2" + mkdir -p "$ARVBOX_BASE/$2" + cp -a "$ARVBOX_BASE/$1/passenger" \ + "$ARVBOX_BASE/$1/gems" \ + "$ARVBOX_BASE/$1/pip" \ + "$ARVBOX_BASE/$1/npm" \ + "$ARVBOX_BASE/$1/gopath" \ + "$ARVBOX_BASE/$1/Rlibs" \ + "$ARVBOX_BASE/$1/arvados" \ + "$ARVBOX_BASE/$1/composer" \ + "$ARVBOX_BASE/$1/workbench2" \ + "$ARVBOX_BASE/$2" echo "Created new arvbox $2" echo "export ARVBOX_CONTAINER=$2" else @@ -488,27 +571,82 @@ case "$subcmd" in fi ;; + root-cert) + CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt + if test -n "$1" ; then + CERT="$1" + fi + docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" + echo "Certificate copied to $CERT" + ;; + + psql) + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' + ;; + + checkpoint) + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql' + ;; + + restore) + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql' + ;; + + hotreset) + exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - < build arvbox Docker image" - echo "rebuild build arvbox Docker image, no layer cache" - echo "start|run [tag] start $ARVBOX_CONTAINER container" - echo "open open arvbox workbench in a web browser" - echo "shell enter arvbox shell" - echo "ip print arvbox docker container ip address" - echo "host print arvbox published host" - echo "status print some information about current arvbox" - echo "stop stop arvbox container" - echo "restart stop, then run again" - echo "reboot stop, build arvbox Docker image, run" - echo "reset delete arvbox arvados data (be careful!)" - echo "destroy delete all arvbox code and data (be careful!)" - echo "log tail log of specified service" - echo "ls list directories inside arvbox" - echo "cat get contents of files inside arvbox" - echo "pipe run a bash script piped in from stdin" - echo "sv change state of service inside arvbox" - echo "clone clone an arvbox" + echo "start|run [tag] start $ARVBOX_CONTAINER container" + echo "stop stop arvbox container" + echo "restart stop, then run again" + echo "status print some information about current arvbox" + echo "ip print arvbox docker container ip address" + echo "host print arvbox published host" + echo "shell enter shell as root" + echo "ashell enter shell as 'arvbox'" + echo "psql enter postgres console" + echo "open open arvbox workbench in a web browser" + echo "root-cert get copy of root certificate" + echo "update stop, pull latest image, run" + echo "build build arvbox Docker image" + echo "reboot stop, build arvbox Docker image, run" + echo "rebuild build arvbox Docker image, no layer cache" + echo "checkpoint create database backup" + echo "restore restore checkpoint" + echo "hotreset reset database and restart API without restarting container" + echo "reset delete arvbox arvados data (be careful!)" + echo "destroy delete all arvbox code and data (be careful!)" + echo "log tail log of specified service" + echo "ls list directories inside arvbox" + echo "cat get contents of files inside arvbox" + echo "pipe run a bash script piped in from stdin" + echo "sv " + echo " change state of service inside arvbox" + echo "clone clone dev arvbox" ;; esac