X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/aeff8498a0e64e2cbf842e09ac1ad26ca9b8e76c..e97a5d828bdf7cb9626d72efd94d3bcf718a18c9:/tools/arvbox/bin/arvbox diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox index 878119634b..122e2bec7c 100755 --- a/tools/arvbox/bin/arvbox +++ b/tools/arvbox/bin/arvbox @@ -19,11 +19,13 @@ if ! which docker >/dev/null 2>/dev/null ; then fi if test -z "$ARVBOX_DOCKER" ; then + set +e if which greadlink >/dev/null 2>/dev/null ; then ARVBOX_DOCKER=$(greadlink -f $(dirname $0)/../lib/arvbox/docker) else ARVBOX_DOCKER=$(readlink -f $(dirname $0)/../lib/arvbox/docker) fi + set -e fi if test -z "$ARVBOX_CONTAINER" ; then @@ -42,10 +44,6 @@ if test -z "$ARVADOS_ROOT" ; then ARVADOS_ROOT="$ARVBOX_DATA/arvados" fi -if test -z "$SSO_ROOT" ; then - SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider" -fi - if test -z "$COMPOSER_ROOT" ; then COMPOSER_ROOT="$ARVBOX_DATA/composer" fi @@ -97,6 +95,10 @@ EOF fi } +listusers() { + docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list +} + wait_for_arvbox() { FF=/tmp/arvbox-fifo-$$ mkfifo $FF @@ -105,11 +107,11 @@ wait_for_arvbox() { while read line ; do if [[ $line =~ "ok: down: ready:" ]] ; then kill $LOGPID - set +e - wait $LOGPID 2>/dev/null - set -e - else - echo $line + set +e + wait $LOGPID 2>/dev/null + set -e + else + echo $line fi done < $FF rm $FF @@ -123,8 +125,7 @@ wait_for_arvbox() { docker_run_dev() { docker run \ - "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ - "--volume=$SSO_ROOT:/usr/src/sso:rw" \ + "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ "--volume=$PG_DATA:/var/lib/postgresql:rw" \ @@ -135,8 +136,8 @@ docker_run_dev() { "--volume=$NPMCACHE:/var/lib/npm:rw" \ "--volume=$GOSTUFF:/var/lib/gopath:rw" \ "--volume=$RLIBS:/var/lib/Rlibs:rw" \ - --label "org.arvados.arvbox_config=$CONFIG" \ - "$@" + --label "org.arvados.arvbox_config=$CONFIG" \ + "$@" } running_config() { @@ -152,10 +153,10 @@ run() { need_setup=1 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - if [[ $(running_config) != "$CONFIG" ]] ; then - echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" - return 1 - fi + if [[ $(running_config) != "$CONFIG" ]] ; then + echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" + return 1 + fi if test "$CONFIG" = test -o "$CONFIG" = devenv ; then need_setup=0 else @@ -174,12 +175,12 @@ run() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else - if [[ $TAG = '-' ]] ; then - shift - fi + if [[ $TAG = '-' ]] ; then + shift + fi unset TAG fi fi @@ -191,7 +192,8 @@ run() { defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }') localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/') fi - iptemp=$(tempfile) + echo "Public arvbox will use address $localip" + iptemp=$(mktemp) echo $localip > $iptemp chmod og+r $iptemp PUBLIC="--volume=$iptemp:/var/run/localip_override @@ -199,13 +201,13 @@ run() { --publish=3001:3001 --publish=8000:8000 --publish=8900:8900 - --publish=9001:9001 + --publish=9000:9000 --publish=9002:9002 - --publish=25100:25100 - --publish=25107:25107 - --publish=25108:25108 + --publish=25101:25101 --publish=8001:8001 - --publish=8002:8002" + --publish=8002:8002 + --publish=4202:4202 + --publish=45000-45020:45000-45020" else PUBLIC="" fi @@ -226,7 +228,7 @@ run() { --name=$ARVBOX_CONTAINER \ --privileged \ --volumes-from $ARVBOX_CONTAINER-data \ - --label "org.arvados.arvbox_config=$CONFIG" \ + --label "org.arvados.arvbox_config=$CONFIG" \ $PUBLIC \ arvados/arvbox-demo$TAG updateconf @@ -235,16 +237,15 @@ run() { mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF" "$RLIBS" if ! test -d "$ARVADOS_ROOT" ; then - git clone https://github.com/curoverse/arvados.git "$ARVADOS_ROOT" - fi - if ! test -d "$SSO_ROOT" ; then - git clone https://github.com/curoverse/sso-devise-omniauth-provider.git "$SSO_ROOT" + git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT" fi if ! test -d "$COMPOSER_ROOT" ; then - git clone https://github.com/curoverse/composer.git "$COMPOSER_ROOT" + git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT" + git -C "$COMPOSER_ROOT" checkout arvados-fork + git -C "$COMPOSER_ROOT" pull fi if ! test -d "$WORKBENCH2_ROOT" ; then - git clone https://github.com/curoverse/arvados-workbench2.git "$WORKBENCH2_ROOT" + git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT" fi if [[ "$CONFIG" = test ]] ; then @@ -256,7 +257,7 @@ run() { --detach \ --name=$ARVBOX_CONTAINER \ --privileged \ - "--env=SVDIR=/etc/test-service" \ + "--env=SVDIR=/etc/test-service" \ arvados/arvbox-dev$TAG docker exec -ti \ @@ -264,47 +265,49 @@ run() { /usr/local/lib/arvbox/runsu.sh \ /usr/local/lib/arvbox/waitforpostgres.sh - docker exec -ti \ - $ARVBOX_CONTAINER \ - /usr/local/lib/arvbox/runsu.sh \ - /var/lib/arvbox/service/sso/run-service --only-setup - docker exec -ti \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /var/lib/arvbox/service/api/run-service --only-setup fi + interactive="" + if [[ -z "$@" ]] ; then + interactive=--interactive + fi + docker exec -ti \ -e LINES=$(tput lines) \ -e COLUMNS=$(tput cols) \ -e TERM=$TERM \ -e WORKSPACE=/usr/src/arvados \ -e GEM_HOME=/var/lib/gems \ + -e CONFIGSRC=/var/lib/arvados/run_tests \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /usr/src/arvados/build/run-tests.sh \ --temp /var/lib/arvados/test \ + $interactive \ "$@" elif [[ "$CONFIG" = devenv ]] ; then - if [[ $need_setup = 1 ]] ; then - docker_run_dev \ + if [[ $need_setup = 1 ]] ; then + docker_run_dev \ --detach \ - --name=${ARVBOX_CONTAINER} \ - "--env=SVDIR=/etc/devenv-service" \ - "--volume=$HOME:$HOME:rw" \ - --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ - arvados/arvbox-dev$TAG - fi - exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e "ARVBOX_HOME=$HOME" \ - -e "DISPLAY=$DISPLAY" \ - --workdir=$PWD \ - ${ARVBOX_CONTAINER} \ - /usr/local/lib/arvbox/devenv.sh "$@" + --name=${ARVBOX_CONTAINER} \ + "--env=SVDIR=/etc/devenv-service" \ + "--volume=$HOME:$HOME:rw" \ + --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ + arvados/arvbox-dev$TAG + fi + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e "ARVBOX_HOME=$HOME" \ + -e "DISPLAY=$DISPLAY" \ + --workdir=$PWD \ + ${ARVBOX_CONTAINER} \ + /usr/local/lib/arvbox/devenv.sh "$@" elif [[ "$CONFIG" =~ dev$ ]] ; then docker_run_dev \ --detach \ @@ -315,7 +318,12 @@ run() { updateconf wait_for_arvbox echo "The Arvados source code is checked out at: $ARVADOS_ROOT" - echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" + echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" + if [[ "$(listusers)" =~ ^\{\} ]] ; then + echo "No users defined, use 'arvbox adduser' to add user logins" + else + echo "Use 'arvbox listusers' to see user logins" + fi else echo "Unknown configuration '$CONFIG'" fi @@ -329,7 +337,7 @@ update() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else unset TAG @@ -337,9 +345,9 @@ update() { fi if echo "$CONFIG" | grep 'demo$' ; then - docker pull arvados/arvbox-demo$TAG + docker pull arvados/arvbox-demo$TAG else - docker pull arvados/arvbox-dev$TAG + docker pull arvados/arvbox-dev$TAG fi } @@ -413,22 +421,22 @@ case "$subcmd" in sh*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - $ARVBOX_CONTAINER /bin/bash + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + $ARVBOX_CONTAINER /bin/bash ;; ash*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - -u arvbox \ - -w /usr/src/arvados \ - $ARVBOX_CONTAINER /bin/bash --login + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=/var/lib/gems \ + -u arvbox \ + -w /usr/src/arvados \ + $ARVBOX_CONTAINER /bin/bash --login ;; pipe) @@ -455,7 +463,7 @@ case "$subcmd" in update) check $@ stop - update $@ + update $@ run $@ ;; @@ -474,7 +482,7 @@ case "$subcmd" in status) echo "Container: $ARVBOX_CONTAINER" if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - echo "Cluster id: $(getclusterid)" + echo "Cluster id: $(getclusterid)" echo "Status: running" echo "Container IP: $(getip)" echo "Published host: $(gethost)" @@ -500,6 +508,7 @@ case "$subcmd" in exit 1 fi set -x + chmod -R u+w "$ARVBOX_DATA" rm -rf "$ARVBOX_DATA" else if test "$1" != -f ; then @@ -554,7 +563,17 @@ case "$subcmd" in clone) if test -n "$2" ; then - cp -r "$ARVBOX_BASE/$1" "$ARVBOX_BASE/$2" + mkdir -p "$ARVBOX_BASE/$2" + cp -a "$ARVBOX_BASE/$1/passenger" \ + "$ARVBOX_BASE/$1/gems" \ + "$ARVBOX_BASE/$1/pip" \ + "$ARVBOX_BASE/$1/npm" \ + "$ARVBOX_BASE/$1/gopath" \ + "$ARVBOX_BASE/$1/Rlibs" \ + "$ARVBOX_BASE/$1/arvados" \ + "$ARVBOX_BASE/$1/composer" \ + "$ARVBOX_BASE/$1/workbench2" \ + "$ARVBOX_BASE/$2" echo "Created new arvbox $2" echo "export ARVBOX_CONTAINER=$2" else @@ -564,17 +583,65 @@ case "$subcmd" in ;; root-cert) - CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.pem - if test -n "$1" ; then - CERT="$1" - fi - docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" - echo "Certificate copied to $CERT" - ;; + CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt + if test -n "$1" ; then + CERT="$1" + fi + docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" + echo "Certificate copied to $CERT" + ;; psql) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' + ;; + + checkpoint) + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql' + ;; + + restore) + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql' + ;; + + hotreset) + exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - < build arvbox Docker image" echo "reboot stop, build arvbox Docker image, run" echo "rebuild build arvbox Docker image, no layer cache" + echo "checkpoint create database backup" + echo "restore restore checkpoint" + echo "hotreset reset database and restart API without restarting container" echo "reset delete arvbox arvados data (be careful!)" echo "destroy delete all arvbox code and data (be careful!)" echo "log tail log of specified service" @@ -601,7 +671,12 @@ case "$subcmd" in echo "cat get contents of files inside arvbox" echo "pipe run a bash script piped in from stdin" echo "sv " - echo " change state of service inside arvbox" + echo " change state of service inside arvbox" echo "clone clone dev arvbox" + echo "adduser " + echo " add a user login" + echo "removeuser " + echo " remove user login" + echo "listusers list user logins" ;; esac