2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
8 if ! test -d /sys/fs/cgroup ; then
9 echo "Arvbox requires cgroups to be mounted at /sys/fs/cgroup in order to use"
10 echo "Docker-in-Docker. Older operating systems that put cgroups in other"
11 echo "places (such as /cgroup) are not supported."
15 if ! which docker >/dev/null 2>/dev/null ; then
16 echo "Arvbox requires Docker. To install, run the following command as root:"
17 echo "curl -sSL https://get.docker.com/ | sh"
21 if test -z "$ARVBOX_DOCKER" ; then
23 if which greadlink >/dev/null 2>/dev/null ; then
24 ARVBOX_DOCKER=$(greadlink -f $(dirname $0)/../lib/arvbox/docker)
26 ARVBOX_DOCKER=$(readlink -f $(dirname $0)/../lib/arvbox/docker)
31 if test -z "$ARVBOX_CONTAINER" ; then
32 ARVBOX_CONTAINER=arvbox
35 if test -z "$ARVBOX_BASE" ; then
36 ARVBOX_BASE="$HOME/.arvbox"
39 if test -z "$ARVBOX_DATA" ; then
40 ARVBOX_DATA="$ARVBOX_BASE/$ARVBOX_CONTAINER"
43 if test -z "$ARVADOS_ROOT" ; then
44 ARVADOS_ROOT="$ARVBOX_DATA/arvados"
47 if test -z "$COMPOSER_ROOT" ; then
48 COMPOSER_ROOT="$ARVBOX_DATA/composer"
51 if test -z "$WORKBENCH2_ROOT" ; then
52 WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
55 PG_DATA="$ARVBOX_DATA/postgres"
56 VAR_DATA="$ARVBOX_DATA/var"
57 PASSENGER="$ARVBOX_DATA/passenger"
58 GEMS="$ARVBOX_DATA/gems"
59 PIPCACHE="$ARVBOX_DATA/pip"
60 NPMCACHE="$ARVBOX_DATA/npm"
61 GOSTUFF="$ARVBOX_DATA/gopath"
62 RLIBS="$ARVBOX_DATA/Rlibs"
65 docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
70 OVERRIDE=$(docker exec -i $ARVBOX_CONTAINER cat /var/run/localip_override 2>/dev/null)
73 if test "$CODE" = 0 ; then
81 docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/api_uuid_prefix
85 if test -f ~/.config/arvados/$ARVBOX_CONTAINER.conf ; then
86 sed "s/ARVADOS_API_HOST=.*/ARVADOS_API_HOST=$(gethost):8000/" <$HOME/.config/arvados/$ARVBOX_CONTAINER.conf >$HOME/.config/arvados/$ARVBOX_CONTAINER.conf.tmp
87 mv ~/.config/arvados/$ARVBOX_CONTAINER.conf.tmp ~/.config/arvados/$ARVBOX_CONTAINER.conf
89 mkdir -p $HOME/.config/arvados
90 cat >$HOME/.config/arvados/$ARVBOX_CONTAINER.conf <<EOF
91 ARVADOS_API_HOST=$(gethost):8000
93 ARVADOS_API_HOST_INSECURE=true
99 FF=/tmp/arvbox-fifo-$$
101 docker logs -f $ARVBOX_CONTAINER > $FF &
104 if [[ $line =~ "ok: down: ready:" ]] ; then
107 wait $LOGPID 2>/dev/null
115 if test -n "$localip" ; then
116 echo "export ARVADOS_API_HOST=$localip:8000"
118 echo "export ARVADOS_API_HOST=$(gethost):8000"
124 "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
125 "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
126 "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
127 "--volume=$PG_DATA:/var/lib/postgresql:rw" \
128 "--volume=$VAR_DATA:/var/lib/arvados:rw" \
129 "--volume=$PASSENGER:/var/lib/passenger:rw" \
130 "--volume=$GEMS:/var/lib/gems:rw" \
131 "--volume=$PIPCACHE:/var/lib/pip:rw" \
132 "--volume=$NPMCACHE:/var/lib/npm:rw" \
133 "--volume=$GOSTUFF:/var/lib/gopath:rw" \
134 "--volume=$RLIBS:/var/lib/Rlibs:rw" \
135 --label "org.arvados.arvbox_config=$CONFIG" \
140 docker inspect $ARVBOX_CONTAINER -f '{{index .Config.Labels "org.arvados.arvbox_config"}}'
151 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
152 if [[ $(running_config) != "$CONFIG" ]] ; then
153 echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot"
156 if test "$CONFIG" = test -o "$CONFIG" = devenv ; then
159 echo "Container $ARVBOX_CONTAINER is already running"
164 if test $need_setup = 1 ; then
165 if docker ps -a | grep -E "$ARVBOX_CONTAINER$" -q ; then
166 echo "Container $ARVBOX_CONTAINER already exists but is not running; use restart or reboot"
173 if test $(echo $TAG | cut -c1-1) != '-' ; then
177 if [[ $TAG = '-' ]] ; then
184 if [[ "$CONFIG" =~ ^public ]] ; then
185 if test -n "$ARVBOX_PUBLISH_IP" ; then
186 localip=$ARVBOX_PUBLISH_IP
188 defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
189 localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
191 echo "Public arvbox will use address $localip"
193 echo $localip > $iptemp
195 PUBLIC="--volume=$iptemp:/var/run/localip_override
202 --publish=25101:25101
209 if [[ "$CONFIG" =~ demo$ ]] ; then
210 if test -d "$ARVBOX_DATA" ; then
211 echo "It looks like you already have a development container named $ARVBOX_CONTAINER."
212 echo "Set environment variable ARVBOX_CONTAINER to set a different name for your demo container"
216 if ! (docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q) ; then
217 docker create -v /var/lib/postgresql -v /var/lib/arvados --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true
222 --name=$ARVBOX_CONTAINER \
224 --volumes-from $ARVBOX_CONTAINER-data \
225 --label "org.arvados.arvbox_config=$CONFIG" \
227 arvados/arvbox-demo$TAG
231 mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF" "$RLIBS"
233 if ! test -d "$ARVADOS_ROOT" ; then
234 git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
236 if ! test -d "$COMPOSER_ROOT" ; then
237 git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
238 git -C "$COMPOSER_ROOT" checkout arvados-fork
239 git -C "$COMPOSER_ROOT" pull
241 if ! test -d "$WORKBENCH2_ROOT" ; then
242 git clone https://github.com/arvados/arvados-workbench2.git "$WORKBENCH2_ROOT"
245 if [[ "$CONFIG" = test ]] ; then
247 mkdir -p $VAR_DATA/test
249 if test "$need_setup" = 1 ; then
252 --name=$ARVBOX_CONTAINER \
254 "--env=SVDIR=/etc/test-service" \
255 arvados/arvbox-dev$TAG
259 /usr/local/lib/arvbox/runsu.sh \
260 /usr/local/lib/arvbox/waitforpostgres.sh
264 /usr/local/lib/arvbox/runsu.sh \
265 /var/lib/arvbox/service/api/run-service --only-setup
269 if [[ -z "$@" ]] ; then
270 interactive=--interactive
274 -e LINES=$(tput lines) \
275 -e COLUMNS=$(tput cols) \
277 -e WORKSPACE=/usr/src/arvados \
278 -e GEM_HOME=/var/lib/gems \
279 -e CONFIGSRC=/var/lib/arvados/run_tests \
281 /usr/local/lib/arvbox/runsu.sh \
282 /usr/src/arvados/build/run-tests.sh \
283 --temp /var/lib/arvados/test \
286 elif [[ "$CONFIG" = devenv ]] ; then
287 if [[ $need_setup = 1 ]] ; then
290 --name=${ARVBOX_CONTAINER} \
291 "--env=SVDIR=/etc/devenv-service" \
292 "--volume=$HOME:$HOME:rw" \
293 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \
294 arvados/arvbox-dev$TAG
296 exec docker exec --interactive --tty \
297 -e LINES=$(tput lines) \
298 -e COLUMNS=$(tput cols) \
300 -e "ARVBOX_HOME=$HOME" \
301 -e "DISPLAY=$DISPLAY" \
303 ${ARVBOX_CONTAINER} \
304 /usr/local/lib/arvbox/devenv.sh "$@"
305 elif [[ "$CONFIG" =~ dev$ ]] ; then
308 --name=$ARVBOX_CONTAINER \
311 arvados/arvbox-dev$TAG
314 echo "The Arvados source code is checked out at: $ARVADOS_ROOT"
315 echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem"
317 echo "Unknown configuration '$CONFIG'"
328 if test $(echo $TAG | cut -c1-1) != '-' ; then
336 if echo "$CONFIG" | grep 'demo$' ; then
337 docker pull arvados/arvbox-demo$TAG
339 docker pull arvados/arvbox-dev$TAG
344 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
345 docker stop $ARVBOX_CONTAINER
348 VOLUMES=--volumes=true
349 if docker ps -a --filter "status=created" | grep -E "$ARVBOX_CONTAINER$" -q ; then
350 docker rm $VOLUMES $ARVBOX_CONTAINER
352 if docker ps -a --filter "status=exited" | grep -E "$ARVBOX_CONTAINER$" -q ; then
353 docker rm $VOLUMES $ARVBOX_CONTAINER
358 if ! test -f "$ARVBOX_DOCKER/Dockerfile.base" ; then
359 echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
362 if docker --version |grep " 1\.[0-9]\." ; then
363 # Docker version prior 1.10 require -f flag
364 # -f flag removed in Docker 1.12
367 GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD)
368 docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
369 docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
370 if test "$1" = localdemo -o "$1" = publicdemo ; then
371 docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
372 docker tag $FORCE arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest
374 docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
375 docker tag $FORCE arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest
381 localdemo|publicdemo|dev|publicdev|test|devenv)
385 echo "Argument to $subcmd must be one of localdemo, publicdemo, dev, publicdev, test, devenv"
392 if test -n "$subcmd" ; then
403 NO_CACHE=--no-cache build $@
412 exec docker exec --interactive --tty \
413 -e LINES=$(tput lines) \
414 -e COLUMNS=$(tput cols) \
416 -e GEM_HOME=/var/lib/gems \
417 $ARVBOX_CONTAINER /bin/bash
421 exec docker exec --interactive --tty \
422 -e LINES=$(tput lines) \
423 -e COLUMNS=$(tput cols) \
425 -e GEM_HOME=/var/lib/gems \
427 -w /usr/src/arvados \
428 $ARVBOX_CONTAINER /bin/bash --login
432 exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash -
468 exec xdg-open http://$(gethost)
472 echo "Container: $ARVBOX_CONTAINER"
473 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
474 echo "Cluster id: $(getclusterid)"
475 echo "Status: running"
476 echo "Container IP: $(getip)"
477 echo "Published host: $(gethost)"
479 echo "Status: not running"
481 if test -d "$ARVBOX_DATA" ; then
482 echo "Data: $ARVBOX_DATA"
483 elif docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q ; then
484 echo "Data: $ARVBOX_CONTAINER-data"
492 if test -d "$ARVBOX_DATA" ; then
493 if test "$subcmd" = destroy ; then
494 if test "$1" != -f ; then
495 echo "WARNING! This will delete your entire arvbox ($ARVBOX_DATA)."
496 echo "Use destroy -f if you really mean it."
500 chmod -R u+w "$ARVBOX_DATA"
501 rm -rf "$ARVBOX_DATA"
503 if test "$1" != -f ; then
504 echo "WARNING! This will delete your arvbox data ($ARVBOX_DATA)."
505 echo "Code and downloaded packages will be preserved."
506 echo "Use reset -f if you really mean it."
510 rm -rf "$ARVBOX_DATA/postgres"
511 rm -rf "$ARVBOX_DATA/var"
514 if test "$1" != -f ; then
515 echo "WARNING! This will delete your data container $ARVBOX_CONTAINER-data. Use -f if you really mean it."
519 docker rm "$ARVBOX_CONTAINER-data"
524 if test -n "$1" ; then
525 exec docker exec -ti -e LINES=$(tput lines) -e COLUMNS=$(tput cols) -e TERM=$TERM $ARVBOX_CONTAINER less --follow-name -R +GF "/etc/service/$1/log/main/current"
527 exec docker exec -ti $ARVBOX_CONTAINER tail $(docker exec -ti $ARVBOX_CONTAINER find -L /etc -path '/etc/service/*/log/main/current' -printf " %p")
532 if test -n "$1" ; then
533 exec docker exec $ARVBOX_CONTAINER cat "$@"
535 echo "Usage: $0 $subcmd <files>"
540 exec docker exec -ti $ARVBOX_CONTAINER /usr/bin/env TERM=$TERM ls "$@"
544 if test -n "$1" -a -n "$2" ; then
545 exec docker exec $ARVBOX_CONTAINER sv "$@"
547 echo "Usage: $0 $subcmd <start|stop|restart> <service>"
548 echo "Available services:"
549 exec docker execa $ARVBOX_CONTAINER ls /etc/service
554 if test -n "$2" ; then
555 mkdir -p "$ARVBOX_BASE/$2"
556 cp -a "$ARVBOX_BASE/$1/passenger" \
557 "$ARVBOX_BASE/$1/gems" \
558 "$ARVBOX_BASE/$1/pip" \
559 "$ARVBOX_BASE/$1/npm" \
560 "$ARVBOX_BASE/$1/gopath" \
561 "$ARVBOX_BASE/$1/Rlibs" \
562 "$ARVBOX_BASE/$1/arvados" \
563 "$ARVBOX_BASE/$1/composer" \
564 "$ARVBOX_BASE/$1/workbench2" \
566 echo "Created new arvbox $2"
567 echo "export ARVBOX_CONTAINER=$2"
569 echo "clone <from> <to> clone an arvbox"
570 echo "available arvboxes: $(ls $ARVBOX_BASE)"
575 CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt
576 if test -n "$1" ; then
579 docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT"
580 echo "Certificate copied to $CERT"
584 exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados'
588 exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql'
592 exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql'
596 exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - <<EOF
603 cd /usr/src/arvados/services/api
604 export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
605 export RAILS_ENV=development
606 bundle exec rake db:drop
607 rm /var/lib/arvados/api_database_setup
608 rm /var/lib/arvados/superuser_token
609 rm /var/lib/arvados/keep0-uuid
610 rm /var/lib/arvados/keep1-uuid
611 rm /var/lib/arvados/keepproxy-uuid
615 sv restart keepstore0
616 sv restart keepstore1
622 echo "Arvados-in-a-box https://doc.arvados.org/install/arvbox.html"
624 echo "start|run <config> [tag] start $ARVBOX_CONTAINER container"
625 echo "stop stop arvbox container"
626 echo "restart <config> stop, then run again"
627 echo "status print some information about current arvbox"
628 echo "ip print arvbox docker container ip address"
629 echo "host print arvbox published host"
630 echo "shell enter shell as root"
631 echo "ashell enter shell as 'arvbox'"
632 echo "psql enter postgres console"
633 echo "open open arvbox workbench in a web browser"
634 echo "root-cert get copy of root certificate"
635 echo "update <config> stop, pull latest image, run"
636 echo "build <config> build arvbox Docker image"
637 echo "reboot <config> stop, build arvbox Docker image, run"
638 echo "rebuild <config> build arvbox Docker image, no layer cache"
639 echo "checkpoint create database backup"
640 echo "restore restore checkpoint"
641 echo "hotreset reset database and restart API without restarting container"
642 echo "reset delete arvbox arvados data (be careful!)"
643 echo "destroy delete all arvbox code and data (be careful!)"
644 echo "log <service> tail log of specified service"
645 echo "ls <options> list directories inside arvbox"
646 echo "cat <files> get contents of files inside arvbox"
647 echo "pipe run a bash script piped in from stdin"
648 echo "sv <start|stop|restart> <service> "
649 echo " change state of service inside arvbox"
650 echo "clone <from> <to> clone dev arvbox"