docker/arvdock

   1 #!/bin/bash
   2
   3 DOCKER=`which docker.io`
   4
   5 if [[ "$DOCKER" == "" ]]; then
   6     DOCKER=`which docker`
   7 fi
   8
   9 COMPUTE_COUNTER=0
  10
  11 function usage {
  12     echo >&2
  13     echo >&2 "usage: $0 (start|stop|restart|test) [options]"
  14     echo >&2
  15     echo >&2 "$0 start/stop/restart options:"
  16     echo >&2 "  -d[port], --doc[=port]        Documentation server (default port 9898)"
  17     echo >&2 "  -w[port], --workbench[=port]  Workbench server (default port 9899)"
  18     echo >&2 "  -s[port], --sso[=port]        SSO server (default port 9901)"
  19     echo >&2 "  -a[port], --api[=port]        API server (default port 9900)"
  20     echo >&2 "  -c, --compute                 Compute nodes (starts 2)"
  21     echo >&2 "  -v, --vm                      Shell server"
  22     echo >&2 "  -n, --nameserver              Nameserver"
  23     echo >&2 "  -k, --keep                    Keep servers"
  24     echo >&2 "  -p, --keepproxy               Keepproxy server"
  25     echo >&2 "  -h, --help                    Display this help and exit"
  26     echo >&2
  27     echo >&2 "  If no options are given, the action is applied to all servers."
  28     echo >&2
  29     echo >&2 "$0 test [testname] [testname] ..."
  30     echo >&2 "  By default, all tests are run."
  31 }
  32
  33 function ip_address {
  34     local container=$1
  35     echo `$DOCKER inspect $container  |grep IPAddress |cut -f4 -d\"`
  36 }
  37
  38 function start_container {
  39     local args="-d -i -t"
  40     if [[ "$1" != '' ]]; then
  41       local port="$1"
  42       args="$args -p $port"
  43     fi
  44     if [[ "$2" != '' ]]; then
  45       local name="$2"
  46       if [[ "$name" == "api_server" ]]; then
  47         args="$args --dns=172.17.42.1 --dns-search=compute.dev.arvados --hostname api -P --name $name"
  48       elif [[ "$name" == "compute" ]]; then
  49         name=$name$COMPUTE_COUNTER
  50         # We need --privileged because we run docker-inside-docker on the compute nodes
  51         args="$args --dns=172.17.42.1 --dns-search=compute.dev.arvados --hostname compute$COMPUTE_COUNTER -P --privileged --name $name"
  52         let COMPUTE_COUNTER=$(($COMPUTE_COUNTER + 1))
  53       else
  54         args="$args --dns=172.17.42.1 --dns-search=dev.arvados --hostname ${name#_server} --name $name"
  55       fi
  56     fi
  57     if [[ "$3" != '' ]]; then
  58       local volume="$3"
  59       args="$args -v $volume"
  60     fi
  61     if [[ "$4" != '' ]]; then
  62       local link="$4"
  63       args="$args --link $link"
  64     fi
  65     local image=$5
  66
  67     `$DOCKER ps |grep -P "$name[^/]" -q`
  68     if [[ "$?" == "0" ]]; then
  69       echo "You have a running container with name $name -- skipping."
  70       return
  71     fi
  72
  73     # Remove any existing container by this name.
  74     $DOCKER rm "$name" 2>/dev/null
  75
  76     echo "Starting container:"
  77     echo "  $DOCKER run $args $image"
  78     container=`$DOCKER run $args $image`
  79     if [ "$?" != "0" -o "$container" = "" ]; then
  80       echo "Unable to start container"
  81       exit 1
  82     else
  83       echo "Started container: $container"
  84     fi
  85
  86     if [[ "$name" == "doc_server" ]]; then
  87       echo
  88       echo "*****************************************************************"
  89       echo "You can access the Arvados documentation at http://localhost:${port%:*}"
  90       echo "*****************************************************************"
  91       echo
  92     fi
  93
  94     if [[ "$name" == "workbench_server" ]]; then
  95       echo
  96       echo "*****************************************************************"
  97       echo "You can access the Arvados workbench at http://localhost:${port%:*}"
  98       echo "*****************************************************************"
  99       echo
 100    fi
 101
 102
 103 }
 104
 105 declare -a keep_volumes
 106
 107 # Initialize the global `keep_volumes' array. If any keep volumes
 108 # already appear to exist (mounted volumes with a top-level "keep"
 109 # directory), use them; create temporary volumes if necessary.
 110 #
 111 function make_keep_volumes () {
 112     # Mount a keep volume if we don't already have one
 113     for mountpoint in $(cut -d ' ' -f 2 /proc/mounts); do
 114       if [[ -d "$mountpoint/keep" && "$mountpoint" != "/" ]]; then
 115         keep_volumes+=($mountpoint)
 116       fi
 117     done
 118
 119     # Create any keep volumes that do not yet exist.
 120     while [ ${#keep_volumes[*]} -lt 2 ]
 121     do
 122         new_keep=$(mktemp -d)
 123         echo >&2 "mounting 2G tmpfs keep volume in $new_keep"
 124         sudo mount -t tmpfs -o size=2G tmpfs $new_keep
 125         mkdir $new_keep/keep
 126         keep_volumes+=($new_keep)
 127     done
 128 }
 129
 130 function do_start {
 131     local start_doc=false
 132     local start_sso=false
 133     local start_api=false
 134     local start_compute=false
 135     local start_workbench=false
 136     local start_vm=false
 137     local start_nameserver=false
 138     local start_keep=false
 139     local start_keepproxy=false
 140
 141     # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
 142     local TEMP=`getopt -o d::s::a::cw::nkpvh \
 143                   --long doc::,sso::,api::,compute,workbench::,nameserver,keep,keepproxy,vm,help \
 144                   -n "$0" -- "$@"`
 145
 146     if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
 147
 148     # Note the quotes around `$TEMP': they are essential!
 149     eval set -- "$TEMP"
 150
 151     while [ $# -ge 1 ]
 152     do
 153         case $1 in
 154             -d | --doc)
 155                 case "$2" in
 156                     "") start_doc=9898; shift 2 ;;
 157                     *)  start_doc=$2; shift 2 ;;
 158                 esac
 159                 ;;
 160             -s | --sso)
 161                 case "$2" in
 162                     "") start_sso=9901; shift 2 ;;
 163                     *)  start_sso=$2; shift 2 ;;
 164                 esac
 165                 ;;
 166             -a | --api)
 167                 case "$2" in
 168                     "") start_api=9900; shift 2 ;;
 169                     *)  start_api=$2; shift 2 ;;
 170                 esac
 171                 ;;
 172             -c | --compute)
 173                 start_compute=2
 174                 shift
 175                 ;;
 176             -w | --workbench)
 177                 case "$2" in
 178                     "") start_workbench=9899; shift 2 ;;
 179                     *)  start_workbench=$2; shift 2 ;;
 180                 esac
 181                 ;;
 182             -v | --vm)
 183                 start_vm=true
 184                 shift
 185                 ;;
 186             -n | --nameserver)
 187                 start_nameserver=true
 188                 shift
 189                 ;;
 190             -k | --keep)
 191                 start_keep=true
 192                 shift
 193                 ;;
 194             -p | --keepproxy)
 195                 start_keepproxy=true
 196                 shift
 197                 ;;
 198             --)
 199                 shift
 200                 break
 201                 ;;
 202             *)
 203                 usage
 204                 exit 1
 205                 ;;
 206         esac
 207     done
 208
 209     # If no options were selected, then start all servers.
 210     if [[ $start_doc == false &&
 211           $start_sso == false &&
 212           $start_api == false &&
 213           $start_compute == false &&
 214           $start_workbench == false &&
 215           $start_vm == false &&
 216           $start_nameserver == false &&
 217           $start_keep == false &&
 218           $start_keepproxy == false ]]
 219     then
 220         start_doc=9898
 221         #the sso server is currently not used by default so don't start it unless explicitly requested
 222         #start_sso=9901
 223         start_api=9900
 224         start_compute=2
 225         start_workbench=9899
 226         start_vm=true
 227         start_nameserver=true
 228         start_keep=true
 229         start_keepproxy=true
 230     fi
 231
 232     if [[ $start_nameserver != false ]]
 233     then
 234       # We rely on skydock and skydns for dns discovery between the slurm controller and compute nodes,
 235       # so make sure they are running
 236       $DOCKER ps | grep skydns >/dev/null
 237       if [[ "$?" != "0" ]]; then
 238         echo "Starting crosbymichael/skydns container..."
 239         $DOCKER rm "skydns" 2>/dev/null
 240         $DOCKER run -d -p 172.17.42.1:53:53/udp --name skydns crosbymichael/skydns -nameserver 8.8.8.8:53 -domain arvados
 241       fi
 242       $DOCKER ps | grep skydock >/dev/null
 243       if [[ "$?" != "0" ]]; then
 244         echo "Starting crosbymichael/skydock container..."
 245         $DOCKER rm "skydock" 2>/dev/null
 246         $DOCKER run -d -v /var/run/docker.sock:/docker.sock --name skydock crosbymichael/skydock -ttl 30 -environment dev -s /docker.sock -domain arvados -name skydns
 247       fi
 248     fi
 249
 250     if [[ $start_sso != false ]]
 251     then
 252         start_container "$start_sso:443" "sso_server" '' '' "arvados/sso"
 253     fi
 254
 255     if [[ $start_api != false ]]
 256     then
 257       if [[ $start_sso != false ]]; then
 258         start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
 259       else
 260         start_container "$start_api:443" "api_server" '' '' "arvados/api"
 261       fi
 262     fi
 263
 264     if [[ $start_compute != false ]]
 265     then
 266         for i in `seq 0 $(($start_compute - 1))`; do
 267           start_container "" "compute" '' "api_server:api" "arvados/compute"
 268         done
 269     fi
 270
 271     if [[ $start_keep != false ]]
 272     then
 273         # create `keep_volumes' array with a list of keep mount points
 274         # remove any stale metadata from those volumes before starting them
 275         make_keep_volumes
 276         for v in ${keep_volumes[*]}
 277         do
 278             [ -f $v/keep/.metadata.yml ] && sudo rm $v/keep/.metadata.yml
 279         done
 280         start_container "25107:25107" "keep_server_0" \
 281             "${keep_volumes[0]}:/keep-data" \
 282             "api_server:api" \
 283             "arvados/keep"
 284         start_container "25108:25107" "keep_server_1" \
 285             "${keep_volumes[1]}:/keep-data" \
 286             "api_server:api" \
 287             "arvados/keep"
 288     fi
 289
 290     if [[ $start_keepproxy != false ]]
 291     then
 292         start_container "9902:9100" "keepproxy_server" '' \
 293             "api_server:api" \
 294             "arvados/keepproxy"
 295     fi
 296
 297     if [[ $start_doc != false ]]
 298     then
 299         start_container "$start_doc:80" "doc_server" '' '' "arvados/doc"
 300     fi
 301
 302     if [[ $start_vm != false ]]
 303     then
 304         start_container "" "shell" '' "api_server:api" "arvados/shell"
 305     fi
 306
 307     if [[ $start_workbench != false ]]
 308     then
 309         start_container "$start_workbench:80" "workbench_server" '' "api_server:api" "arvados/workbench"
 310     fi
 311
 312     if [[ $start_api != false ]]
 313     then
 314         if [[ -f "api/generated/superuser_token" ]]
 315         then
 316           if [ -d $HOME/.config/arvados ] || mkdir -p $HOME/.config/arvados
 317           then
 318             cat >$HOME/.config/arvados/settings.conf <<EOF
 319 ARVADOS_API_HOST=$(ip_address "api_server")
 320 ARVADOS_API_HOST_INSECURE=yes
 321 ARVADOS_API_TOKEN=$(cat api/generated/superuser_token)
 322 EOF
 323           fi
 324         fi
 325     fi
 326 }
 327
 328 function do_stop {
 329     local stop_doc=""
 330     local stop_sso=""
 331     local stop_api=""
 332     local stop_compute=""
 333     local stop_workbench=""
 334     local stop_nameserver=""
 335     local stop_vm=""
 336     local stop_keep=""
 337     local stop_keepproxy=""
 338
 339     # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
 340     local TEMP=`getopt -o dsacwnkpvh \
 341                   --long doc,sso,api,compute,workbench,nameserver,keep,keepproxy,vm,help \
 342                   -n "$0" -- "$@"`
 343
 344     if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
 345
 346     # Note the quotes around `$TEMP': they are essential!
 347     eval set -- "$TEMP"
 348
 349     while [ $# -ge 1 ]
 350     do
 351         case $1 in
 352             -d | --doc)
 353                 stop_doc=doc_server ; shift ;;
 354             -s | --sso)
 355                 stop_sso=sso_server ; shift ;;
 356             -a | --api)
 357                 stop_api=api_server ; shift ;;
 358             -c | --compute)
 359                 stop_compute=`$DOCKER ps |grep -P "compute\d+" |grep -v api_server |cut -f1 -d ' '` ; shift ;;
 360             -w | --workbench)
 361                 stop_workbench=workbench_server ; shift ;;
 362             -n | --nameserver )
 363                 stop_nameserver="skydock skydns" ; shift ;;
 364             -v | --vm )
 365                 stop_vm="shell" ; shift ;;
 366             -k | --keep )
 367                 stop_keep="keep_server_0 keep_server_1" ; shift ;;
 368             -p | --keepproxy )
 369                 stop_keep="keepproxy_server" ; shift ;;
 370             --)
 371                 shift
 372                 break
 373                 ;;
 374             *)
 375                 usage
 376                 exit 1
 377                 ;;
 378         esac
 379     done
 380
 381     # If no options were selected, then stop all servers.
 382     if [[ $stop_doc == "" &&
 383           $stop_sso == "" &&
 384           $stop_api == "" &&
 385           $stop_compute == "" &&
 386           $stop_workbench == "" &&
 387           $stop_vm == "" &&
 388           $stop_nameserver == "" &&
 389           $stop_keep == "" &&
 390           $stop_keepproxy == "" ]]
 391     then
 392         stop_doc=doc_server
 393         stop_sso=sso_server
 394         stop_api=api_server
 395         stop_compute=`$DOCKER ps |grep -P "compute\d+" |grep -v api_server |cut -f1 -d ' '`
 396         stop_workbench=workbench_server
 397         stop_vm=shell
 398         stop_nameserver="skydock skydns"
 399         stop_keep="keep_server_0 keep_server_1"
 400         stop_keepproxy="keepproxy_server"
 401     fi
 402
 403     $DOCKER stop $stop_doc $stop_sso $stop_api $stop_compute $stop_workbench $stop_nameserver $stop_keep $stop_keepproxy $stop_vm \
 404         2>/dev/null
 405 }
 406
 407 function do_test {
 408     local alltests
 409     if [ $# -lt 1 ]
 410     then
 411         alltests="python-sdk api"
 412     else
 413         alltests="$@"
 414     fi
 415
 416     for testname in $alltests
 417     do
 418         echo "testing $testname..."
 419         case $testname in
 420             python-sdk)
 421                 do_start --api --keep --sso
 422                 export ARVADOS_API_HOST=$(ip_address "api_server")
 423                 export ARVADOS_API_HOST_INSECURE=yes
 424                 export ARVADOS_API_TOKEN=$(cat api/generated/superuser_token)
 425                 python -m unittest discover ../sdk/python
 426                 ;;
 427             api)
 428                 $DOCKER run -t -i arvados/api \
 429                     /usr/src/arvados/services/api/script/rake_test.sh
 430                 ;;
 431             *)
 432                 echo >&2 "unknown test $testname"
 433                 ;;
 434         esac
 435     done
 436 }
 437
 438 if [ $# -lt 1 ]
 439 then
 440   usage
 441   exit 1
 442 fi
 443
 444 case $1 in
 445     start)
 446         shift
 447         do_start $@
 448         ;;
 449     stop)
 450         shift
 451         do_stop $@
 452         ;;
 453     restart)
 454         shift
 455         do_stop $@
 456         do_start $@
 457         ;;
 458     test)
 459         shift
 460         do_test $@
 461         ;;
 462     *)
 463         usage
 464         exit 1
 465         ;;
 466 esac