Merge branch 'master' into 14670-new-java-sdk-docs

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --combined build/run-tests.sh
index a4596bd23698565e121ae89bd9c4f193e5f27858,a37a0f731e5a12c192c3a17094a9891ae920f9a6..15ad4eae853eb4087d9e18af95429813518b977c
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -122,7 -122,6 +122,7 @@@ sdk/go/stat
  sdk/go/crunchrunner
  sdk/cwl
  sdk/R
 +sdk/java-v2
  tools/sync-groups
  tools/crunchstat-summary
  tools/crunchstat-summary:py3
@@@ -149,6 -148,7 +149,7 @@@ PYTHONPATH
  GEMHOME=
  PERLINSTALLBASE=
  R_LIBS=
+ export LANG=en_US.UTF-8
  
  short=
  only_install=
@@@ -190,6 -190,9 +191,9 @@@ sanity_checks() 
      ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
          || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
      echo Checking dependencies:
+     echo "locale: ${LANG}"
+     [[ "$(locale charmap)" = "UTF-8" ]] \
+         || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen"
      echo -n 'virtualenv: '
      virtualenv --version \
          || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
@@@ -365,6 -368,27 +369,27 @@@ if [[ $NEED_SDK_R == false ]]; the
        echo "R SDK not needed, it will not be installed."
  fi
  
+ checkpidfile() {
+     svc="$1"
+     pid="$(cat "$WORKSPACE/tmp/${svc}.pid")"
+     if [[ -z "$pid" ]] || ! kill -0 "$pid"; then
+         tail $WORKSPACE/tmp/${1}*.log
+         echo "${svc} pid ${pid} not running"
+         return 1
+     fi
+     echo "${svc} pid ${pid} ok"
+ }
+ 
+ checkdiscoverydoc() {
+     dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
+     if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
+         echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd"
+         tail -v $WORKSPACE/services/api/log/test.log
+         return 1
+     fi
+     echo "${dd} ok"
+ }
+ 
  start_services() {
      if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
          return 0
@@@ -379,19 -403,29 +404,29 @@@
        rm -f "$WORKSPACE/tmp/api.pid"
      fi
      all_services_stopped=
-     fail=0
+     fail=1
      cd "$WORKSPACE" \
-         && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo "fail=1; false") \
+         && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
          && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
          && export ARVADOS_TEST_API_INSTALLED="$$" \
+         && checkpidfile api \
+         && checkdiscoverydoc $ARVADOS_API_HOST \
          && python sdk/python/tests/run_test_server.py start_controller \
+         && checkpidfile controller \
          && python sdk/python/tests/run_test_server.py start_keep_proxy \
+         && checkpidfile keepproxy \
          && python sdk/python/tests/run_test_server.py start_keep-web \
+         && checkpidfile keep-web \
          && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+         && checkpidfile arv-git-httpd \
          && python sdk/python/tests/run_test_server.py start_ws \
-         && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo "fail=1; false") \
+         && checkpidfile ws \
+         && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+         && checkdiscoverydoc $ARVADOS_API_HOST \
+         && checkpidfile nginx \
+         && export ARVADOS_TEST_PROXY_SERVICES=1 \
          && (env | egrep ^ARVADOS) \
-         || fail=1
+         && fail=0
      deactivate
      if [[ $fail != 0 ]]; then
          unset ARVADOS_TEST_API_HOST
@@@ -403,7 -437,7 +438,7 @@@ stop_services() 
      if [[ -n "$all_services_stopped" ]]; then
          return
      fi
-     unset ARVADOS_TEST_API_HOST
+     unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES
      . "$VENVDIR/bin/activate" || return
      cd "$WORKSPACE" \
          && python sdk/python/tests/run_test_server.py stop_nginx \
@@@ -696,7 -730,7 +731,7 @@@ do_test() 
          services/api)
              stop_services
              ;;
-         doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+         gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
              # don't care whether services are running
              ;;
          *)
@@@ -729,7 -763,6 +764,6 @@@ do_test_once() 
          # compilation errors.
          go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
              cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
-             [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
              if [[ -n "${testargs[$1]}" ]]
          then
              # "go test -check.vv giturl" doesn't work, but this
@@@ -746,6 -779,7 +780,7 @@@
              go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
              rm "$WORKSPACE/tmp/.$covername.tmp"
          fi
+         [[ $result = 0 ]] && gofmt -e -d *.go
      elif [[ "$2" == "pip" ]]
      then
          tries=0
@@@ -896,7 -930,7 +931,7 @@@ install_services/api() 
      # database, so that we can drop it. This assumes the current user
      # is a postgresql superuser.
      cd "$WORKSPACE/services/api" \
-         && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+         && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
          && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
  
      mkdir -p "$WORKSPACE/services/api/tmp/pids"
@@@ -1003,6 -1037,12 +1038,12 @@@ test_doc() 
      )
  }
  
+ test_gofmt() {
+     cd "$WORKSPACE" || return 1
+     dirs=$(ls -d */ | egrep -v 'vendor|tmp')
+     [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+ }
+ 
  test_services/api() {
      rm -f "$WORKSPACE/services/api/git-commit.version"
      cd "$WORKSPACE/services/api" \
@@@ -1027,10 -1067,6 +1068,10 @@@ test_sdk/cli() 
          && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
  }
  
 +test_sdk/java-v2() {
 +    cd "$WORKSPACE/sdk/java-v2" && ./gradlew test
 +}
 +
  test_services/login-sync() {
      cd "$WORKSPACE/services/login-sync" \
          && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
@@@ -1043,17 -1079,17 +1084,17 @@@ test_services/nodemanager_integration(
  
  test_apps/workbench_units() {
      cd "$WORKSPACE/apps/workbench" \
-         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
+         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
  }
  
  test_apps/workbench_functionals() {
      cd "$WORKSPACE/apps/workbench" \
-         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
+         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
  }
  
  test_apps/workbench_integration() {
      cd "$WORKSPACE/apps/workbench" \
-         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
+         && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
  }
  
  test_apps/workbench_benchmark() {
@@@ -1122,12 -1158,12 +1163,13 @@@ test_all() 
          exit_cleanly
      fi
  
+     do_test gofmt
      do_test doc
      do_test sdk/ruby
      do_test sdk/R
      do_test sdk/cli
      do_test services/login-sync
 +    do_test sdk/java-v2
      do_test services/nodemanager_integration
      for p in "${pythonstuff[@]}"
      do
@@@ -1177,11 -1213,8 +1219,8 @@@ for g in "${gostuff[@]}"; d
  done
  for p in "${pythonstuff[@]}"; do
      dir=${p%:py3}
-     if [[ ${dir} = ${p} ]]; then
-         testfuncargs[$p]="$dir pip $VENVDIR/bin/"
-     else
-         testfuncargs[$p]="$dir pip $VENV3DIR/bin/"
-     fi
+     testfuncargs[$dir]="$dir pip $VENVDIR/bin/"
+     testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/"
  done
  
  if [[ -z ${interactive} ]]; then
@@@ -1193,7 -1226,11 +1232,11 @@@ els
      only_install=()
      if [[ -e "$VENVDIR/bin/activate" ]]; then stop_services; fi
      setnextcmd() {
-         if [[ "$nextcmd" != "install deps" ]]; then
+         if [[ "$TERM" = dumb ]]; then
+             # assume emacs, or something, is offering a history buffer
+             # and pre-populating the command will only cause trouble
+             nextcmd=
+         elif [[ "$nextcmd" != "install deps" ]]; then
              :
          elif [[ -e "$VENVDIR/bin/activate" ]]; then
              nextcmd="test lib/cmd"
@@@ -1207,30 -1244,34 +1250,34 @@@
      setnextcmd
      while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
          read verb target opts <<<"${nextcmd}"
+         target="${target%/}"
+         target="${target/\/:/:}"
          case "${verb}" in
-             "" | "help")
-                 help_interactive
-                 ;;
              "exit" | "quit")
                  exit_cleanly
                  ;;
              "reset")
                  stop_services
                  ;;
-             *)
-                 target="${target%/}"
-                 testargs["$target"]="${opts}"
+             "test" | "install")
                  case "$target" in
+                     "")
+                         help_interactive
+                         ;;
                      all | deps)
                          ${verb}_${target}
                          ;;
                      *)
+                         testargs["$target"]="${opts}"
                          tt="${testfuncargs[${target}]}"
                          tt="${tt:-$target}"
                          do_$verb $tt
                          ;;
                  esac
                  ;;
+             "" | "help" | *)
+                 help_interactive
+                 ;;
          esac
          if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then
              report_outcomes

diff --combined doc/_config.yml
index 837e2a86808b985fb54feac845d8ec12b28b019b,a5b53442ca1848118a8065342b78ebe4460ee31c..98c572b0119f50e7613bbea7f7776f91a5955583
--- 1/doc/_config.yml
--- 2/doc/_config.yml
+++ b/doc/_config.yml
@@@ -104,10 -104,7 +104,10 @@@ navbar
      - Ruby:
        - sdk/ruby/index.html.textile.liquid
        - sdk/ruby/example.html.textile.liquid
 -    - Java:
 +    - Java v2:
 +      - sdk/java-v2/index.html.textile.liquid
 +      - sdk/java-v2/javadoc.html.textile.liquid
 +    - Java v1:
        - sdk/java/index.html.textile.liquid
        - sdk/java/example.html.textile.liquid
    api:
@@@ -215,6 -212,8 +215,8 @@@
        - install/crunch2-slurm/install-test.html.textile.liquid
        - install/install-nodemanager.html.textile.liquid
        - install/install-compute-ping.html.textile.liquid
+     - Containers API support on cloud (experimental):
+       - install/install-dispatch-cloud.html.textile.liquid
      - Jobs API support (deprecated):
        - install/install-crunch-dispatch.html.textile.liquid
        - install/install-compute-node.html.textile.liquid