14287: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --combined build/run-tests.sh
index 7c21a296406d42c7f06fbc88fcfcae0dd2509ad8,3cf96f3b4ae01eb85873d5f8cfca9ce1dbfd8995..a46f2ec7660590cd82a3969e22ebecaceafe1dc2
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -96,6 -96,7 +96,7 @@@ services/arv-git-http
  services/crunchstat
  services/dockercleaner
  services/fuse
+ services/fuse:py3
  services/health
  services/keep-web
  services/keepproxy
@@@ -226,6 -227,8 +227,8 @@@ sanity_checks() 
      echo -n 'Python3 pyconfig.h: '
      find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
          || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+     which netstat \
+         || fatal "No netstat. Try: apt-get install net-tools"
      echo -n 'nginx: '
      PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
          || fatal "No nginx. Try: apt-get install nginx"
@@@ -255,12 -258,6 +258,6 @@@
      echo -n 'libpq libpq-fe.h: '
      find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
          || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
-     echo -n 'services/api/config/database.yml: '
-     if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-           fatal "Please provide a database.yml file for the test suite"
-     else
-           echo "OK"
-     fi
      echo -n 'postgresql: '
      psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
      echo -n 'phantomjs: '
@@@ -575,8 -572,6 +572,6 @@@ setup_virtualenv() 
      else
          "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
      fi
-     # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
-     "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
  }
  
  initialize() {
@@@ -628,11 -623,30 +623,30 @@@
  
      unset http_proxy https_proxy no_proxy
  
- 
      # Note: this must be the last time we change PATH, otherwise rvm will
      # whine a lot.
      setup_ruby_environment
  
+     if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+       cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
+       export ARVADOS_CONFIG="$temp/test-config.yml"
+     else
+       if [[ -s /etc/arvados/config.yml ]] ; then
+           python > "$temp/test-config.yml" <<EOF
+ import yaml
+ import json
+ v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0]['PostgreSQL']
+ v['Connection']['dbname'] = 'arvados_test'
+ print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
+ EOF
+           export ARVADOS_CONFIG="$temp/test-config.yml"
+       else
+           if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+               fatal "Please provide a database.yml file for the test suite"
+           fi
+       fi
+     fi
+ 
      echo "PATH is $PATH"
  }
  
@@@ -658,7 -672,6 +672,7 @@@ install_env() 
          cd "$GOPATH/src/git.curoverse.com/arvados.git"
          go get -v -d ...
          "$GOPATH/bin/govendor" sync
 +        which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports
      ) || fatal "Go setup failed"
  
      setup_virtualenv "$VENVDIR" --python python2.7
@@@ -775,7 -788,7 +789,7 @@@ do_test_once() 
          # before trying "go test". Otherwise, coverage-reporting
          # mode makes Go show the wrong line numbers when reporting
          # compilation errors.
-         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
+         go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
              cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
              if [[ -n "${testargs[$1]}" ]]
          then
@@@ -802,7 -815,10 +816,10 @@@
              tries=$((${tries}+1))
              # $3 can name a path directory for us to use, including trailing
              # slash; e.g., the bin/ subdirectory of a virtualenv.
-             "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+             if [[ -e "${3}activate" ]]; then
+                 . "${3}activate"
+             fi
+             python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
              result=$?
              if [[ ${tries} < 3 && ${result} == 137 ]]
              then
@@@ -841,7 -857,7 +858,7 @@@ do_install_once() 
          result=1
      elif [[ "$2" == "go" ]]
      then
-         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
+         go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
      elif [[ "$2" == "pip" ]]
      then
          # $3 can name a path directory for us to use, including trailing
@@@ -927,7 -943,6 +944,7 @@@ install_services/login-sync() 
  }
  
  install_services/api() {
 +    stop_services
      cd "$WORKSPACE/services/api" \
          && RAILS_ENV=test bundle_install_trylocal
  
@@@ -986,6 -1001,7 +1003,7 @@@ pythonstuff=
      sdk/cwl:py3
      services/dockercleaner:py3
      services/fuse
+     services/fuse:py3
      services/nodemanager
      tools/crunchstat-summary
      tools/crunchstat-summary:py3