8460: Merge branch 'master' into 8460-websocket-go

diff --combined build/run-build-packages.sh
index 116d14bfe786e88429a5d1ca58d2aba9abcba418,ddf93832b327639843d7b4b056e563fac1a6190e..08b2cd204a0046defb3db95741639bd28042008b
--- 1/build/run-build-packages.sh
--- 2/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@@ -110,7 -110,7 +110,7 @@@ case "$TARGET" i
              'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
              shellescape mistune typing avro ruamel.ordereddict
              cachecontrol requests)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
          ;;
      debian8)
          FORMAT=deb
@@@ -121,7 -121,7 +121,7 @@@
              'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
              shellescape mistune typing avro ruamel.ordereddict
              cachecontrol)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
          ;;
      ubuntu1204)
          FORMAT=deb
@@@ -132,7 -132,7 +132,7 @@@
              contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
              shellescape mistune typing avro isodate ruamel.ordereddict
              cachecontrol requests)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
          ;;
      ubuntu1404)
          FORMAT=deb
@@@ -141,7 -141,7 +141,7 @@@
              rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
              shellescape mistune typing avro ruamel.ordereddict
              cachecontrol)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
          ;;
      centos6)
          FORMAT=rpm
@@@ -161,7 -161,7 +161,7 @@@
              'rdflib>=4.2.0' shellescape mistune typing avro requests \
              isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
              ruamel.ordereddict cachecontrol)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
          export PYCURL_SSL_LIBRARY=nss
          ;;
      centos7)
@@@ -181,7 -181,7 +181,7 @@@
              'rdflib>=4.2.0' shellescape mistune typing avro \
              isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
              ruamel.ordereddict cachecontrol)
-         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
+         PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
          export PYCURL_SSL_LIBRARY=nss
          ;;
      *)
@@@ -434,8 -434,6 +434,8 @@@ package_go_binary services/keepstore ke
      "Keep storage daemon, accessible to clients on the LAN"
  package_go_binary services/keep-web keep-web \
      "Static web hosting service for user data stored in Arvados Keep"
 +package_go_binary services/ws arvados-ws \
 +    "Arvados Websocket server"
  package_go_binary tools/keep-block-check keep-block-check \
      "Verify that all data from one set of Keep servers to another was copied"
  package_go_binary tools/keep-rsync keep-rsync \
@@@ -487,7 -485,8 +487,8 @@@ fpm_build ruamel.yaml "" "" python 0.12
  fpm_build cwltest "" "" python 1.0.20160907111242
  
  # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
- fpm_build cwltool "" "" python 1.0.20161128202906
+ cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(1\.0\..*\)'.*/\1/")
+ fpm_build cwltool "" "" python $cwltoolversion
  
  # FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
  fpm_build rdflib-jsonld "" "" python 0.3.0
@@@ -514,7 -513,7 +515,7 @@@ fpm_build $WORKSPACE/services/nodemanag
  # The Docker image cleaner
  cd $WORKSPACE/packages/$TARGET
  rm -rf "$WORKSPACE/services/dockercleaner/build"
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
  
  # The Arvados crunchstat-summary tool
  cd $WORKSPACE/packages/$TARGET

diff --combined build/run-tests.sh
index 379c6168031b37933a89a7053a1a3037fc19f6c2,d08568cde58b97bdede5b430eda4f0b788f4e3ce..efcab595791f6f6f0db88eb23d4445524ca073eb
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -79,7 -79,6 +79,7 @@@ services/nodemanage
  services/crunch-run
  services/crunch-dispatch-local
  services/crunch-dispatch-slurm
 +services/ws
  sdk/cli
  sdk/pam
  sdk/python
@@@ -91,7 -90,6 +91,7 @@@ sdk/go/httpserve
  sdk/go/manifest
  sdk/go/blockdigest
  sdk/go/streamer
 +sdk/go/stats
  sdk/go/crunchrunner
  sdk/cwl
  tools/crunchstat-summary
@@@ -158,6 -156,12 +158,12 @@@ sanity_checks() 
      echo -n 'virtualenv: '
      virtualenv --version \
          || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
+     echo -n 'ruby: '
+     ruby -v \
+         || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)"
+     echo -n 'bundler: '
+     bundle version \
+         || fatal "No bundler. Try: gem install bundler"
      echo -n 'go: '
      go version \
          || fatal "No go binary. See http://golang.org/doc/install"
@@@ -266,18 -270,15 +272,18 @@@ start_api() 
          && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
          && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
          && export ARVADOS_TEST_API_INSTALLED="$$" \
 +        && python sdk/python/tests/run_test_server.py start_ws \
 +        && python sdk/python/tests/run_test_server.py start_nginx \
          && (env | egrep ^ARVADOS)
  }
  
  start_nginx_proxy_services() {
 -    echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
 +    echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
      cd "$WORKSPACE" \
          && python sdk/python/tests/run_test_server.py start_keep_proxy \
          && python sdk/python/tests/run_test_server.py start_keep-web \
          && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
 +        && python sdk/python/tests/run_test_server.py start_ws \
          && python sdk/python/tests/run_test_server.py start_nginx \
          && export ARVADOS_TEST_PROXY_SERVICES=1
  }
@@@ -288,15 -289,12 +294,15 @@@ stop_services() 
          cd "$WORKSPACE" \
              && python sdk/python/tests/run_test_server.py stop_nginx \
              && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
 +            && python sdk/python/tests/run_test_server.py stop_ws \
              && python sdk/python/tests/run_test_server.py stop_keep-web \
              && python sdk/python/tests/run_test_server.py stop_keep_proxy
      fi
      if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
          unset ARVADOS_TEST_API_HOST
          cd "$WORKSPACE" \
 +            && python sdk/python/tests/run_test_server.py stop_nginx \
 +            && python sdk/python/tests/run_test_server.py stop_ws \
              && python sdk/python/tests/run_test_server.py stop
      fi
  }
@@@ -758,7 -756,6 +764,7 @@@ gostuff=
      sdk/go/manifest
      sdk/go/streamer
      sdk/go/crunchrunner
 +    sdk/go/stats
      lib/crunchstat
      services/arv-git-httpd
      services/crunchstat
@@@ -774,7 -771,6 +780,7 @@@
      services/crunch-dispatch-local
      services/crunch-dispatch-slurm
      services/crunch-run
 +    services/ws
      tools/keep-block-check
      tools/keep-exercise
      tools/keep-rsync