12483: Merge branch 'master' into 12483-writable-fs

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --combined build/run-tests.sh
index 43e601ec07a54f55fdf84b9d6089b5f068d4eb86,3cfc692aaec5bc1f595b9333fa6be9179164f200..365931d33281d533e88442d2beb7b9f7f454878f
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -100,7 -100,7 +100,7 @@@ sdk/go/healt
  sdk/go/httpserver
  sdk/go/manifest
  sdk/go/blockdigest
- sdk/go/streamer
+ sdk/go/asyncbuf
  sdk/go/stats
  sdk/go/crunchrunner
  sdk/cwl
@@@ -212,6 -212,29 +212,29 @@@ sanity_checks() 
      echo -n 'cadaver: '
      cadaver --version | grep -w cadaver \
            || fatal "No cadaver. Try: apt-get install cadaver"
+     echo -n 'libattr1 xattr.h: '
+     find /usr/include -path '*/attr/xattr.h' | egrep --max-count=1 . \
+         || fatal "No libattr1 xattr.h. Try: apt-get install libattr1-dev"
+     echo -n 'libcurl curl.h: '
+     find /usr/include -path '*/curl/curl.h' | egrep --max-count=1 . \
+         || fatal "No libcurl curl.h. Try: apt-get install libcurl4-gnutls-dev"
+     echo -n 'libpq libpq-fe.h: '
+     find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
+         || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
+     echo -n 'services/api/config/database.yml: '
+     if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
+           fatal "Please provide a database.yml file for the test suite"
+     else
+           echo "OK"
+     fi
+     echo -n 'postgresql: '
+     psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
+     echo -n 'phantomjs: '
+     phantomjs --version || fatal "No phantomjs. Try: apt-get install phantomjs"
+     echo -n 'xvfb: '
+     which Xvfb || fatal "No xvfb. Try: apt-get install xvfb"
+     echo -n 'graphviz: '
+     dot -V || fatal "No graphviz. Try: apt-get install graphviz"
  }
  
  rotate_logfile() {
@@@ -283,6 -306,13 +306,13 @@@ don
  
  start_api() {
      echo 'Starting API server...'
+     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
+       mkdir -p "$WORKSPACE/services/api/log"
+     fi
+     # Remove empty api.pid file if it exists
+     if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
+       rm -f "$WORKSPACE/tmp/api.pid"
+     fi
      cd "$WORKSPACE" \
          && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
          && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
@@@ -457,7 -487,6 +487,7 @@@ export PERLLIB="$PERLINSTALLBASE/lib/pe
  
  export GOPATH
  mkdir -p "$GOPATH/src/git.curoverse.com"
 +rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
  ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
      || fatal "symlink failed"
  go get -v github.com/kardianos/govendor \
@@@ -800,7 -829,7 +830,7 @@@ gostuff=
      sdk/go/health
      sdk/go/httpserver
      sdk/go/manifest
-     sdk/go/streamer
+     sdk/go/asyncbuf
      sdk/go/crunchrunner
      sdk/go/stats
      lib/crunchstat

diff --combined sdk/go/keepclient/keepclient.go
index 4bc0fc5996368175c905b267aed26a2265a58003,37d651e31fbd971defa3217d6c11e883c4a073cc..54a4a374b991b44c5a5e51878be980a1b78f9609
--- 1/sdk/go/keepclient/keepclient.go
--- 2/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@@ -21,7 -21,7 +21,7 @@@ import 
        "time"
  
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/streamer"
+       "git.curoverse.com/arvados.git/sdk/go/asyncbuf"
  )
  
  // A Keep "block" is 64MB.
@@@ -156,10 -156,12 +156,12 @@@ func (kc *KeepClient) PutHR(hash string
                bufsize = BLOCKSIZE
        }
  
-       t := streamer.AsyncStreamFromReader(bufsize, HashCheckingReader{r, md5.New(), hash})
-       defer t.Close()
- 
-       return kc.putReplicas(hash, t, dataBytes)
+       buf := asyncbuf.NewBuffer(make([]byte, 0, bufsize))
+       go func() {
+               _, err := io.Copy(buf, HashCheckingReader{r, md5.New(), hash})
+               buf.CloseWithError(err)
+       }()
+       return kc.putReplicas(hash, buf.NewReader, dataBytes)
  }
  
  // PutHB writes a block to Keep. The hash of the bytes is given in
@@@ -167,9 -169,8 +169,8 @@@
  //
  // Return values are the same as for PutHR.
  func (kc *KeepClient) PutHB(hash string, buf []byte) (string, int, error) {
-       t := streamer.AsyncStreamFromSlice(buf)
-       defer t.Close()
-       return kc.putReplicas(hash, t, int64(len(buf)))
+       newReader := func() io.Reader { return bytes.NewBuffer(buf) }
+       return kc.putReplicas(hash, newReader, int64(len(buf)))
  }
  
  // PutB writes a block to Keep. It computes the hash itself.
@@@ -292,12 -293,6 +293,12 @@@ func (kc *KeepClient) Get(locator strin
        return kc.getOrHead("GET", locator)
  }
  
 +// ReadAt() retrieves a portion of block from the cache if it's
 +// present, otherwise from the network.
 +func (kc *KeepClient) ReadAt(locator string, p []byte, off int) (int, error) {
 +      return kc.cache().ReadAt(kc, locator, p, off)
 +}
 +
  // Ask() verifies that a block with the given hash is available and
  // readable, according to at least one Keep service. Unlike Get, it
  // does not retrieve the data or verify that the data content matches