18870: Need to declare NODES as array
[arvados.git] / services / keepproxy / keepproxy_test.go
index a6a73c831721e0c4489c041e2d893f5ac68599f9..8242f5b2b56868b23dfaecf4aefe814100e682ee 100644 (file)
@@ -2,15 +2,16 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepproxy
 
 import (
        "bytes"
+       "context"
        "crypto/md5"
-       "errors"
        "fmt"
        "io/ioutil"
        "math/rand"
+       "net"
        "net/http"
        "net/http/httptest"
        "strings"
@@ -18,14 +19,16 @@ import (
        "testing"
        "time"
 
-       "git.curoverse.com/arvados.git/lib/config"
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-       "git.curoverse.com/arvados.git/sdk/go/ctxlog"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "git.arvados.org/arvados.git/lib/config"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/ctxlog"
+       "git.arvados.org/arvados.git/sdk/go/httpserver"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"
        log "github.com/sirupsen/logrus"
 
+       "gopkg.in/check.v1"
        . "gopkg.in/check.v1"
 )
 
@@ -40,6 +43,12 @@ var _ = Suite(&ServerRequiredSuite{})
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredConfigYmlSuite{})
+
+// Tests that require the Keep servers running as defined in config.yml
+type ServerRequiredConfigYmlSuite struct{}
+
 // Gocheck boilerplate
 var _ = Suite(&NoKeepServerSuite{})
 
@@ -48,29 +57,7 @@ type NoKeepServerSuite struct{}
 
 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
 
-// Wait (up to 1 second) for keepproxy to listen on a port. This
-// avoids a race condition where we hit a "connection refused" error
-// because we start testing the proxy too soon.
-func waitForListener() {
-       const (
-               ms = 5
-       )
-       for i := 0; listener == nil && i < 10000; i += ms {
-               time.Sleep(ms * time.Millisecond)
-       }
-       if listener == nil {
-               panic("Timed out waiting for listener to start")
-       }
-}
-
-func closeListener() {
-       if listener != nil {
-               listener.Close()
-       }
-}
-
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-       arvadostest.StartAPI()
        arvadostest.StartKeep(2, false)
 }
 
@@ -80,11 +67,22 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) {
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
        arvadostest.StopKeep(2)
-       arvadostest.StopAPI()
+}
+
+func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
+       // config.yml defines 4 keepstores
+       arvadostest.StartKeep(4, false)
+}
+
+func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
+       arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
+       arvadostest.StopKeep(4)
 }
 
 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
-       arvadostest.StartAPI()
        // We need API to have some keep services listed, but the
        // services themselves should be unresponsive.
        arvadostest.StartKeep(2, false)
@@ -95,59 +93,82 @@ func (s *NoKeepServerSuite) SetUpTest(c *C) {
        arvadostest.ResetEnv()
 }
 
-func (s *NoKeepServerSuite) TearDownSuite(c *C) {
-       arvadostest.StopAPI()
+type testServer struct {
+       *httpserver.Server
+       proxyHandler *proxyHandler
 }
 
-func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
+func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*testServer, *keepclient.KeepClient, *bytes.Buffer) {
        cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
        c.Assert(err, Equals, nil)
        cluster, err := cfg.GetCluster("")
        c.Assert(err, Equals, nil)
 
-       cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
+       if !loadKeepstoresFromConfig {
+               // Do not load Keepstore InternalURLs from the config file
+               cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
+       }
 
-       listener = nil
-       go func() {
-               run(log.New(), cluster)
-               defer closeListener()
-       }()
-       waitForListener()
+       cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
+
+       if kp != nil {
+               cluster.Collections.KeepproxyPermission = *kp
+       }
+
+       logbuf := &bytes.Buffer{}
+       logger := log.New()
+       logger.Out = logbuf
+       ctx := ctxlog.Context(context.Background(), logger)
+
+       handler := newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*proxyHandler)
+       srv := &testServer{
+               Server: &httpserver.Server{
+                       Server: http.Server{
+                               BaseContext: func(net.Listener) context.Context { return ctx },
+                               Handler: httpserver.AddRequestIDs(
+                                       httpserver.LogRequests(handler)),
+                       },
+                       Addr: ":",
+               },
+               proxyHandler: handler,
+       }
+       err = srv.Start()
+       c.Assert(err, IsNil)
 
        client := arvados.NewClientFromEnv()
        arv, err := arvadosclient.New(client)
-       c.Assert(err, Equals, nil)
+       c.Assert(err, IsNil)
        if bogusClientToken {
                arv.ApiToken = "bogus-token"
        }
        kc := keepclient.New(arv)
        sr := map[string]string{
-               TestProxyUUID: "http://" + listener.Addr().String(),
+               TestProxyUUID: "http://" + srv.Addr,
        }
        kc.SetServiceRoots(sr, sr, sr)
        kc.Arvados.External = true
-
-       return kc
+       return srv, kc, logbuf
 }
 
 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
-       runProxy(c, false)
-       defer closeListener()
+       srv, _, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        req, err := http.NewRequest("POST",
-               "http://"+listener.Addr().String()+"/",
+               "http://"+srv.Addr+"/",
                strings.NewReader("TestViaHeader"))
        c.Assert(err, Equals, nil)
        req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
        resp, err := (&http.Client{}).Do(req)
        c.Assert(err, Equals, nil)
        c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
+       c.Assert(resp.StatusCode, Equals, http.StatusOK)
        locator, err := ioutil.ReadAll(resp.Body)
        c.Assert(err, Equals, nil)
        resp.Body.Close()
 
        req, err = http.NewRequest("GET",
-               "http://"+listener.Addr().String()+"/"+string(locator),
+               "http://"+srv.Addr+"/"+string(locator),
                nil)
        c.Assert(err, Equals, nil)
        resp, err = (&http.Client{}).Do(req)
@@ -157,13 +178,13 @@ func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        sr := map[string]string{
-               TestProxyUUID: "http://" + listener.Addr().String(),
+               TestProxyUUID: "http://" + srv.Addr,
        }
-       router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+       srv.proxyHandler.KeepClient.SetServiceRoots(sr, sr, sr)
 
        content := []byte("TestLoopDetection")
        _, _, err := kc.PutB(content)
@@ -175,8 +196,8 @@ func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        // Set up fake keepstore to record request headers
        var hdr http.Header
@@ -191,7 +212,7 @@ func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
        sr := map[string]string{
                TestProxyUUID: ts.URL,
        }
-       router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+       srv.proxyHandler.KeepClient.SetServiceRoots(sr, sr, sr)
 
        // Set up client to ask for storage classes to keepproxy
        kc.StorageClasses = []string{"secure"}
@@ -201,26 +222,52 @@ func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
        c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
 }
 
+func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
+       srv, _, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
+
+       content := []byte("foo")
+       hash := fmt.Sprintf("%x", md5.Sum(content))
+       client := &http.Client{}
+
+       req, err := http.NewRequest("PUT",
+               fmt.Sprintf("http://%s/%s", srv.Addr, hash),
+               bytes.NewReader(content))
+       c.Assert(err, IsNil)
+       req.Header.Set("X-Keep-Storage-Classes", "default")
+       req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
+       req.Header.Set("Content-Type", "application/octet-stream")
+
+       resp, err := client.Do(req)
+       c.Assert(err, IsNil)
+       c.Assert(resp.StatusCode, Equals, http.StatusOK)
+       c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
+}
+
 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        content := []byte("TestDesiredReplicas")
        hash := fmt.Sprintf("%x", md5.Sum(content))
 
-       for _, kc.Want_replicas = range []int{0, 1, 2} {
+       for _, kc.Want_replicas = range []int{0, 1, 2, 3} {
                locator, rep, err := kc.PutB(content)
-               c.Check(err, Equals, nil)
-               c.Check(rep, Equals, kc.Want_replicas)
-               if rep > 0 {
-                       c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
+               if kc.Want_replicas < 3 {
+                       c.Check(err, Equals, nil)
+                       c.Check(rep, Equals, kc.Want_replicas)
+                       if rep > 0 {
+                               c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
+                       }
+               } else {
+                       c.Check(err, ErrorMatches, ".*503.*")
                }
        }
 }
 
 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        content := []byte("TestPutWrongContentLength")
        hash := fmt.Sprintf("%x", md5.Sum(content))
@@ -230,7 +277,8 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
        // fixes the invalid Content-Length header. In order to test
        // our server behavior, we have to call the handler directly
        // using an httptest.ResponseRecorder.
-       rtr := MakeRESTRouter(kc, 10*time.Second, "")
+       rtr, err := newHandler(context.Background(), kc, 10*time.Second, &arvados.Cluster{})
+       c.Assert(err, check.IsNil)
 
        type testcase struct {
                sendLength   string
@@ -244,7 +292,7 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
                {"abcdef", http.StatusLengthRequired},
        } {
                req, err := http.NewRequest("PUT",
-                       fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
+                       fmt.Sprintf("http://%s/%s+%d", srv.Addr, hash, len(content)),
                        bytes.NewReader(content))
                c.Assert(err, IsNil)
                req.Header.Set("Content-Length", t.sendLength)
@@ -258,9 +306,9 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
-       router.(*proxyHandler).timeout = time.Nanosecond
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
+       srv.proxyHandler.timeout = time.Nanosecond
 
        buf := make([]byte, 1<<20)
        rand.Read(buf)
@@ -285,8 +333,8 @@ func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, logbuf := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
        var hash2 string
@@ -321,6 +369,9 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
                c.Check(rep, Equals, 2)
                c.Check(err, Equals, nil)
                c.Log("Finished PutB (expected success)")
+
+               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="TestCase Administrator".* userUUID=zzzzz-tpzed-d9tiejq69daie8f.*`)
+               logbuf.Reset()
        }
 
        {
@@ -328,6 +379,8 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
                c.Assert(err, Equals, nil)
                c.Check(blocklen, Equals, int64(3))
                c.Log("Finished Ask (expected success)")
+               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="TestCase Administrator".* userUUID=zzzzz-tpzed-d9tiejq69daie8f.*`)
+               logbuf.Reset()
        }
 
        {
@@ -338,6 +391,8 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
                c.Check(all, DeepEquals, []byte("foo"))
                c.Check(blocklen, Equals, int64(3))
                c.Log("Finished Get (expected success)")
+               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="TestCase Administrator".* userUUID=zzzzz-tpzed-d9tiejq69daie8f.*`)
+               logbuf.Reset()
        }
 
        {
@@ -362,8 +417,8 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
-       kc := runProxy(c, true)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, true, false, nil)
+       defer srv.Close()
 
        hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 
@@ -373,28 +428,126 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
        hash2, rep, err := kc.PutB([]byte("bar"))
        c.Check(hash2, Equals, "")
        c.Check(rep, Equals, 0)
-       c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
+       c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
 
        blocklen, _, err := kc.Ask(hash)
        c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
-       c.Check(err, ErrorMatches, ".*not found.*")
+       c.Check(err, ErrorMatches, ".*HTTP 403.*")
        c.Check(blocklen, Equals, int64(0))
 
        _, blocklen, _, err = kc.Get(hash)
        c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
-       c.Check(err, ErrorMatches, ".*not found.*")
+       c.Check(err, ErrorMatches, ".*HTTP 403.*")
        c.Check(blocklen, Equals, int64(0))
+}
+
+func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) {
+       kp := arvados.UploadDownloadRolePermissions{}
+       if admin {
+               kp.Admin = perm
+               kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true}
+       } else {
+               kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true}
+               kp.User = perm
+       }
+
+       srv, kc, logbuf := runProxy(c, false, false, &kp)
+       defer srv.Close()
+       if admin {
+               kc.Arvados.ApiToken = arvadostest.AdminToken
+       } else {
+               kc.Arvados.ApiToken = arvadostest.ActiveToken
+       }
+
+       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+       var hash2 string
+
+       {
+               var rep int
+               var err error
+               hash2, rep, err = kc.PutB([]byte("foo"))
+
+               if perm.Upload {
+                       c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
+                       c.Check(rep, Equals, 2)
+                       c.Check(err, Equals, nil)
+                       c.Log("Finished PutB (expected success)")
+                       if admin {
+                               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="TestCase Administrator".* userUUID=zzzzz-tpzed-d9tiejq69daie8f.*`)
+                       } else {
+
+                               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="Active User".* userUUID=zzzzz-tpzed-xurymjxw79nv3jz.*`)
+                       }
+               } else {
+                       c.Check(hash2, Equals, "")
+                       c.Check(rep, Equals, 0)
+                       c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
+               }
+               logbuf.Reset()
+       }
+       if perm.Upload {
+               // can't test download without upload.
+
+               reader, blocklen, _, err := kc.Get(hash2)
+               if perm.Download {
+                       c.Assert(err, Equals, nil)
+                       all, err := ioutil.ReadAll(reader)
+                       c.Check(err, IsNil)
+                       c.Check(all, DeepEquals, []byte("foo"))
+                       c.Check(blocklen, Equals, int64(3))
+                       c.Log("Finished Get (expected success)")
+                       if admin {
+                               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="TestCase Administrator".* userUUID=zzzzz-tpzed-d9tiejq69daie8f.*`)
+                       } else {
+                               c.Check(logbuf.String(), Matches, `(?ms).* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* userFullName="Active User".* userUUID=zzzzz-tpzed-xurymjxw79nv3jz.*`)
+                       }
+               } else {
+                       c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
+                       c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*")
+                       c.Check(blocklen, Equals, int64(0))
+               }
+               logbuf.Reset()
+       }
 
 }
 
+func (s *ServerRequiredSuite) TestPutGetPermission(c *C) {
+
+       for _, adminperm := range []bool{true, false} {
+               for _, userperm := range []bool{true, false} {
+
+                       testPermission(c, true,
+                               arvados.UploadDownloadPermission{
+                                       Upload:   adminperm,
+                                       Download: true,
+                               })
+                       testPermission(c, true,
+                               arvados.UploadDownloadPermission{
+                                       Upload:   true,
+                                       Download: adminperm,
+                               })
+                       testPermission(c, false,
+                               arvados.UploadDownloadPermission{
+                                       Upload:   true,
+                                       Download: userperm,
+                               })
+                       testPermission(c, false,
+                               arvados.UploadDownloadPermission{
+                                       Upload:   true,
+                                       Download: userperm,
+                               })
+               }
+       }
+}
+
 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
-       runProxy(c, false)
-       defer closeListener()
+       srv, _, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        {
                client := http.Client{}
                req, err := http.NewRequest("OPTIONS",
-                       fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
+                       fmt.Sprintf("http://%s/%x+3", srv.Addr, md5.Sum([]byte("foo"))),
                        nil)
                c.Assert(err, IsNil)
                req.Header.Add("Access-Control-Request-Method", "PUT")
@@ -410,8 +563,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
        }
 
        {
-               resp, err := http.Get(
-                       fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
+               resp, err := http.Get(fmt.Sprintf("http://%s/%x+3", srv.Addr, md5.Sum([]byte("foo"))))
                c.Check(err, Equals, nil)
                c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
                c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
@@ -419,13 +571,13 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
-       runProxy(c, false)
-       defer closeListener()
+       srv, _, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        {
                client := http.Client{}
                req, err := http.NewRequest("POST",
-                       "http://"+listener.Addr().String()+"/",
+                       "http://"+srv.Addr+"/",
                        strings.NewReader("qux"))
                c.Check(err, IsNil)
                req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
@@ -462,8 +614,23 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
 //   With a valid but non-existing prefix (expect "\n")
 //   With an invalid prefix (expect error)
 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       getIndexWorker(c, false)
+}
+
+// Test GetIndex
+//   Uses config.yml
+//   Put one block, with 2 replicas
+//   With no prefix (expect the block locator, twice)
+//   With an existing prefix (expect the block locator, twice)
+//   With a valid but non-existing prefix (expect "\n")
+//   With an invalid prefix (expect error)
+func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
+       getIndexWorker(c, true)
+}
+
+func getIndexWorker(c *C, useConfig bool) {
+       srv, kc, _ := runProxy(c, false, useConfig, nil)
+       defer srv.Close()
 
        // Put "index-data" blocks
        data := []byte("index-data")
@@ -485,7 +652,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
        _, _, err = kc.PutB([]byte("some-more-index-data"))
        c.Check(err, IsNil)
 
-       kc.Arvados.ApiToken = arvadostest.DataManagerToken
+       kc.Arvados.ApiToken = arvadostest.SystemRootToken
 
        // Invoke GetIndex
        for _, spec := range []struct {
@@ -525,8 +692,8 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
        hash, _, err := kc.PutB([]byte("shareddata"))
        c.Check(err, IsNil)
        kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
@@ -538,8 +705,8 @@ func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        // Put a test block
        hash, rep, err := kc.PutB([]byte("foo"))
@@ -566,23 +733,23 @@ func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
                        _, _, _, err = kc.Get(hash)
                        c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
                        c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
-                       c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
+                       c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*")
                }
 
                _, _, err = kc.PutB([]byte("foo"))
-               c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
+               c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed")
        }
 }
 
 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        // Point keepproxy at a non-existent keepstore
        locals := map[string]string{
                TestProxyUUID: "http://localhost:12345",
        }
-       router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
+       srv.proxyHandler.KeepClient.SetServiceRoots(locals, nil, nil)
 
        // Ask should result in temporary bad gateway error
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -601,8 +768,8 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 }
 
 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
        hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
        for _, f := range []func() error{
@@ -624,13 +791,14 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPing(c *C) {
-       kc := runProxy(c, false)
-       defer closeListener()
+       srv, kc, _ := runProxy(c, false, false, nil)
+       defer srv.Close()
 
-       rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
+       rtr, err := newHandler(context.Background(), kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken})
+       c.Assert(err, check.IsNil)
 
        req, err := http.NewRequest("GET",
-               "http://"+listener.Addr().String()+"/_health/ping",
+               "http://"+srv.Addr+"/_health/ping",
                nil)
        c.Assert(err, IsNil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)