"Make a Keep cluster accessible to clients that are not on the LAN"
package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
"Keep storage daemon, accessible to clients on the LAN"
-package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \
"Static web hosting service for user data stored in Arvados Keep"
package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
"Arvados Websocket server"
"git.arvados.org/arvados.git/lib/lsf"
"git.arvados.org/arvados.git/lib/recovercollection"
"git.arvados.org/arvados.git/services/githttpd"
+ keepweb "git.arvados.org/arvados.git/services/keep-web"
"git.arvados.org/arvados.git/services/keepproxy"
"git.arvados.org/arvados.git/services/keepstore"
"git.arvados.org/arvados.git/services/ws"
"git-httpd": githttpd.Command,
"install": install.Command,
"init": install.InitCommand,
+ "keep-web": keepweb.Command,
"keepproxy": keepproxy.Command,
"keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
# SPDX-License-Identifier: AGPL-3.0
[Unit]
-Description=Arvados Keep web gateway
+Description=Arvados Keep WebDAV gateway
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/config.yml
# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
Type=notify
+EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/keep-web
# Set a reasonable default for the open file limit
LimitNOFILE=65536
runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
- runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
+ runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
// fmt.Println(args[0])
// return 2
// }),
-// })("/usr/bin/multi", []string{"foobar", "baz"}))
+// })("/usr/bin/multi", []string{"foobar", "baz"}, os.Stdin, os.Stdout, os.Stderr))
//
// ...prints "baz" and exits 2.
type Multi map[string]Handler
"services/crunch-dispatch-slurm",
"services/health",
"services/keep-balance",
- "services/keep-web",
"services/keepstore",
"services/ws",
} {
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-systemd/daemon"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
)
type Handler interface {
http.Handler
CheckHealth() error
+ // Done returns a channel that closes when the handler shuts
+ // itself down, or nil if this never happens.
Done() <-chan struct{}
}
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
+
+ // prog is [keepstore, keep-web, git-httpd, ...] but the
+ // legacy config flags are [-legacy-keepstore-config,
+ // -legacy-keepweb-config, -legacy-git-httpd-config, ...]
+ legacyFlag := "-legacy-" + strings.Replace(prog, "keep-", "keep", 1) + "-config"
+ args = loader.MungeLegacyConfigArgs(log, args, legacyFlag)
+
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
pprofAddr := flags.String("pprof", "", "Serve Go profile data at `[addr]:port`")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
httpserver.AddRequestIDs(
httpserver.LogRequests(
interceptHealthReqs(cluster.ManagementToken, handler.CheckHealth,
- interceptMetricsReqs(cluster.ManagementToken, reg, log,
- httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))))
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg))))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
+ Handler: ifCollectionInHost(instrumented, instrumented.ServeAPI(cluster.ManagementToken, instrumented)),
BaseContext: func(net.Listener) context.Context { return ctx },
},
Addr: listenURL.Host,
return 0
}
+// If an incoming request's target vhost has an embedded collection
+// UUID or PDH, handle it with hTrue, otherwise handle it with
+// hFalse.
+//
+// Facilitates routing "http://collections.example/metrics" to metrics
+// and "http://{uuid}.collections.example/metrics" to a file in a
+// collection.
+func ifCollectionInHost(hTrue, hFalse http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if arvados.CollectionIDFromDNSName(r.Host) != "" {
+ hTrue.ServeHTTP(w, r)
+ } else {
+ hFalse.ServeHTTP(w, r)
+ }
+ })
+}
+
func interceptHealthReqs(mgtToken string, checkHealth func() error, next http.Handler) http.Handler {
mux := httprouter.New()
mux.Handler("GET", "/_health/ping", &health.Handler{
Routes: health.Routes{"ping": checkHealth},
})
mux.NotFound = next
- return mux
-}
-
-func interceptMetricsReqs(mgtToken string, reg *prometheus.Registry, log logrus.FieldLogger, next http.Handler) http.Handler {
- mux := httprouter.New()
- metricsH := auth.RequireLiteralToken(mgtToken,
- promhttp.HandlerFor(reg, promhttp.HandlerOpts{
- ErrorLog: log,
- }))
- mux.Handler("GET", "/metrics", metricsH)
- mux.Handler("GET", "/metrics.json", metricsH)
- mux.NotFound = next
- return mux
+ return ifCollectionInHost(next, mux)
}
func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) {
"crypto/md5"
"fmt"
"regexp"
+ "strings"
"time"
"git.arvados.org/arvados.git/sdk/go/blockdigest"
)
+var (
+ UUIDMatch = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
+ PDHMatch = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+)
+
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid"`
})
return fmt.Sprintf("%x+%d", h.Sum(nil), size)
}
+
+// CollectionIDFromDNSName returns a UUID or PDH if s begins with a
+// UUID or URL-encoded PDH; otherwise "".
+func CollectionIDFromDNSName(s string) string {
+ // Strip domain.
+ if i := strings.IndexRune(s, '.'); i >= 0 {
+ s = s[:i]
+ }
+ // Names like {uuid}--collections.example.com serve the same
+ // purpose as {uuid}.collections.example.com but can reduce
+ // cost/effort of using [additional] wildcard certificates.
+ if i := strings.Index(s, "--"); i >= 0 {
+ s = s[:i]
+ }
+ if UUIDMatch(s) {
+ return s
+ }
+ if pdh := strings.Replace(s, "-", "+", 1); PDHMatch(pdh) {
+ return pdh
+ }
+ return ""
+}
"net/http"
"net/url"
"os"
- "regexp"
"strings"
"sync"
"time"
type StringMatcher func(string) bool
-var UUIDMatch StringMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
-var PDHMatch StringMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+var UUIDMatch StringMatcher = arvados.UUIDMatch
+var PDHMatch StringMatcher = arvados.PDHMatch
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
AnonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
DataManagerToken = "320mkve8qkswstz7ff61glpk3mhgghmg67wmic7elw4z41pke1"
SystemRootToken = "systemusertesttoken1234567890aoeuidhtnsqjkxbmwvzpy"
- ManagementToken = "jg3ajndnq63sywcd50gbs5dskdc9ckkysb0nsqmfz08nwf17nl"
+ ManagementToken = "e687950a23c3a9bceec28c6223a06c79"
ActiveUserUUID = "zzzzz-tpzed-xurymjxw79nv3jz"
FederatedActiveUserUUID = "zbbbb-tpzed-xurymjxw79nv3jz"
SpectatorUserUUID = "zzzzz-tpzed-l1s2piq4t4mps8r"
# If keepproxy and/or keep-web is running, send SIGHUP to make
# them discover the new keepstore services.
for svc in ('keepproxy', 'keep-web'):
- pidfile = _pidfile('keepproxy')
+ pidfile = _pidfile(svc)
if os.path.exists(pidfile):
try:
with open(pidfile) as pid:
env = os.environ.copy()
logf = open(_logfilename('keep-web'), WRITE_MODE)
keepweb = subprocess.Popen(
- ['keep-web'],
+ ['arvados-server', 'keep-web'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('keep-web'), 'w') as f:
f.write(str(keepweb.pid))
keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)])
keep_web_port = find_available_port()
keep_web_external_port = find_available_port()
- keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
configsrc = os.environ.get("CONFIGSRC", None)
"WebDAVDownload": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_dl_port): {},
+ "http://%s:%s"%(localhost, keep_web_port): {},
},
},
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"sync"
type cache struct {
cluster *arvados.Cluster
- config *arvados.WebDAVCacheConfig // TODO: use cluster.Collections.WebDAV instead
logger logrus.FieldLogger
registry *prometheus.Registry
metrics cacheMetrics
func (c *cache) setup() {
var err error
- c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries)
+ c.pdhs, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxUUIDEntries)
if err != nil {
panic(err)
}
- c.collections, err = lru.New2Q(c.config.MaxCollectionEntries)
+ c.collections, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxCollectionEntries)
if err != nil {
panic(err)
}
- c.sessions, err = lru.New2Q(c.config.MaxSessions)
+ c.sessions, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxSessions)
if err != nil {
panic(err)
}
return err
}
c.collections.Add(client.AuthToken+"\000"+updated.PortableDataHash, &cachedCollection{
- expire: time.Now().Add(time.Duration(c.config.TTL)),
+ expire: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
collection: &updated,
})
c.pdhs.Add(coll.UUID, &cachedPDH{
- expire: time.Now().Add(time.Duration(c.config.TTL)),
- refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+ expire: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
+ refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
pdh: updated.PortableDataHash,
})
return nil
if sess == nil {
c.metrics.sessionMisses.Inc()
sess = &cachedSession{
- expire: now.Add(c.config.TTL.Duration()),
+ expire: now.Add(c.cluster.Collections.WebDAVCache.TTL.Duration()),
}
var err error
sess.client, err = arvados.NewClientFromConfig(c.cluster)
return nil, err
}
c.logger.Debugf("cache(%s): retrieved manifest, caching with pdh %s", targetID, retrieved.PortableDataHash)
- exp := time.Now().Add(time.Duration(c.config.TTL))
+ exp := time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL))
if targetID != retrieved.PortableDataHash {
c.pdhs.Add(targetID, &cachedPDH{
expire: exp,
- refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+ refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
pdh: retrieved.PortableDataHash,
})
}
expire: exp,
collection: &retrieved,
})
- if int64(len(retrieved.ManifestText)) > c.config.MaxCollectionBytes/int64(c.config.MaxCollectionEntries) {
+ if int64(len(retrieved.ManifestText)) > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/int64(c.cluster.Collections.WebDAVCache.MaxCollectionEntries) {
select {
case c.chPruneCollections <- struct{}{}:
default:
}
}
for i, k := range keys {
- if size <= c.config.MaxCollectionBytes/2 {
+ if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
break
}
if expired[i] {
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
// Hit the same collection 5 times using the same token. Only
// the first req should cause an API call; the next 4 should
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollectionPDH, forceReload)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
}
func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
-
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
},
} {
- c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ c.Logf("%s %+v", s.testServer.URL, trial)
if skip != nil && skip(trial.path) {
c.Log("(skip)")
continue
c.Assert(err, check.IsNil)
defer os.RemoveAll(tempdir)
- cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
+ cmd := exec.Command("cadaver", s.testServer.URL+path)
if password != "" {
// cadaver won't try username/password authentication
// unless the server responds 401 to an
// unauthenticated request, which it only does in
// AttachmentOnlyHost, TrustAllContent, and
// per-collection vhost cases.
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.URL[7:]
cmd.Env = append(os.Environ(), "HOME="+tempdir)
f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
//
// Proxy configuration
//
-// Keep-web does not support TLS natively. Typically, it is installed
-// behind a proxy like nginx.
+// Typically, keep-web is installed behind a proxy like nginx.
//
// Here is an example nginx configuration.
//
// }
//
// It is not necessary to run keep-web on the same host as the nginx
-// proxy. However, TLS is not used between nginx and keep-web, so
+// proxy. However, if TLS is not used between nginx and keep-web, the
// intervening networks must be secured by other means.
//
// Anonymous downloads
// /metrics. The same information is also available as JSON at
// /metrics.json.
//
-package main
+package keepweb
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"encoding/json"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
type handler struct {
- Config *Config
- MetricsAPI http.Handler
- clientPool *arvadosclient.ClientPool
- setupOnce sync.Once
- healthHandler http.Handler
- webdavLS webdav.LockSystem
-}
-
-// parseCollectionIDFromDNSName returns a UUID or PDH if s begins with
-// a UUID or URL-encoded PDH; otherwise "".
-func parseCollectionIDFromDNSName(s string) string {
- // Strip domain.
- if i := strings.IndexRune(s, '.'); i >= 0 {
- s = s[:i]
- }
- // Names like {uuid}--collections.example.com serve the same
- // purpose as {uuid}.collections.example.com but can reduce
- // cost/effort of using [additional] wildcard certificates.
- if i := strings.Index(s, "--"); i >= 0 {
- s = s[:i]
- }
- if arvadosclient.UUIDMatch(s) {
- return s
- }
- if pdh := strings.Replace(s, "-", "+", 1); arvadosclient.PDHMatch(pdh) {
- return pdh
- }
- return ""
+ Cache cache
+ Cluster *arvados.Cluster
+ clientPool *arvadosclient.ClientPool
+ setupOnce sync.Once
+ webdavLS webdav.LockSystem
}
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
func (h *handler) setup() {
// Errors will be handled at the client pool.
- arv, _ := arvados.NewClientFromConfig(h.Config.cluster)
+ arv, _ := arvados.NewClientFromConfig(h.Cluster)
h.clientPool = arvadosclient.MakeClientPoolWith(arv)
- keepclient.RefreshServiceDiscoveryOnSIGHUP()
- keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
-
- h.healthHandler = &health.Handler{
- Token: h.Config.cluster.ManagementToken,
- Prefix: "/_health/",
- }
+ keepclient.DefaultBlockCache.MaxBlocks = h.Cluster.Collections.WebDAVCache.MaxBlockEntries
// Even though we don't accept LOCK requests, every webdav
// handler must have a non-nil LockSystem.
}
}
+// CheckHealth implements service.Handler.
+func (h *handler) CheckHealth() error {
+ return nil
+}
+
+// Done implements service.Handler.
+func (h *handler) Done() <-chan struct{} {
+ return nil
+}
+
// ServeHTTP implements http.Handler.
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h.setupOnce.Do(h.setup)
w := httpserver.WrapResponseWriter(wOrig)
- if strings.HasPrefix(r.URL.Path, "/_health/") && r.Method == "GET" {
- h.healthHandler.ServeHTTP(w, r)
- return
- }
-
if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" {
if !browserMethod[method] && !webdavMethod[method] {
w.WriteHeader(http.StatusMethodNotAllowed)
var pathToken bool
var attachment bool
var useSiteFS bool
- credentialsOK := h.Config.cluster.Collections.TrustAllContent
+ credentialsOK := h.Cluster.Collections.TrustAllContent
reasonNotAcceptingCredentials := ""
- if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Config.cluster.Services.WebDAVDownload.ExternalURL.Host) {
+ if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Cluster.Services.WebDAVDownload.ExternalURL.Host) {
credentialsOK = true
attachment = true
} else if r.FormValue("disposition") == "attachment" {
if !credentialsOK {
reasonNotAcceptingCredentials = fmt.Sprintf("vhost %q does not specify a single collection ID or match Services.WebDAVDownload.ExternalURL %q, and Collections.TrustAllContent is false",
- r.Host, h.Config.cluster.Services.WebDAVDownload.ExternalURL)
+ r.Host, h.Cluster.Services.WebDAVDownload.ExternalURL)
}
- if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
+ if collectionID = arvados.CollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if strings.HasPrefix(r.URL.Path, "/metrics") {
- h.MetricsAPI.ServeHTTP(w, r)
- return
} else if siteFSDir[pathParts[0]] {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
if tokens == nil {
tokens = reqTokens
- if h.Config.cluster.Users.AnonymousUserToken != "" {
- tokens = append(tokens, h.Config.cluster.Users.AnonymousUserToken)
+ if h.Cluster.Users.AnonymousUserToken != "" {
+ tokens = append(tokens, h.Cluster.Users.AnonymousUserToken)
}
}
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
+ collection, err = h.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
}
// Check configured permission
- _, sess, err := h.Config.Cache.GetSession(arv.ApiToken)
- tokenUser, err = h.Config.Cache.GetTokenUser(arv.ApiToken)
+ _, sess, err := h.Cache.GetSession(arv.ApiToken)
+ tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
if webdavMethod[r.Method] {
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
ResponseWriter: w,
logger: ctxlog.FromContext(r.Context()),
update: func() error {
- return h.Config.Cache.Update(client, *collection, writefs)
+ return h.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
return
}
- fs, sess, err := h.Config.Cache.GetSession(tokens[0])
+ fs, sess, err := h.Cache.GetSession(tokens[0])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
+ fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution)
f, err := fs.Open(r.URL.Path)
if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
- tokenUser, err := h.Config.Cache.GetTokenUser(tokens[0])
+ tokenUser, err := h.Cache.GetTokenUser(tokens[0])
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return
var permitDownload bool
var permitUpload bool
if tokenUser != nil && tokenUser.IsAdmin {
- permitUpload = h.Config.cluster.Collections.WebDAVPermission.Admin.Upload
- permitDownload = h.Config.cluster.Collections.WebDAVPermission.Admin.Download
+ permitUpload = h.Cluster.Collections.WebDAVPermission.Admin.Upload
+ permitDownload = h.Cluster.Collections.WebDAVPermission.Admin.Download
} else {
- permitUpload = h.Config.cluster.Collections.WebDAVPermission.User.Upload
- permitDownload = h.Config.cluster.Collections.WebDAVPermission.User.Download
+ permitUpload = h.Cluster.Collections.WebDAVPermission.User.Upload
+ permitDownload = h.Cluster.Collections.WebDAVPermission.User.Download
}
if (method == "PUT" || method == "POST") && !permitUpload {
// Disallow operations that upload new files.
WithField("user_full_name", user.FullName)
useruuid = user.UUID
} else {
- useruuid = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Config.cluster.ClusterID)
+ useruuid = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Cluster.ClusterID)
}
if collection == nil && fs != nil {
collection, filepath = h.determineCollection(fs, filepath)
}
if r.Method == "PUT" || r.Method == "POST" {
log.Info("File upload")
- if h.Config.cluster.Collections.WebDAVLogEvents {
+ if h.Cluster.Collections.WebDAVLogEvents {
go func() {
lr := arvadosclient.Dict{"log": arvadosclient.Dict{
"object_uuid": useruuid,
props["portable_data_hash"] = collection.PortableDataHash
}
log.Info("File download")
- if h.Config.cluster.Collections.WebDAVLogEvents {
+ if h.Cluster.Collections.WebDAVLogEvents {
go func() {
lr := arvadosclient.Dict{"log": arvadosclient.Dict{
"object_uuid": useruuid,
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
}
type UnitSuite struct {
- Config *arvados.Config
+ cluster *arvados.Cluster
+ handler *handler
}
func (s *UnitSuite) SetUpTest(c *check.C) {
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
+ logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
ldr.Path = "-"
cfg, err := ldr.Load()
c.Assert(err, check.IsNil)
- s.Config = cfg
+ cc, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.cluster = cc
+ s.handler = &handler{
+ Cluster: cc,
+ Cache: cache{
+ cluster: cc,
+ logger: logger,
+ registry: prometheus.NewRegistry(),
+ },
+ }
}
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
+ h := s.handler
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
req := &http.Request{
Method: "OPTIONS",
c.Assert(err, check.IsNil)
}
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
req := &http.Request{
Method: "GET",
req = req.WithContext(ctxlog.Context(context.Background(), logger))
resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, trial.expectStatus)
c.Check(resp.Body.String(), check.Equals, "")
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- cfg := newConfig(ctxlog.TestLogger(c), s.Config)
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- h := handler{Config: cfg}
- h.ServeHTTP(resp, req)
+ s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
}
}
URL: u,
RequestURI: u.RequestURI(),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
for _, port := range []string{"80", "443", "8000"} {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
req := &http.Request{
Method: "GET",
func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return req, resp
}
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- s.testServer.Config.cluster.Collections.TrustAllContent = true
+ s.handler.Cluster.Collections.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
+ s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
}
func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
}
func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
arv := arvados.NewClientFromEnv()
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
name := "foo/bar/baz"
nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, expectRegexp)
}
}.Encode())),
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
},
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
c.Check(resp.Body.String(), check.Equals, expectRespBody)
}()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return resp
}
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Header().Get("Location"), check.Equals, "")
return resp
}
func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+ s.handler.Cluster.Users.AnonymousUserToken = ""
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
authHeader := http.Header{
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
RequestURI: u.RequestURI(),
Header: copyHeader(trial.header),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
for resp.Code == http.StatusSeeOther {
u, _ := req.URL.Parse(resp.Header().Get("Location"))
req.AddCookie(c)
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
}
if trial.redirect != "" {
c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
}
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
var updated arvados.Collection
for _, fnm := range []string{"foo.txt", "bar.txt"} {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
req := &http.Request{
Method: "DELETE",
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNoContent)
updated = arvados.Collection{}
c.Check(updated.ManifestText, check.Equals, "")
}
-func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
- s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
- authHeader := http.Header{
- "Authorization": {"Bearer " + arvadostest.ManagementToken},
- }
-
- resp := httptest.NewRecorder()
- u := mustParseURL("http://download.example.com/_health/ping")
- req := &http.Request{
- Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
- Header: authHeader,
- }
- s.testServer.Handler.ServeHTTP(resp, req)
-
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
-}
-
func (s *IntegrationSuite) TestFileContentType(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- arv, err := arvadosclient.New(&client)
+ arv, err := arvadosclient.New(client)
c.Assert(err, check.Equals, nil)
kc, err := keepclient.MakeKeepClient(arv)
c.Assert(err, check.Equals, nil)
- fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+ fs, err := (&arvados.Collection{}).FileSystem(client, kc)
c.Assert(err, check.IsNil)
trials := []struct {
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
c.Check(resp.Body.String(), check.Equals, trial.content)
}
func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
- s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
req := &http.Request{
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
}
req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
req.Host = req.URL.Host
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, status)
}
reqPut.Host = req.URL.Host
reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, &reqPut)
+ s.handler.ServeHTTP(resp, &reqPut)
c.Check(resp.Code, check.Equals, http.StatusCreated)
// new file should not appear in colls[1]
return hc
}
-func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, req *http.Request,
+func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.AdminToken
var logentries arvados.LogList
limit1 := 1
logger.Out = &logbuf
resp := httptest.NewRecorder()
req = req.WithContext(ctxlog.Context(context.Background(), logger))
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if perm {
c.Check(resp.Result().StatusCode, check.Equals, successCode)
}
func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
- config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
- h := handler{Config: config}
u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
- config.cluster.Collections.TrustAllContent = true
+ s.handler.Cluster.Collections.TrustAllContent = true
for _, adminperm := range []bool{true, false} {
for _, userperm := range []bool{true, false} {
- config.cluster.Collections.WebDAVPermission.Admin.Download = adminperm
- config.cluster.Collections.WebDAVPermission.User.Download = userperm
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
// Test admin permission
req := &http.Request{
"Authorization": {"Bearer " + arvadostest.AdminToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", adminperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
// Test user permission
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", userperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
}
}
- config.cluster.Collections.WebDAVPermission.User.Download = true
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
"http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
}
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
}
func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
- config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
- h := handler{Config: config}
-
for _, adminperm := range []bool{true, false} {
for _, userperm := range []bool{true, false} {
- arv := s.testServer.Config.Client
+ arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
var coll arvados.Collection
u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
- config.cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
- config.cluster.Collections.WebDAVPermission.User.Upload = userperm
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
// Test admin permission
req := &http.Request{
},
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", adminperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
arvadostest.AdminUserUUID, coll.UUID, "bar")
// Test user permission
},
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", userperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
arvadostest.ActiveUserUUID, coll.UUID, "bar")
}
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"context"
- "flag"
- "fmt"
"mime"
"os"
- "git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/coreos/go-systemd/daemon"
- "github.com/ghodss/yaml"
- "github.com/sirupsen/logrus"
- log "github.com/sirupsen/logrus"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
)
var (
version = "dev"
)
-// Config specifies server configuration.
-type Config struct {
- Client arvados.Client
- Cache cache
- cluster *arvados.Cluster
-}
-
-func newConfig(logger logrus.FieldLogger, arvCfg *arvados.Config) *Config {
- cfg := Config{}
- var cls *arvados.Cluster
- var err error
- if cls, err = arvCfg.GetCluster(""); err != nil {
- log.Fatal(err)
- }
- cfg.cluster = cls
- cfg.Cache.config = &cfg.cluster.Collections.WebDAVCache
- cfg.Cache.cluster = cls
- cfg.Cache.logger = logger
- return &cfg
-}
-
-func init() {
- // MakeArvadosClient returns an error if this env var isn't
- // available as a default token (even if we explicitly set a
- // different token before doing anything with the client). We
- // set this dummy value during init so it doesn't clobber the
- // one used by "run test servers".
- if os.Getenv("ARVADOS_API_TOKEN") == "" {
- os.Setenv("ARVADOS_API_TOKEN", "xxx")
- }
+var Command = service.Command(arvados.ServiceNameKeepweb, newHandlerOrErrorHandler)
- log.SetFormatter(&log.JSONFormatter{
- TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
- })
-}
-
-func configure(logger log.FieldLogger, args []string) (*Config, error) {
- flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
-
- loader := config.NewLoader(os.Stdin, logger)
- loader.SetupFlags(flags)
-
- dumpConfig := flags.Bool("dump-config", false,
- "write current configuration to stdout and exit")
- getVersion := flags.Bool("version", false,
- "print version information and exit.")
-
- prog := args[0]
- args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepweb-config")
- if ok, code := cmd.ParseFlags(flags, prog, args, "", os.Stderr); !ok {
- os.Exit(code)
- } else if *getVersion {
- fmt.Printf("%s %s\n", args[0], version)
- return nil, nil
- }
-
- arvCfg, err := loader.Load()
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ h, err := newHandler(ctx, cluster, token, reg)
if err != nil {
- return nil, err
- }
- cfg := newConfig(logger, arvCfg)
-
- if *dumpConfig {
- out, err := yaml.Marshal(cfg)
- if err != nil {
- return nil, err
- }
- _, err = os.Stdout.Write(out)
- return nil, err
+ return service.ErrorHandler(ctx, cluster, err)
}
- return cfg, nil
+ return h
}
-func main() {
- initLogger := log.StandardLogger()
- logger := initLogger.WithField("PID", os.Getpid())
- cfg, err := configure(logger, os.Args)
- if err != nil {
- log.Fatal(err)
- } else if cfg == nil {
- return
- }
- logger = logger.WithField("ClusterID", cfg.cluster.ClusterID)
- logger.Printf("keep-web %s started", version)
- ctx := ctxlog.Context(context.Background(), logger)
-
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) (*handler, error) {
+ logger := ctxlog.FromContext(ctx)
if ext := ".txt"; mime.TypeByExtension(ext) == "" {
- log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
- }
-
- os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
- srv := &server{Config: cfg}
- if err := srv.Start(ctx, initLogger); err != nil {
- logger.Fatal(err)
- }
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- logger.Printf("Error notifying init daemon: %v", err)
- }
- logger.Println("Listening at", srv.Addr)
- if err := srv.Wait(); err != nil {
- logger.Fatal(err)
- }
+ logger.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
+ }
+
+ keepclient.RefreshServiceDiscoveryOnSIGHUP()
+ os.Setenv("ARVADOS_API_HOST", cluster.Services.Controller.ExternalURL.Host)
+ return &handler{
+ Cluster: cluster,
+ Cache: cache{
+ cluster: cluster,
+ logger: logger,
+ registry: reg,
+ },
+ }, nil
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"fmt"
"Range": {"bytes=" + trial.header},
},
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expectObey {
c.Check(resp.Code, check.Equals, http.StatusPartialContent)
c.Check(resp.Body.Len(), check.Equals, len(trial.expectBody))
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"crypto/hmac"
}
client := (&arvados.Client{
- APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host,
- Insecure: h.Config.cluster.TLS.Insecure,
+ APIHost: h.Cluster.Services.Controller.ExternalURL.Host,
+ Insecure: h.Cluster.TLS.Insecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
var aca arvados.APIClientAuthorization
var secret string
if len(key) == 27 && key[5:12] == "-gj3su-" {
// Access key is the UUID of an Arvados token, secret
// key is the secret part.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Cluster.SystemRootToken)
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil)
secret = aca.APIToken
} else {
// Use a single session (cached FileSystem) across
// multiple read requests.
var sess *cachedSession
- fs, sess, err = h.Config.Cache.GetSession(token)
+ fs, sess, err = h.Cache.GetSession(token)
if err != nil {
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
defer release()
fs = client.SiteFileSystem(kc)
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
+ fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution)
}
var objectNameGiven bool
var bucketName string
fspath := "/by_id"
- if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ if id := arvados.CollectionIDFromDNSName(r.Host); id != "" {
fspath += "/" + id
bucketName = id
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
fmt.Fprintln(w, `<LocationConstraint><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
- h.Config.cluster.ClusterID+
+ h.Cluster.ClusterID+
`</LocationConstraint></LocationConstraint>`)
} else if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
// GetBucketWebsite ("GET /bucketid/?website"), GetBucketTagging, etc.
}
return true
}
- if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Config.cluster.Collections.S3FolderObjects {
+ if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Cluster.Collections.S3FolderObjects {
w.Header().Set("Content-Type", "application/x-directory")
w.WriteHeader(http.StatusOK)
return true
return true
}
- tokenUser, err := h.Config.Cache.GetTokenUser(token)
+ tokenUser, err := h.Cache.GetTokenUser(token)
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
}
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
- if !h.Config.cluster.Collections.S3FolderObjects {
+ if !h.Cluster.Collections.S3FolderObjects {
s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest)
return true
}
}
defer f.Close()
- tokenUser, err := h.Config.Cache.GetTokenUser(token)
+ tokenUser, err := h.Cache.GetTokenUser(token)
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
return true
}
// Ensure a subsequent read operation will see the changes.
- h.Config.Cache.ResetSession(token)
+ h.Cache.ResetSession(token)
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
return true
}
// Ensure a subsequent read operation will see the changes.
- h.Config.Cache.ResetSession(token)
+ h.Cache.ResetSession(token)
w.WriteHeader(http.StatusNoContent)
return true
default:
if path < params.marker || path < params.prefix || path <= params.startAfter {
return nil
}
- if fi.IsDir() && !h.Config.cluster.Collections.S3FolderObjects {
+ if fi.IsDir() && !h.Cluster.Collections.S3FolderObjects {
// Note we don't add anything to
// commonPrefixes here even if delimiter is
// "/". We descend into the directory, and
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
region := aws.Region{
Name: "zzzzz",
- S3Endpoint: "http://" + s.testServer.Addr,
+ S3Endpoint: s.testServer.URL,
}
client := s3.New(*auth, region)
client.Signature = aws.V4Signature
c.Check(err, check.IsNil)
rdr, err := bucket.GetReader(objname)
- if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
+ if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
c.Check(err, check.NotNil)
continue
} else if !c.Check(err, check.IsNil) {
s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
}
func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
- s.testServer.Config.cluster.Collections.S3FolderObjects = true
+ s.handler.Cluster.Collections.S3FolderObjects = true
for _, trial := range []struct {
path string
}{
s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
}
func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
- s.testServer.Config.cluster.Collections.S3FolderObjects = false
+ s.handler.Cluster.Collections.S3FolderObjects = false
var wg sync.WaitGroup
for _, trial := range []struct {
c.Assert(err, check.IsNil)
s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
rr := httptest.NewRecorder()
- s.testServer.Server.Handler.ServeHTTP(rr, req)
+ s.handler.ServeHTTP(rr, req)
resp := rr.Result()
c.Check(resp.StatusCode, check.Equals, trial.responseCode)
body, err := ioutil.ReadAll(resp.Body)
defer stage.teardown(c)
var markers int
- for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
+ for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
dirs := 2
filesPerDir := 1001
stage.writeBigDirs(c, dirs, filesPerDir)
}
}
func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
- c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
+ c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
expectPageSize := pageSize
if expectPageSize > 1000 {
expectPageSize = 1000
}
func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
- for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
+ for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
s.testS3CollectionListRollup(c)
}
}
}
}
markers := 0
- if s.testServer.Config.cluster.Collections.S3FolderObjects {
+ if s.handler.Cluster.Collections.S3FolderObjects {
markers = 1
}
c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
Region: aws_aws.String("auto"),
- Endpoint: aws_aws.String("http://" + s.testServer.Addr),
+ Endpoint: aws_aws.String(s.testServer.URL),
Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
S3ForcePathStyle: aws_aws.Bool(true),
}))
sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
Region: aws_aws.String("auto"),
- Endpoint: aws_aws.String("http://" + s.testServer.Addr),
+ Endpoint: aws_aws.String(s.testServer.URL),
Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
S3ForcePathStyle: aws_aws.Bool(true),
}))
stage := s.s3setup(c)
defer stage.teardown(c)
- cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
+ cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
buf, err := cmd.CombinedOutput()
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
// keep-web's signature verification wrt chars like "|"
// (neither reserved nor unreserved) and "," (not normally
// percent-encoded in a path).
- cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar")
+ tmpfile := c.MkDir() + "/dstfile"
+ cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
buf, err = cmd.CombinedOutput()
c.Check(err, check.NotNil)
c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
if service == "s3" {
return aws.Endpoint{
- URL: "http://" + s.testServer.Addr,
+ URL: s.testServer.URL,
SigningRegion: "custom-signing-region",
}, nil
}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "context"
- "net"
- "net/http"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
-)
-
-type server struct {
- httpserver.Server
- Config *Config
-}
-
-func (srv *server) Start(ctx context.Context, logger *logrus.Logger) error {
- h := &handler{Config: srv.Config}
- reg := prometheus.NewRegistry()
- h.Config.Cache.registry = reg
- // Warning: when updating this to use Command() from
- // lib/service, make sure to implement an exemption in
- // httpserver.HandlerWithDeadline() so large file uploads are
- // allowed to take longer than the usual API.RequestTimeout.
- // See #13697.
- mh := httpserver.Instrument(reg, logger, httpserver.AddRequestIDs(httpserver.LogRequests(h)))
- h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler())
- srv.Handler = mh
- srv.BaseContext = func(net.Listener) context.Context { return ctx }
- var listen arvados.URL
- for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs {
- break
- }
- if len(srv.Config.cluster.Services.WebDAV.InternalURLs) > 1 {
- logrus.Warn("Services.WebDAV.InternalURLs has more than one key; picked: ", listen)
- }
- srv.Addr = listen.Host
- return srv.Server.Start()
-}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
"io/ioutil"
"net"
"net/http"
+ "net/http/httptest"
"os"
"os/exec"
+ "regexp"
"strings"
"testing"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
// IntegrationSuite tests need an API server and a keep-web server
type IntegrationSuite struct {
- testServer *server
- ArvConfig *arvados.Config
+ testServer *httptest.Server
+ handler *handler
}
func (s *IntegrationSuite) TestNoToken(c *check.C) {
}
func (s *IntegrationSuite) Test200(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
for _, spec := range []curlCase{
// My collection
{
// Return header block and body.
func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
- testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
+ testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
if strings.Contains(auth, " ") {
// caller supplied entire Authorization header value
return
}
+// Run a full-featured server, including the metrics/health routes
+// that are added by service.Command.
+func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
+ logbuf = &bytes.Buffer{}
+ cluster = *s.handler.Cluster
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+
+ var configjson bytes.Buffer
+ json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
+ go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
+ for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
+ if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
+ srvaddr = "http://" + m[1]
+ break
+ }
+ }
+ if srvaddr == "" {
+ c.Fatal("timed out")
+ }
+ return
+}
+
+// Ensure uploads can take longer than API.RequestTimeout.
+//
+// Currently, this works only by accident: service.Command cancels the
+// request context as usual (there is no exemption), but
+// webdav.Handler doesn't notice if the request context is cancelled
+// while waiting to send or receive file data.
+func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
+ s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
+ _, srvaddr, _ := s.runServer(c)
+
+ var coll arvados.Collection
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveTokenV2
+ err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+ c.Assert(err, check.IsNil)
+
+ pr, pw := io.Pipe()
+ go func() {
+ time.Sleep(time.Second)
+ pw.Write(make([]byte, 10000000))
+ pw.Close()
+ }()
+ req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ time.Sleep(time.Second)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(len(body), check.Equals, 10000000)
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+ cluster, srvaddr, _ := s.runServer(c)
+ req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
+ req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ body, _ := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
+}
+
func (s *IntegrationSuite) TestMetrics(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
- origin := "http://" + s.testServer.Addr
- req, _ := http.NewRequest("GET", origin+"/notfound", nil)
+ cluster, srvaddr, _ := s.runServer(c)
+
+ req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
_, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
for i := 0; i < 2; i++ {
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
req.Host = arvadostest.FooCollection + ".example.com"
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err = http.DefaultClient.Do(req)
resp.Body.Close()
}
- s.testServer.Config.Cache.updateGauges()
+ time.Sleep(metricsUpdateInterval * 2)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer badtoken")
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
- // must not intercept that route.
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
- req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp, err = http.DefaultClient.Do(req)
- c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ // must not intercept that route. Ditto health check paths.
+ for _, path := range []string{"/metrics.json", "/_health/ping"} {
+ c.Logf("path: %q", path)
+ req, _ = http.NewRequest("GET", srvaddr+path, nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ }
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
- ldr.Path = "-"
- arvCfg, err := ldr.Load()
- c.Check(err, check.IsNil)
- cfg := newConfig(ctxlog.TestLogger(c), arvCfg)
- c.Assert(err, check.IsNil)
- cfg.Client = arvados.Client{
- APIHost: testAPIHost,
- Insecure: true,
- }
- listen := "127.0.0.1:0"
- cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.ManagementToken = arvadostest.ManagementToken
- cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- s.ArvConfig = arvCfg
- s.testServer = &server{Config: cfg}
logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(&bytes.Buffer{}, logger)
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+
ctx := ctxlog.Context(context.Background(), logger)
- err = s.testServer.Start(ctx, logger)
- c.Assert(err, check.Equals, nil)
+
+ s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
+ s.testServer = httptest.NewUnstartedServer(
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ s.handler)))
+ s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
+ s.testServer.Start()
+
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
- var err error
if s.testServer != nil {
- err = s.testServer.Close()
+ s.testServer.Close()
}
- c.Check(err, check.Equals, nil)
}
// Gocheck boilerplate
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"encoding/json"
"net/url"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
"gopkg.in/check.v1"
)
func (s *UnitSuite) TestStatus(c *check.C) {
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
u, _ := url.Parse("http://keep-web.example/status.json")
req := &http.Request{
Method: "GET",
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
var status map[string]interface{}
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"crypto/rand"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import "golang.org/x/net/webdav"
. /usr/local/lib/arvbox/common.sh
. /usr/local/lib/arvbox/go-setup.sh
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/keep-web"
-install $GOPATH/bin/keep-web /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server keep-web)
if test "$1" = "--only-deps" ; then
exit
fi
+flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
exec /usr/local/bin/keep-web
projects / arvados.git / commitdiff