services/keep-web/handler.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "encoding/json"
   9         "fmt"
  10         "html"
  11         "html/template"
  12         "io"
  13         "net/http"
  14         "net/url"
  15         "os"
  16         "sort"
  17         "strconv"
  18         "strings"
  19         "sync"
  20
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/auth"
  24         "git.curoverse.com/arvados.git/sdk/go/health"
  25         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  26         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  27         "golang.org/x/net/webdav"
  28 )
  29
  30 type handler struct {
  31         Config        *Config
  32         clientPool    *arvadosclient.ClientPool
  33         setupOnce     sync.Once
  34         healthHandler http.Handler
  35         webdavLS      webdav.LockSystem
  36 }
  37
  38 // parseCollectionIDFromDNSName returns a UUID or PDH if s begins with
  39 // a UUID or URL-encoded PDH; otherwise "".
  40 func parseCollectionIDFromDNSName(s string) string {
  41         // Strip domain.
  42         if i := strings.IndexRune(s, '.'); i >= 0 {
  43                 s = s[:i]
  44         }
  45         // Names like {uuid}--collections.example.com serve the same
  46         // purpose as {uuid}.collections.example.com but can reduce
  47         // cost/effort of using [additional] wildcard certificates.
  48         if i := strings.Index(s, "--"); i >= 0 {
  49                 s = s[:i]
  50         }
  51         if arvadosclient.UUIDMatch(s) {
  52                 return s
  53         }
  54         if pdh := strings.Replace(s, "-", "+", 1); arvadosclient.PDHMatch(pdh) {
  55                 return pdh
  56         }
  57         return ""
  58 }
  59
  60 var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
  61
  62 // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
  63 // PDH (even if it is a PDH with "+" replaced by " " or "-");
  64 // otherwise "".
  65 func parseCollectionIDFromURL(s string) string {
  66         if arvadosclient.UUIDMatch(s) {
  67                 return s
  68         }
  69         if pdh := urlPDHDecoder.Replace(s); arvadosclient.PDHMatch(pdh) {
  70                 return pdh
  71         }
  72         return ""
  73 }
  74
  75 func (h *handler) setup() {
  76         h.clientPool = arvadosclient.MakeClientPool()
  77
  78         keepclient.RefreshServiceDiscoveryOnSIGHUP()
  79
  80         h.healthHandler = &health.Handler{
  81                 Token:  h.Config.ManagementToken,
  82                 Prefix: "/_health/",
  83         }
  84
  85         // Even though we don't accept LOCK requests, every webdav
  86         // handler must have a non-nil LockSystem.
  87         h.webdavLS = &noLockSystem{}
  88 }
  89
  90 func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
  91         status := struct {
  92                 cacheStats
  93         }{
  94                 cacheStats: h.Config.Cache.Stats(),
  95         }
  96         json.NewEncoder(w).Encode(status)
  97 }
  98
  99 var (
 100         webdavMethod = map[string]bool{
 101                 "OPTIONS":  true,
 102                 "PROPFIND": true,
 103         }
 104         browserMethod = map[string]bool{
 105                 "GET":  true,
 106                 "HEAD": true,
 107                 "POST": true,
 108         }
 109 )
 110
 111 // ServeHTTP implements http.Handler.
 112 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 113         h.setupOnce.Do(h.setup)
 114
 115         var statusCode = 0
 116         var statusText string
 117
 118         remoteAddr := r.RemoteAddr
 119         if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
 120                 remoteAddr = xff + "," + remoteAddr
 121         }
 122
 123         w := httpserver.WrapResponseWriter(wOrig)
 124         defer func() {
 125                 if statusCode == 0 {
 126                         statusCode = w.WroteStatus()
 127                 } else if w.WroteStatus() == 0 {
 128                         w.WriteHeader(statusCode)
 129                 } else if w.WroteStatus() != statusCode {
 130                         httpserver.Log(r.RemoteAddr, "WARNING",
 131                                 fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
 132                 }
 133                 if statusText == "" {
 134                         statusText = http.StatusText(statusCode)
 135                 }
 136                 httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
 137         }()
 138
 139         if strings.HasPrefix(r.URL.Path, "/_health/") && r.Method == "GET" {
 140                 h.healthHandler.ServeHTTP(w, r)
 141                 return
 142         }
 143
 144         if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" {
 145                 if !browserMethod[method] && !webdavMethod[method] {
 146                         statusCode = http.StatusMethodNotAllowed
 147                         return
 148                 }
 149                 w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Range")
 150                 w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, PROPFIND")
 151                 w.Header().Set("Access-Control-Allow-Origin", "*")
 152                 w.Header().Set("Access-Control-Max-Age", "86400")
 153                 statusCode = http.StatusOK
 154                 return
 155         }
 156
 157         if !browserMethod[r.Method] && !webdavMethod[r.Method] {
 158                 statusCode, statusText = http.StatusMethodNotAllowed, r.Method
 159                 return
 160         }
 161
 162         if r.Header.Get("Origin") != "" {
 163                 // Allow simple cross-origin requests without user
 164                 // credentials ("user credentials" as defined by CORS,
 165                 // i.e., cookies, HTTP authentication, and client-side
 166                 // SSL certificates. See
 167                 // http://www.w3.org/TR/cors/#user-credentials).
 168                 w.Header().Set("Access-Control-Allow-Origin", "*")
 169                 w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
 170         }
 171
 172         arv := h.clientPool.Get()
 173         if arv == nil {
 174                 statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
 175                 return
 176         }
 177         defer h.clientPool.Put(arv)
 178
 179         pathParts := strings.Split(r.URL.Path[1:], "/")
 180
 181         var stripParts int
 182         var targetID string
 183         var tokens []string
 184         var reqTokens []string
 185         var pathToken bool
 186         var attachment bool
 187         credentialsOK := h.Config.TrustAllContent
 188
 189         if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
 190                 credentialsOK = true
 191                 attachment = true
 192         } else if r.FormValue("disposition") == "attachment" {
 193                 attachment = true
 194         }
 195
 196         if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
 197                 // http://ID.collections.example/PATH...
 198                 credentialsOK = true
 199         } else if r.URL.Path == "/status.json" {
 200                 h.serveStatus(w, r)
 201                 return
 202         } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
 203                 // /c=ID[/PATH...]
 204                 targetID = parseCollectionIDFromURL(pathParts[0][2:])
 205                 stripParts = 1
 206         } else if len(pathParts) >= 2 && pathParts[0] == "collections" {
 207                 if len(pathParts) >= 4 && pathParts[1] == "download" {
 208                         // /collections/download/ID/TOKEN/PATH...
 209                         targetID = parseCollectionIDFromURL(pathParts[2])
 210                         tokens = []string{pathParts[3]}
 211                         stripParts = 4
 212                         pathToken = true
 213                 } else {
 214                         // /collections/ID/PATH...
 215                         targetID = parseCollectionIDFromURL(pathParts[1])
 216                         tokens = h.Config.AnonymousTokens
 217                         stripParts = 2
 218                 }
 219         }
 220
 221         if targetID == "" {
 222                 statusCode = http.StatusNotFound
 223                 return
 224         }
 225
 226         formToken := r.FormValue("api_token")
 227         if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
 228                 // The client provided an explicit token in the POST
 229                 // body. The Origin header indicates this *might* be
 230                 // an AJAX request, in which case redirect-with-cookie
 231                 // won't work: we should just serve the content in the
 232                 // POST response. This is safe because:
 233                 //
 234                 // * We're supplying an attachment, not inline
 235                 //   content, so we don't need to convert the POST to
 236                 //   a GET and avoid the "really resubmit form?"
 237                 //   problem.
 238                 //
 239                 // * The token isn't embedded in the URL, so we don't
 240                 //   need to worry about bookmarks and copy/paste.
 241                 tokens = append(tokens, formToken)
 242         } else if formToken != "" && browserMethod[r.Method] {
 243                 // The client provided an explicit token in the query
 244                 // string, or a form in POST body. We must put the
 245                 // token in an HttpOnly cookie, and redirect to the
 246                 // same URL with the query param redacted and method =
 247                 // GET.
 248                 h.seeOtherWithCookie(w, r, "", credentialsOK)
 249                 return
 250         }
 251
 252         targetPath := pathParts[stripParts:]
 253         if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
 254                 // http://ID.example/t=TOKEN/PATH...
 255                 // /c=ID/t=TOKEN/PATH...
 256                 //
 257                 // This form must only be used to pass scoped tokens
 258                 // that give permission for a single collection. See
 259                 // FormValue case above.
 260                 tokens = []string{targetPath[0][2:]}
 261                 pathToken = true
 262                 targetPath = targetPath[1:]
 263                 stripParts++
 264         }
 265
 266         if tokens == nil {
 267                 if credentialsOK {
 268                         reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
 269                 }
 270                 tokens = append(reqTokens, h.Config.AnonymousTokens...)
 271         }
 272
 273         if len(targetPath) > 0 && targetPath[0] == "_" {
 274                 // If a collection has a directory called "t=foo" or
 275                 // "_", it can be served at
 276                 // //collections.example/_/t=foo/ or
 277                 // //collections.example/_/_/ respectively:
 278                 // //collections.example/t=foo/ won't work because
 279                 // t=foo will be interpreted as a token "foo".
 280                 targetPath = targetPath[1:]
 281                 stripParts++
 282         }
 283
 284         forceReload := false
 285         if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
 286                 forceReload = true
 287         }
 288
 289         var collection *arvados.Collection
 290         tokenResult := make(map[string]int)
 291         for _, arv.ApiToken = range tokens {
 292                 var err error
 293                 collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
 294                 if err == nil {
 295                         // Success
 296                         break
 297                 }
 298                 if srvErr, ok := err.(arvadosclient.APIServerError); ok {
 299                         switch srvErr.HttpStatusCode {
 300                         case 404, 401:
 301                                 // Token broken or insufficient to
 302                                 // retrieve collection
 303                                 tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
 304                                 continue
 305                         }
 306                 }
 307                 // Something more serious is wrong
 308                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 309                 return
 310         }
 311         if collection == nil {
 312                 if pathToken || !credentialsOK {
 313                         // Either the URL is a "secret sharing link"
 314                         // that didn't work out (and asking the client
 315                         // for additional credentials would just be
 316                         // confusing), or we don't even accept
 317                         // credentials at this path.
 318                         statusCode = http.StatusNotFound
 319                         return
 320                 }
 321                 for _, t := range reqTokens {
 322                         if tokenResult[t] == 404 {
 323                                 // The client provided valid token(s), but the
 324                                 // collection was not found.
 325                                 statusCode = http.StatusNotFound
 326                                 return
 327                         }
 328                 }
 329                 // The client's token was invalid (e.g., expired), or
 330                 // the client didn't even provide one.  Propagate the
 331                 // 401 to encourage the client to use a [different]
 332                 // token.
 333                 //
 334                 // TODO(TC): This response would be confusing to
 335                 // someone trying (anonymously) to download public
 336                 // data that has been deleted.  Allow a referrer to
 337                 // provide this context somehow?
 338                 w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 339                 statusCode = http.StatusUnauthorized
 340                 return
 341         }
 342
 343         kc, err := keepclient.MakeKeepClient(arv)
 344         if err != nil {
 345                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 346                 return
 347         }
 348
 349         var basename string
 350         if len(targetPath) > 0 {
 351                 basename = targetPath[len(targetPath)-1]
 352         }
 353         applyContentDispositionHdr(w, r, basename, attachment)
 354
 355         fs := collection.FileSystem(&arvados.Client{
 356                 APIHost:   arv.ApiServer,
 357                 AuthToken: arv.ApiToken,
 358                 Insecure:  arv.ApiInsecure,
 359         }, kc)
 360         if webdavMethod[r.Method] {
 361                 h := webdav.Handler{
 362                         Prefix:     "/" + strings.Join(pathParts[:stripParts], "/"),
 363                         FileSystem: &webdavFS{collfs: fs},
 364                         LockSystem: h.webdavLS,
 365                         Logger: func(_ *http.Request, err error) {
 366                                 if os.IsNotExist(err) {
 367                                         statusCode, statusText = http.StatusNotFound, err.Error()
 368                                 } else if err != nil {
 369                                         statusCode, statusText = http.StatusInternalServerError, err.Error()
 370                                 }
 371                         },
 372                 }
 373                 h.ServeHTTP(w, r)
 374                 return
 375         }
 376
 377         openPath := "/" + strings.Join(targetPath, "/")
 378         if f, err := fs.Open(openPath); os.IsNotExist(err) {
 379                 // Requested non-existent path
 380                 statusCode = http.StatusNotFound
 381         } else if err != nil {
 382                 // Some other (unexpected) error
 383                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 384         } else if stat, err := f.Stat(); err != nil {
 385                 // Can't get Size/IsDir (shouldn't happen with a collectionFS!)
 386                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 387         } else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
 388                 // If client requests ".../dirname", redirect to
 389                 // ".../dirname/". This way, relative links in the
 390                 // listing for "dirname" can always be "fnm", never
 391                 // "dirname/fnm".
 392                 h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
 393         } else if stat.IsDir() {
 394                 h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
 395         } else {
 396                 http.ServeContent(w, r, basename, stat.ModTime(), f)
 397                 if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
 398                         // If we wrote fewer bytes than expected, it's
 399                         // too late to change the real response code
 400                         // or send an error message to the client, but
 401                         // at least we can try to put some useful
 402                         // debugging info in the logs.
 403                         n, err := f.Read(make([]byte, 1024))
 404                         statusCode, statusText = http.StatusInternalServerError, fmt.Sprintf("f.Size()==%d but only wrote %d bytes; read(1024) returns %d, %s", stat.Size(), w.WroteBodyBytes(), n, err)
 405
 406                 }
 407         }
 408 }
 409
 410 var dirListingTemplate = `<!DOCTYPE HTML>
 411 <HTML><HEAD>
 412   <META name="robots" content="NOINDEX">
 413   <TITLE>{{ .Collection.Name }}</TITLE>
 414   <STYLE type="text/css">
 415     body {
 416       margin: 1.5em;
 417     }
 418     pre {
 419       background-color: #D9EDF7;
 420       border-radius: .25em;
 421       padding: .75em;
 422       overflow: auto;
 423     }
 424     .footer p {
 425       font-size: 82%;
 426     }
 427     ul {
 428       padding: 0;
 429     }
 430     ul li {
 431       font-family: monospace;
 432       list-style: none;
 433     }
 434   </STYLE>
 435 </HEAD>
 436 <BODY>
 437 <H1>{{ .CollectionName }}</H1>
 438
 439 <P>This collection of data files is being shared with you through
 440 Arvados.  You can download individual files listed below.  To download
 441 the entire collection with wget, try:</P>
 442
 443 <PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
 444
 445 <H2>File Listing</H2>
 446
 447 {{if .Files}}
 448 <UL>
 449 {{range .Files}}  <LI>{{.Size | printf "%15d  " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
 450 </UL>
 451 {{else}}
 452 <P>(No files; this collection is empty.)</P>
 453 {{end}}
 454
 455 <HR noshade>
 456 <DIV class="footer">
 457   <P>
 458     About Arvados:
 459     Arvados is a free and open source software bioinformatics platform.
 460     To learn more, visit arvados.org.
 461     Arvados is not responsible for the files listed on this page.
 462   </P>
 463 </DIV>
 464
 465 </BODY>
 466 `
 467
 468 type fileListEnt struct {
 469         Name string
 470         Size int64
 471 }
 472
 473 func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
 474         var files []fileListEnt
 475         var walk func(string) error
 476         if !strings.HasSuffix(base, "/") {
 477                 base = base + "/"
 478         }
 479         walk = func(path string) error {
 480                 dirname := base + path
 481                 if dirname != "/" {
 482                         dirname = strings.TrimSuffix(dirname, "/")
 483                 }
 484                 d, err := fs.Open(dirname)
 485                 if err != nil {
 486                         return err
 487                 }
 488                 ents, err := d.Readdir(-1)
 489                 if err != nil {
 490                         return err
 491                 }
 492                 for _, ent := range ents {
 493                         if ent.IsDir() {
 494                                 err = walk(path + ent.Name() + "/")
 495                                 if err != nil {
 496                                         return err
 497                                 }
 498                         } else {
 499                                 files = append(files, fileListEnt{
 500                                         Name: path + ent.Name(),
 501                                         Size: ent.Size(),
 502                                 })
 503                         }
 504                 }
 505                 return nil
 506         }
 507         if err := walk(""); err != nil {
 508                 http.Error(w, err.Error(), http.StatusInternalServerError)
 509                 return
 510         }
 511
 512         funcs := template.FuncMap{
 513                 "nbsp": func(s string) template.HTML {
 514                         return template.HTML(strings.Replace(s, " ", "&nbsp;", -1))
 515                 },
 516         }
 517         tmpl, err := template.New("dir").Funcs(funcs).Parse(dirListingTemplate)
 518         if err != nil {
 519                 http.Error(w, err.Error(), http.StatusInternalServerError)
 520                 return
 521         }
 522         sort.Slice(files, func(i, j int) bool {
 523                 return files[i].Name < files[j].Name
 524         })
 525         w.WriteHeader(http.StatusOK)
 526         tmpl.Execute(w, map[string]interface{}{
 527                 "CollectionName": collectionName,
 528                 "Files":          files,
 529                 "Request":        r,
 530                 "StripParts":     stripParts,
 531         })
 532 }
 533
 534 func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
 535         disposition := "inline"
 536         if isAttachment {
 537                 disposition = "attachment"
 538         }
 539         if strings.ContainsRune(r.RequestURI, '?') {
 540                 // Help the UA realize that the filename is just
 541                 // "filename.txt", not
 542                 // "filename.txt?disposition=attachment".
 543                 //
 544                 // TODO(TC): Follow advice at RFC 6266 appendix D
 545                 disposition += "; filename=" + strconv.QuoteToASCII(filename)
 546         }
 547         if disposition != "inline" {
 548                 w.Header().Set("Content-Disposition", disposition)
 549         }
 550 }
 551
 552 func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
 553         if formToken := r.FormValue("api_token"); formToken != "" {
 554                 if !credentialsOK {
 555                         // It is not safe to copy the provided token
 556                         // into a cookie unless the current vhost
 557                         // (origin) serves only a single collection or
 558                         // we are in TrustAllContent mode.
 559                         w.WriteHeader(http.StatusBadRequest)
 560                         return
 561                 }
 562
 563                 // The HttpOnly flag is necessary to prevent
 564                 // JavaScript code (included in, or loaded by, a page
 565                 // in the collection being served) from employing the
 566                 // user's token beyond reading other files in the same
 567                 // domain, i.e., same collection.
 568                 //
 569                 // The 303 redirect is necessary in the case of a GET
 570                 // request to avoid exposing the token in the Location
 571                 // bar, and in the case of a POST request to avoid
 572                 // raising warnings when the user refreshes the
 573                 // resulting page.
 574                 http.SetCookie(w, &http.Cookie{
 575                         Name:     "arvados_api_token",
 576                         Value:    auth.EncodeTokenCookie([]byte(formToken)),
 577                         Path:     "/",
 578                         HttpOnly: true,
 579                 })
 580         }
 581
 582         // Propagate query parameters (except api_token) from
 583         // the original request.
 584         redirQuery := r.URL.Query()
 585         redirQuery.Del("api_token")
 586
 587         u := r.URL
 588         if location != "" {
 589                 newu, err := u.Parse(location)
 590                 if err != nil {
 591                         w.WriteHeader(http.StatusInternalServerError)
 592                         return
 593                 }
 594                 u = newu
 595         }
 596         redir := (&url.URL{
 597                 Host:     r.Host,
 598                 Path:     u.Path,
 599                 RawQuery: redirQuery.Encode(),
 600         }).String()
 601
 602         w.Header().Add("Location", redir)
 603         w.WriteHeader(http.StatusSeeOther)
 604         io.WriteString(w, `<A href="`)
 605         io.WriteString(w, html.EscapeString(redir))
 606         io.WriteString(w, `">Continue</A>`)
 607 }