services/keep-web/handler.go

   1 package main
   2
   3 import (
   4         "encoding/json"
   5         "fmt"
   6         "html"
   7         "html/template"
   8         "io"
   9         "net/http"
  10         "net/url"
  11         "os"
  12         "sort"
  13         "strconv"
  14         "strings"
  15         "sync"
  16
  17         "git.curoverse.com/arvados.git/sdk/go/arvados"
  18         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  19         "git.curoverse.com/arvados.git/sdk/go/auth"
  20         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  21         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  22 )
  23
  24 type handler struct {
  25         Config     *Config
  26         clientPool *arvadosclient.ClientPool
  27         setupOnce  sync.Once
  28 }
  29
  30 // parseCollectionIDFromDNSName returns a UUID or PDH if s begins with
  31 // a UUID or URL-encoded PDH; otherwise "".
  32 func parseCollectionIDFromDNSName(s string) string {
  33         // Strip domain.
  34         if i := strings.IndexRune(s, '.'); i >= 0 {
  35                 s = s[:i]
  36         }
  37         // Names like {uuid}--collections.example.com serve the same
  38         // purpose as {uuid}.collections.example.com but can reduce
  39         // cost/effort of using [additional] wildcard certificates.
  40         if i := strings.Index(s, "--"); i >= 0 {
  41                 s = s[:i]
  42         }
  43         if arvadosclient.UUIDMatch(s) {
  44                 return s
  45         }
  46         if pdh := strings.Replace(s, "-", "+", 1); arvadosclient.PDHMatch(pdh) {
  47                 return pdh
  48         }
  49         return ""
  50 }
  51
  52 var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
  53
  54 // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
  55 // PDH (even if it is a PDH with "+" replaced by " " or "-");
  56 // otherwise "".
  57 func parseCollectionIDFromURL(s string) string {
  58         if arvadosclient.UUIDMatch(s) {
  59                 return s
  60         }
  61         if pdh := urlPDHDecoder.Replace(s); arvadosclient.PDHMatch(pdh) {
  62                 return pdh
  63         }
  64         return ""
  65 }
  66
  67 func (h *handler) setup() {
  68         h.clientPool = arvadosclient.MakeClientPool()
  69         keepclient.RefreshServiceDiscoveryOnSIGHUP()
  70 }
  71
  72 func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
  73         status := struct {
  74                 cacheStats
  75         }{
  76                 cacheStats: h.Config.Cache.Stats(),
  77         }
  78         json.NewEncoder(w).Encode(status)
  79 }
  80
  81 // ServeHTTP implements http.Handler.
  82 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
  83         h.setupOnce.Do(h.setup)
  84
  85         var statusCode = 0
  86         var statusText string
  87
  88         remoteAddr := r.RemoteAddr
  89         if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
  90                 remoteAddr = xff + "," + remoteAddr
  91         }
  92
  93         w := httpserver.WrapResponseWriter(wOrig)
  94         defer func() {
  95                 if statusCode == 0 {
  96                         statusCode = w.WroteStatus()
  97                 } else if w.WroteStatus() == 0 {
  98                         w.WriteHeader(statusCode)
  99                 } else if w.WroteStatus() != statusCode {
 100                         httpserver.Log(r.RemoteAddr, "WARNING",
 101                                 fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
 102                 }
 103                 if statusText == "" {
 104                         statusText = http.StatusText(statusCode)
 105                 }
 106                 httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
 107         }()
 108
 109         if r.Method == "OPTIONS" {
 110                 method := r.Header.Get("Access-Control-Request-Method")
 111                 if method != "GET" && method != "POST" {
 112                         statusCode = http.StatusMethodNotAllowed
 113                         return
 114                 }
 115                 w.Header().Set("Access-Control-Allow-Headers", "Range")
 116                 w.Header().Set("Access-Control-Allow-Methods", "GET, POST")
 117                 w.Header().Set("Access-Control-Allow-Origin", "*")
 118                 w.Header().Set("Access-Control-Max-Age", "86400")
 119                 statusCode = http.StatusOK
 120                 return
 121         }
 122
 123         if r.Method != "GET" && r.Method != "POST" {
 124                 statusCode, statusText = http.StatusMethodNotAllowed, r.Method
 125                 return
 126         }
 127
 128         if r.Header.Get("Origin") != "" {
 129                 // Allow simple cross-origin requests without user
 130                 // credentials ("user credentials" as defined by CORS,
 131                 // i.e., cookies, HTTP authentication, and client-side
 132                 // SSL certificates. See
 133                 // http://www.w3.org/TR/cors/#user-credentials).
 134                 w.Header().Set("Access-Control-Allow-Origin", "*")
 135         }
 136
 137         arv := h.clientPool.Get()
 138         if arv == nil {
 139                 statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
 140                 return
 141         }
 142         defer h.clientPool.Put(arv)
 143
 144         pathParts := strings.Split(r.URL.Path[1:], "/")
 145
 146         var stripParts int
 147         var targetID string
 148         var tokens []string
 149         var reqTokens []string
 150         var pathToken bool
 151         var attachment bool
 152         credentialsOK := h.Config.TrustAllContent
 153
 154         if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
 155                 credentialsOK = true
 156                 attachment = true
 157         } else if r.FormValue("disposition") == "attachment" {
 158                 attachment = true
 159         }
 160
 161         if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
 162                 // http://ID.collections.example/PATH...
 163                 credentialsOK = true
 164         } else if r.URL.Path == "/status.json" {
 165                 h.serveStatus(w, r)
 166                 return
 167         } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
 168                 // /c=ID[/PATH...]
 169                 targetID = parseCollectionIDFromURL(pathParts[0][2:])
 170                 stripParts = 1
 171         } else if len(pathParts) >= 2 && pathParts[0] == "collections" {
 172                 if len(pathParts) >= 4 && pathParts[1] == "download" {
 173                         // /collections/download/ID/TOKEN/PATH...
 174                         targetID = parseCollectionIDFromURL(pathParts[2])
 175                         tokens = []string{pathParts[3]}
 176                         stripParts = 4
 177                         pathToken = true
 178                 } else {
 179                         // /collections/ID/PATH...
 180                         targetID = parseCollectionIDFromURL(pathParts[1])
 181                         tokens = h.Config.AnonymousTokens
 182                         stripParts = 2
 183                 }
 184         }
 185
 186         if targetID == "" {
 187                 statusCode = http.StatusNotFound
 188                 return
 189         }
 190
 191         formToken := r.FormValue("api_token")
 192         if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
 193                 // The client provided an explicit token in the POST
 194                 // body. The Origin header indicates this *might* be
 195                 // an AJAX request, in which case redirect-with-cookie
 196                 // won't work: we should just serve the content in the
 197                 // POST response. This is safe because:
 198                 //
 199                 // * We're supplying an attachment, not inline
 200                 //   content, so we don't need to convert the POST to
 201                 //   a GET and avoid the "really resubmit form?"
 202                 //   problem.
 203                 //
 204                 // * The token isn't embedded in the URL, so we don't
 205                 //   need to worry about bookmarks and copy/paste.
 206                 tokens = append(tokens, formToken)
 207         } else if formToken != "" {
 208                 // The client provided an explicit token in the query
 209                 // string, or a form in POST body. We must put the
 210                 // token in an HttpOnly cookie, and redirect to the
 211                 // same URL with the query param redacted and method =
 212                 // GET.
 213                 h.seeOtherWithCookie(w, r, "", credentialsOK)
 214                 return
 215         }
 216
 217         targetPath := pathParts[stripParts:]
 218         if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
 219                 // http://ID.example/t=TOKEN/PATH...
 220                 // /c=ID/t=TOKEN/PATH...
 221                 //
 222                 // This form must only be used to pass scoped tokens
 223                 // that give permission for a single collection. See
 224                 // FormValue case above.
 225                 tokens = []string{targetPath[0][2:]}
 226                 pathToken = true
 227                 targetPath = targetPath[1:]
 228                 stripParts++
 229         }
 230
 231         if tokens == nil {
 232                 if credentialsOK {
 233                         reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
 234                 }
 235                 tokens = append(reqTokens, h.Config.AnonymousTokens...)
 236         }
 237
 238         if len(targetPath) > 0 && targetPath[0] == "_" {
 239                 // If a collection has a directory called "t=foo" or
 240                 // "_", it can be served at
 241                 // //collections.example/_/t=foo/ or
 242                 // //collections.example/_/_/ respectively:
 243                 // //collections.example/t=foo/ won't work because
 244                 // t=foo will be interpreted as a token "foo".
 245                 targetPath = targetPath[1:]
 246                 stripParts++
 247         }
 248
 249         forceReload := false
 250         if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
 251                 forceReload = true
 252         }
 253
 254         var collection *arvados.Collection
 255         tokenResult := make(map[string]int)
 256         for _, arv.ApiToken = range tokens {
 257                 var err error
 258                 collection, err = h.Config.Cache.Get(arv, targetID, forceReload)
 259                 if err == nil {
 260                         // Success
 261                         break
 262                 }
 263                 if srvErr, ok := err.(arvadosclient.APIServerError); ok {
 264                         switch srvErr.HttpStatusCode {
 265                         case 404, 401:
 266                                 // Token broken or insufficient to
 267                                 // retrieve collection
 268                                 tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
 269                                 continue
 270                         }
 271                 }
 272                 // Something more serious is wrong
 273                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 274                 return
 275         }
 276         if collection == nil {
 277                 if pathToken || !credentialsOK {
 278                         // Either the URL is a "secret sharing link"
 279                         // that didn't work out (and asking the client
 280                         // for additional credentials would just be
 281                         // confusing), or we don't even accept
 282                         // credentials at this path.
 283                         statusCode = http.StatusNotFound
 284                         return
 285                 }
 286                 for _, t := range reqTokens {
 287                         if tokenResult[t] == 404 {
 288                                 // The client provided valid token(s), but the
 289                                 // collection was not found.
 290                                 statusCode = http.StatusNotFound
 291                                 return
 292                         }
 293                 }
 294                 // The client's token was invalid (e.g., expired), or
 295                 // the client didn't even provide one.  Propagate the
 296                 // 401 to encourage the client to use a [different]
 297                 // token.
 298                 //
 299                 // TODO(TC): This response would be confusing to
 300                 // someone trying (anonymously) to download public
 301                 // data that has been deleted.  Allow a referrer to
 302                 // provide this context somehow?
 303                 w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 304                 statusCode = http.StatusUnauthorized
 305                 return
 306         }
 307
 308         kc, err := keepclient.MakeKeepClient(arv)
 309         if err != nil {
 310                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 311                 return
 312         }
 313
 314         basename := targetPath[len(targetPath)-1]
 315         applyContentDispositionHdr(w, r, basename, attachment)
 316
 317         fs := collection.FileSystem(&arvados.Client{
 318                 APIHost:   arv.ApiServer,
 319                 AuthToken: arv.ApiToken,
 320                 Insecure:  arv.ApiInsecure,
 321         }, kc)
 322         openPath := "/" + strings.Join(targetPath, "/")
 323         if f, err := fs.Open(openPath); os.IsNotExist(err) {
 324                 // Requested non-existent path
 325                 statusCode = http.StatusNotFound
 326         } else if err != nil {
 327                 // Some other (unexpected) error
 328                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 329         } else if stat, err := f.Stat(); err != nil {
 330                 // Can't get Size/IsDir (shouldn't happen with a collectionFS!)
 331                 statusCode, statusText = http.StatusInternalServerError, err.Error()
 332         } else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
 333                 // If client requests ".../dirname", redirect to
 334                 // ".../dirname/". This way, relative links in the
 335                 // listing for "dirname" can always be "fnm", never
 336                 // "dirname/fnm".
 337                 h.seeOtherWithCookie(w, r, basename+"/", credentialsOK)
 338         } else if stat.IsDir() {
 339                 h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
 340         } else {
 341                 http.ServeContent(w, r, basename, stat.ModTime(), f)
 342                 if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
 343                         // If we wrote fewer bytes than expected, it's
 344                         // too late to change the real response code
 345                         // or send an error message to the client, but
 346                         // at least we can try to put some useful
 347                         // debugging info in the logs.
 348                         n, err := f.Read(make([]byte, 1024))
 349                         statusCode, statusText = http.StatusInternalServerError, fmt.Sprintf("f.Size()==%d but only wrote %d bytes; read(1024) returns %d, %s", stat.Size(), w.WroteBodyBytes(), n, err)
 350
 351                 }
 352         }
 353 }
 354
 355 var dirListingTemplate = `<!DOCTYPE HTML>
 356 <HTML><HEAD>
 357   <META name="robots" content="NOINDEX">
 358   <TITLE>{{ .Collection.Name }}</TITLE>
 359   <STYLE type="text/css">
 360     body {
 361       margin: 1.5em;
 362     }
 363     pre {
 364       background-color: #D9EDF7;
 365       border-radius: .25em;
 366       padding: .75em;
 367       overflow: auto;
 368     }
 369     .footer p {
 370       font-size: 82%;
 371     }
 372     ul {
 373       padding: 0;
 374     }
 375     ul li {
 376       font-family: monospace;
 377       list-style: none;
 378     }
 379   </STYLE>
 380 </HEAD>
 381 <BODY>
 382 <H1>{{ .CollectionName }}</H1>
 383
 384 <P>This collection of data files is being shared with you through
 385 Arvados.  You can download individual files listed below.  To download
 386 the entire collection with wget, try:</P>
 387
 388 <PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
 389
 390 <H2>File Listing</H2>
 391
 392 {{if .Files}}
 393 <UL>
 394 {{range .Files}}  <LI>{{.Size | printf "%15d  " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
 395 </UL>
 396 {{else}}
 397 <P>(No files; this collection is empty.)</P>
 398 {{end}}
 399
 400 <HR noshade>
 401 <DIV class="footer">
 402   <P>
 403     About Arvados:
 404     Arvados is a free and open source software bioinformatics platform.
 405     To learn more, visit arvados.org.
 406     Arvados is not responsible for the files listed on this page.
 407   </P>
 408 </DIV>
 409
 410 </BODY>
 411 `
 412
 413 type fileListEnt struct {
 414         Name string
 415         Size int64
 416 }
 417
 418 func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
 419         var files []fileListEnt
 420         var walk func(string) error
 421         if !strings.HasSuffix(base, "/") {
 422                 base = base + "/"
 423         }
 424         walk = func(path string) error {
 425                 dirname := base + path
 426                 if dirname != "/" {
 427                         dirname = strings.TrimSuffix(dirname, "/")
 428                 }
 429                 d, err := fs.Open(dirname)
 430                 if err != nil {
 431                         return err
 432                 }
 433                 ents, err := d.Readdir(-1)
 434                 if err != nil {
 435                         return err
 436                 }
 437                 for _, ent := range ents {
 438                         if ent.IsDir() {
 439                                 err = walk(path + ent.Name() + "/")
 440                                 if err != nil {
 441                                         return err
 442                                 }
 443                         } else {
 444                                 files = append(files, fileListEnt{
 445                                         Name: path + ent.Name(),
 446                                         Size: ent.Size(),
 447                                 })
 448                         }
 449                 }
 450                 return nil
 451         }
 452         if err := walk(""); err != nil {
 453                 http.Error(w, err.Error(), http.StatusInternalServerError)
 454                 return
 455         }
 456
 457         funcs := template.FuncMap{
 458                 "nbsp": func(s string) template.HTML {
 459                         return template.HTML(strings.Replace(s, " ", "&nbsp;", -1))
 460                 },
 461         }
 462         tmpl, err := template.New("dir").Funcs(funcs).Parse(dirListingTemplate)
 463         if err != nil {
 464                 http.Error(w, err.Error(), http.StatusInternalServerError)
 465                 return
 466         }
 467         sort.Slice(files, func(i, j int) bool {
 468                 return files[i].Name < files[j].Name
 469         })
 470         w.WriteHeader(http.StatusOK)
 471         tmpl.Execute(w, map[string]interface{}{
 472                 "CollectionName": collectionName,
 473                 "Files":          files,
 474                 "Request":        r,
 475                 "StripParts":     stripParts,
 476         })
 477 }
 478
 479 func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
 480         disposition := "inline"
 481         if isAttachment {
 482                 disposition = "attachment"
 483         }
 484         if strings.ContainsRune(r.RequestURI, '?') {
 485                 // Help the UA realize that the filename is just
 486                 // "filename.txt", not
 487                 // "filename.txt?disposition=attachment".
 488                 //
 489                 // TODO(TC): Follow advice at RFC 6266 appendix D
 490                 disposition += "; filename=" + strconv.QuoteToASCII(filename)
 491         }
 492         if disposition != "inline" {
 493                 w.Header().Set("Content-Disposition", disposition)
 494         }
 495 }
 496
 497 func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
 498         if !credentialsOK {
 499                 // It is not safe to copy the provided token
 500                 // into a cookie unless the current vhost
 501                 // (origin) serves only a single collection or
 502                 // we are in TrustAllContent mode.
 503                 w.WriteHeader(http.StatusBadRequest)
 504                 return
 505         }
 506
 507         if formToken := r.FormValue("api_token"); formToken != "" {
 508                 // The HttpOnly flag is necessary to prevent
 509                 // JavaScript code (included in, or loaded by, a page
 510                 // in the collection being served) from employing the
 511                 // user's token beyond reading other files in the same
 512                 // domain, i.e., same collection.
 513                 //
 514                 // The 303 redirect is necessary in the case of a GET
 515                 // request to avoid exposing the token in the Location
 516                 // bar, and in the case of a POST request to avoid
 517                 // raising warnings when the user refreshes the
 518                 // resulting page.
 519
 520                 http.SetCookie(w, &http.Cookie{
 521                         Name:     "arvados_api_token",
 522                         Value:    auth.EncodeTokenCookie([]byte(formToken)),
 523                         Path:     "/",
 524                         HttpOnly: true,
 525                 })
 526         }
 527
 528         // Propagate query parameters (except api_token) from
 529         // the original request.
 530         redirQuery := r.URL.Query()
 531         redirQuery.Del("api_token")
 532
 533         u := r.URL
 534         if location != "" {
 535                 newu, err := u.Parse(location)
 536                 if err != nil {
 537                         w.WriteHeader(http.StatusInternalServerError)
 538                         return
 539                 }
 540                 u = newu
 541         }
 542         redir := (&url.URL{
 543                 Host:     r.Host,
 544                 Path:     u.Path,
 545                 RawQuery: redirQuery.Encode(),
 546         }).String()
 547
 548         w.Header().Add("Location", redir)
 549         w.WriteHeader(http.StatusSeeOther)
 550         io.WriteString(w, `<A href="`)
 551         io.WriteString(w, html.EscapeString(redir))
 552         io.WriteString(w, `">Continue</A>`)
 553 }