services/keep-web/handler.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "html"
  12         "html/template"
  13         "io"
  14         "net/http"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "sort"
  19         "strconv"
  20         "strings"
  21         "sync"
  22
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/auth"
  26         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  27         "git.arvados.org/arvados.git/sdk/go/httpserver"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/sirupsen/logrus"
  30         "golang.org/x/net/webdav"
  31 )
  32
  33 type handler struct {
  34         Cache      cache
  35         Cluster    *arvados.Cluster
  36         clientPool *arvadosclient.ClientPool
  37         setupOnce  sync.Once
  38         webdavLS   webdav.LockSystem
  39 }
  40
  41 var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
  42
  43 var notFoundMessage = "404 Not found\r\n\r\nThe requested path was not found, or you do not have permission to access it.\r"
  44 var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r"
  45
  46 // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
  47 // PDH (even if it is a PDH with "+" replaced by " " or "-");
  48 // otherwise "".
  49 func parseCollectionIDFromURL(s string) string {
  50         if arvadosclient.UUIDMatch(s) {
  51                 return s
  52         }
  53         if pdh := urlPDHDecoder.Replace(s); arvadosclient.PDHMatch(pdh) {
  54                 return pdh
  55         }
  56         return ""
  57 }
  58
  59 func (h *handler) setup() {
  60         // Errors will be handled at the client pool.
  61         arv, _ := arvados.NewClientFromConfig(h.Cluster)
  62         h.clientPool = arvadosclient.MakeClientPoolWith(arv)
  63
  64         keepclient.DefaultBlockCache.MaxBlocks = h.Cluster.Collections.WebDAVCache.MaxBlockEntries
  65
  66         // Even though we don't accept LOCK requests, every webdav
  67         // handler must have a non-nil LockSystem.
  68         h.webdavLS = &noLockSystem{}
  69 }
  70
  71 func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
  72         json.NewEncoder(w).Encode(struct{ Version string }{version})
  73 }
  74
  75 // updateOnSuccess wraps httpserver.ResponseWriter. If the handler
  76 // sends an HTTP header indicating success, updateOnSuccess first
  77 // calls the provided update func. If the update func fails, a 500
  78 // response is sent, and the status code and body sent by the handler
  79 // are ignored (all response writes return the update error).
  80 type updateOnSuccess struct {
  81         httpserver.ResponseWriter
  82         logger     logrus.FieldLogger
  83         update     func() error
  84         sentHeader bool
  85         err        error
  86 }
  87
  88 func (uos *updateOnSuccess) Write(p []byte) (int, error) {
  89         if !uos.sentHeader {
  90                 uos.WriteHeader(http.StatusOK)
  91         }
  92         if uos.err != nil {
  93                 return 0, uos.err
  94         }
  95         return uos.ResponseWriter.Write(p)
  96 }
  97
  98 func (uos *updateOnSuccess) WriteHeader(code int) {
  99         if !uos.sentHeader {
 100                 uos.sentHeader = true
 101                 if code >= 200 && code < 400 {
 102                         if uos.err = uos.update(); uos.err != nil {
 103                                 code := http.StatusInternalServerError
 104                                 if err, ok := uos.err.(*arvados.TransactionError); ok {
 105                                         code = err.StatusCode
 106                                 }
 107                                 uos.logger.WithError(uos.err).Errorf("update() returned error type %T, changing response to HTTP %d", uos.err, code)
 108                                 http.Error(uos.ResponseWriter, uos.err.Error(), code)
 109                                 return
 110                         }
 111                 }
 112         }
 113         uos.ResponseWriter.WriteHeader(code)
 114 }
 115
 116 var (
 117         corsAllowHeadersHeader = strings.Join([]string{
 118                 "Authorization", "Content-Type", "Range",
 119                 // WebDAV request headers:
 120                 "Depth", "Destination", "If", "Lock-Token", "Overwrite", "Timeout",
 121         }, ", ")
 122         writeMethod = map[string]bool{
 123                 "COPY":      true,
 124                 "DELETE":    true,
 125                 "LOCK":      true,
 126                 "MKCOL":     true,
 127                 "MOVE":      true,
 128                 "PROPPATCH": true,
 129                 "PUT":       true,
 130                 "RMCOL":     true,
 131                 "UNLOCK":    true,
 132         }
 133         webdavMethod = map[string]bool{
 134                 "COPY":      true,
 135                 "DELETE":    true,
 136                 "LOCK":      true,
 137                 "MKCOL":     true,
 138                 "MOVE":      true,
 139                 "OPTIONS":   true,
 140                 "PROPFIND":  true,
 141                 "PROPPATCH": true,
 142                 "PUT":       true,
 143                 "RMCOL":     true,
 144                 "UNLOCK":    true,
 145         }
 146         browserMethod = map[string]bool{
 147                 "GET":  true,
 148                 "HEAD": true,
 149                 "POST": true,
 150         }
 151         // top-level dirs to serve with siteFS
 152         siteFSDir = map[string]bool{
 153                 "":      true, // root directory
 154                 "by_id": true,
 155                 "users": true,
 156         }
 157 )
 158
 159 func stripDefaultPort(host string) string {
 160         // Will consider port 80 and port 443 to be the same vhost.  I think that's fine.
 161         u := &url.URL{Host: host}
 162         if p := u.Port(); p == "80" || p == "443" {
 163                 return strings.ToLower(u.Hostname())
 164         } else {
 165                 return strings.ToLower(host)
 166         }
 167 }
 168
 169 // CheckHealth implements service.Handler.
 170 func (h *handler) CheckHealth() error {
 171         return nil
 172 }
 173
 174 // Done implements service.Handler.
 175 func (h *handler) Done() <-chan struct{} {
 176         return nil
 177 }
 178
 179 // ServeHTTP implements http.Handler.
 180 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 181         h.setupOnce.Do(h.setup)
 182
 183         if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" {
 184                 r.URL.Scheme = xfp
 185         }
 186
 187         w := httpserver.WrapResponseWriter(wOrig)
 188
 189         if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" {
 190                 if !browserMethod[method] && !webdavMethod[method] {
 191                         w.WriteHeader(http.StatusMethodNotAllowed)
 192                         return
 193                 }
 194                 w.Header().Set("Access-Control-Allow-Headers", corsAllowHeadersHeader)
 195                 w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
 196                 w.Header().Set("Access-Control-Allow-Origin", "*")
 197                 w.Header().Set("Access-Control-Max-Age", "86400")
 198                 return
 199         }
 200
 201         if !browserMethod[r.Method] && !webdavMethod[r.Method] {
 202                 w.WriteHeader(http.StatusMethodNotAllowed)
 203                 return
 204         }
 205
 206         if r.Header.Get("Origin") != "" {
 207                 // Allow simple cross-origin requests without user
 208                 // credentials ("user credentials" as defined by CORS,
 209                 // i.e., cookies, HTTP authentication, and client-side
 210                 // SSL certificates. See
 211                 // http://www.w3.org/TR/cors/#user-credentials).
 212                 w.Header().Set("Access-Control-Allow-Origin", "*")
 213                 w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
 214         }
 215
 216         if h.serveS3(w, r) {
 217                 return
 218         }
 219
 220         pathParts := strings.Split(r.URL.Path[1:], "/")
 221
 222         var stripParts int
 223         var collectionID string
 224         var tokens []string
 225         var reqTokens []string
 226         var pathToken bool
 227         var attachment bool
 228         var useSiteFS bool
 229         credentialsOK := h.Cluster.Collections.TrustAllContent
 230         reasonNotAcceptingCredentials := ""
 231
 232         if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Cluster.Services.WebDAVDownload.ExternalURL.Host) {
 233                 credentialsOK = true
 234                 attachment = true
 235         } else if r.FormValue("disposition") == "attachment" {
 236                 attachment = true
 237         }
 238
 239         if !credentialsOK {
 240                 reasonNotAcceptingCredentials = fmt.Sprintf("vhost %q does not specify a single collection ID or match Services.WebDAVDownload.ExternalURL %q, and Collections.TrustAllContent is false",
 241                         r.Host, h.Cluster.Services.WebDAVDownload.ExternalURL)
 242         }
 243
 244         if collectionID = arvados.CollectionIDFromDNSName(r.Host); collectionID != "" {
 245                 // http://ID.collections.example/PATH...
 246                 credentialsOK = true
 247         } else if r.URL.Path == "/status.json" {
 248                 h.serveStatus(w, r)
 249                 return
 250         } else if siteFSDir[pathParts[0]] {
 251                 useSiteFS = true
 252         } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
 253                 // /c=ID[/PATH...]
 254                 collectionID = parseCollectionIDFromURL(pathParts[0][2:])
 255                 stripParts = 1
 256         } else if len(pathParts) >= 2 && pathParts[0] == "collections" {
 257                 if len(pathParts) >= 4 && pathParts[1] == "download" {
 258                         // /collections/download/ID/TOKEN/PATH...
 259                         collectionID = parseCollectionIDFromURL(pathParts[2])
 260                         tokens = []string{pathParts[3]}
 261                         stripParts = 4
 262                         pathToken = true
 263                 } else {
 264                         // /collections/ID/PATH...
 265                         collectionID = parseCollectionIDFromURL(pathParts[1])
 266                         stripParts = 2
 267                         // This path is only meant to work for public
 268                         // data. Tokens provided with the request are
 269                         // ignored.
 270                         credentialsOK = false
 271                         reasonNotAcceptingCredentials = "the '/collections/UUID/PATH' form only works for public data"
 272                 }
 273         }
 274
 275         if collectionID == "" && !useSiteFS {
 276                 http.Error(w, notFoundMessage, http.StatusNotFound)
 277                 return
 278         }
 279
 280         forceReload := false
 281         if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
 282                 forceReload = true
 283         }
 284
 285         if credentialsOK {
 286                 reqTokens = auth.CredentialsFromRequest(r).Tokens
 287         }
 288
 289         formToken := r.FormValue("api_token")
 290         origin := r.Header.Get("Origin")
 291         cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
 292         safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
 293         safeAttachment := attachment && r.URL.Query().Get("api_token") == ""
 294         if formToken == "" {
 295                 // No token to use or redact.
 296         } else if safeAjax || safeAttachment {
 297                 // If this is a cross-origin request, the URL won't
 298                 // appear in the browser's address bar, so
 299                 // substituting a clipboard-safe URL is pointless.
 300                 // Redirect-with-cookie wouldn't work anyway, because
 301                 // it's not safe to allow third-party use of our
 302                 // cookie.
 303                 //
 304                 // If we're supplying an attachment, we don't need to
 305                 // convert POST to GET to avoid the "really resubmit
 306                 // form?" problem, so provided the token isn't
 307                 // embedded in the URL, there's no reason to do
 308                 // redirect-with-cookie in this case either.
 309                 reqTokens = append(reqTokens, formToken)
 310         } else if browserMethod[r.Method] {
 311                 // If this is a page view, and the client provided a
 312                 // token via query string or POST body, we must put
 313                 // the token in an HttpOnly cookie, and redirect to an
 314                 // equivalent URL with the query param redacted and
 315                 // method = GET.
 316                 h.seeOtherWithCookie(w, r, "", credentialsOK)
 317                 return
 318         }
 319
 320         if useSiteFS {
 321                 h.serveSiteFS(w, r, reqTokens, credentialsOK, attachment)
 322                 return
 323         }
 324
 325         targetPath := pathParts[stripParts:]
 326         if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
 327                 // http://ID.example/t=TOKEN/PATH...
 328                 // /c=ID/t=TOKEN/PATH...
 329                 //
 330                 // This form must only be used to pass scoped tokens
 331                 // that give permission for a single collection. See
 332                 // FormValue case above.
 333                 tokens = []string{targetPath[0][2:]}
 334                 pathToken = true
 335                 targetPath = targetPath[1:]
 336                 stripParts++
 337         }
 338
 339         if tokens == nil {
 340                 tokens = reqTokens
 341                 if h.Cluster.Users.AnonymousUserToken != "" {
 342                         tokens = append(tokens, h.Cluster.Users.AnonymousUserToken)
 343                 }
 344         }
 345
 346         if tokens == nil {
 347                 if !credentialsOK {
 348                         http.Error(w, fmt.Sprintf("Authorization tokens are not accepted here: %v, and no anonymous user token is configured.", reasonNotAcceptingCredentials), http.StatusUnauthorized)
 349                 } else {
 350                         http.Error(w, fmt.Sprintf("No authorization token in request, and no anonymous user token is configured."), http.StatusUnauthorized)
 351                 }
 352                 return
 353         }
 354
 355         if len(targetPath) > 0 && targetPath[0] == "_" {
 356                 // If a collection has a directory called "t=foo" or
 357                 // "_", it can be served at
 358                 // //collections.example/_/t=foo/ or
 359                 // //collections.example/_/_/ respectively:
 360                 // //collections.example/t=foo/ won't work because
 361                 // t=foo will be interpreted as a token "foo".
 362                 targetPath = targetPath[1:]
 363                 stripParts++
 364         }
 365
 366         arv := h.clientPool.Get()
 367         if arv == nil {
 368                 http.Error(w, "client pool error: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
 369                 return
 370         }
 371         defer h.clientPool.Put(arv)
 372
 373         var collection *arvados.Collection
 374         var tokenUser *arvados.User
 375         tokenResult := make(map[string]int)
 376         for _, arv.ApiToken = range tokens {
 377                 var err error
 378                 collection, err = h.Cache.Get(arv, collectionID, forceReload)
 379                 if err == nil {
 380                         // Success
 381                         break
 382                 }
 383                 if srvErr, ok := err.(arvadosclient.APIServerError); ok {
 384                         switch srvErr.HttpStatusCode {
 385                         case 404, 401:
 386                                 // Token broken or insufficient to
 387                                 // retrieve collection
 388                                 tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
 389                                 continue
 390                         }
 391                 }
 392                 // Something more serious is wrong
 393                 http.Error(w, "cache error: "+err.Error(), http.StatusInternalServerError)
 394                 return
 395         }
 396         if collection == nil {
 397                 if pathToken || !credentialsOK {
 398                         // Either the URL is a "secret sharing link"
 399                         // that didn't work out (and asking the client
 400                         // for additional credentials would just be
 401                         // confusing), or we don't even accept
 402                         // credentials at this path.
 403                         http.Error(w, notFoundMessage, http.StatusNotFound)
 404                         return
 405                 }
 406                 for _, t := range reqTokens {
 407                         if tokenResult[t] == 404 {
 408                                 // The client provided valid token(s), but the
 409                                 // collection was not found.
 410                                 http.Error(w, notFoundMessage, http.StatusNotFound)
 411                                 return
 412                         }
 413                 }
 414                 // The client's token was invalid (e.g., expired), or
 415                 // the client didn't even provide one.  Redirect to
 416                 // workbench2's login-and-redirect-to-download url if
 417                 // this is a browser navigation request. (The redirect
 418                 // flow can't preserve the original method if it's not
 419                 // GET, and doesn't make sense if the UA is a
 420                 // command-line tool, is trying to load an inline
 421                 // image, etc.; in these cases, there's nothing we can
 422                 // do, so return 401 unauthorized.)
 423                 //
 424                 // Note Sec-Fetch-Mode is sent by all non-EOL
 425                 // browsers, except Safari.
 426                 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Mode
 427                 //
 428                 // TODO(TC): This response would be confusing to
 429                 // someone trying (anonymously) to download public
 430                 // data that has been deleted.  Allow a referrer to
 431                 // provide this context somehow?
 432                 if r.Method == http.MethodGet && r.Header.Get("Sec-Fetch-Mode") == "navigate" {
 433                         target := url.URL(h.Cluster.Services.Workbench2.ExternalURL)
 434                         redirkey := "redirectToPreview"
 435                         if attachment {
 436                                 redirkey = "redirectToDownload"
 437                         }
 438                         callback := "/c=" + collectionID + "/" + strings.Join(targetPath, "/")
 439                         // target.RawQuery = url.Values{redirkey:
 440                         // {target}}.Encode() would be the obvious
 441                         // thing to do here, but wb2 doesn't decode
 442                         // this as a query param -- it takes
 443                         // everything after "${redirkey}=" as the
 444                         // target URL. If we encode "/" as "%2F" etc.,
 445                         // the redirect won't work.
 446                         target.RawQuery = redirkey + "=" + callback
 447                         w.Header().Add("Location", target.String())
 448                         w.WriteHeader(http.StatusSeeOther)
 449                 } else {
 450                         w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 451                         http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
 452                 }
 453                 return
 454         }
 455
 456         kc, err := keepclient.MakeKeepClient(arv)
 457         if err != nil {
 458                 http.Error(w, "error setting up keep client: "+err.Error(), http.StatusInternalServerError)
 459                 return
 460         }
 461         kc.RequestID = r.Header.Get("X-Request-Id")
 462
 463         var basename string
 464         if len(targetPath) > 0 {
 465                 basename = targetPath[len(targetPath)-1]
 466         }
 467         applyContentDispositionHdr(w, r, basename, attachment)
 468
 469         client := (&arvados.Client{
 470                 APIHost:   arv.ApiServer,
 471                 AuthToken: arv.ApiToken,
 472                 Insecure:  arv.ApiInsecure,
 473         }).WithRequestID(r.Header.Get("X-Request-Id"))
 474
 475         fs, err := collection.FileSystem(client, kc)
 476         if err != nil {
 477                 http.Error(w, "error creating collection filesystem: "+err.Error(), http.StatusInternalServerError)
 478                 return
 479         }
 480
 481         writefs, writeOK := fs.(arvados.CollectionFileSystem)
 482         targetIsPDH := arvadosclient.PDHMatch(collectionID)
 483         if (targetIsPDH || !writeOK) && writeMethod[r.Method] {
 484                 http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
 485                 return
 486         }
 487
 488         // Check configured permission
 489         _, sess, err := h.Cache.GetSession(arv.ApiToken)
 490         if err != nil {
 491                 http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
 492         }
 493         tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
 494         if e := (interface{ HTTPStatus() int })(nil); errors.As(err, &e) && e.HTTPStatus() == http.StatusForbidden {
 495                 // Ignore expected error looking up user record when
 496                 // using a scoped token that allows getting
 497                 // collections/X but not users/current
 498         } else if err != nil {
 499                 http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
 500         }
 501
 502         if webdavMethod[r.Method] {
 503                 if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
 504                         http.Error(w, "Not permitted", http.StatusForbidden)
 505                         return
 506                 }
 507                 h.logUploadOrDownload(r, sess.arvadosclient, nil, strings.Join(targetPath, "/"), collection, tokenUser)
 508
 509                 if writeMethod[r.Method] {
 510                         // Save the collection only if/when all
 511                         // webdav->filesystem operations succeed --
 512                         // and send a 500 error if the modified
 513                         // collection can't be saved.
 514                         w = &updateOnSuccess{
 515                                 ResponseWriter: w,
 516                                 logger:         ctxlog.FromContext(r.Context()),
 517                                 update: func() error {
 518                                         return h.Cache.Update(client, *collection, writefs)
 519                                 }}
 520                 }
 521                 h := webdav.Handler{
 522                         Prefix: "/" + strings.Join(pathParts[:stripParts], "/"),
 523                         FileSystem: &webdavFS{
 524                                 collfs:        fs,
 525                                 writing:       writeMethod[r.Method],
 526                                 alwaysReadEOF: r.Method == "PROPFIND",
 527                         },
 528                         LockSystem: h.webdavLS,
 529                         Logger: func(_ *http.Request, err error) {
 530                                 if err != nil {
 531                                         ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler")
 532                                 }
 533                         },
 534                 }
 535                 h.ServeHTTP(w, r)
 536                 return
 537         }
 538
 539         openPath := "/" + strings.Join(targetPath, "/")
 540         f, err := fs.Open(openPath)
 541         if os.IsNotExist(err) {
 542                 // Requested non-existent path
 543                 http.Error(w, notFoundMessage, http.StatusNotFound)
 544                 return
 545         } else if err != nil {
 546                 // Some other (unexpected) error
 547                 http.Error(w, "open: "+err.Error(), http.StatusInternalServerError)
 548                 return
 549         }
 550         defer f.Close()
 551         if stat, err := f.Stat(); err != nil {
 552                 // Can't get Size/IsDir (shouldn't happen with a collectionFS!)
 553                 http.Error(w, "stat: "+err.Error(), http.StatusInternalServerError)
 554         } else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
 555                 // If client requests ".../dirname", redirect to
 556                 // ".../dirname/". This way, relative links in the
 557                 // listing for "dirname" can always be "fnm", never
 558                 // "dirname/fnm".
 559                 h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
 560         } else if stat.IsDir() {
 561                 h.serveDirectory(w, r, collection.Name, fs, openPath, true)
 562         } else {
 563                 if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
 564                         http.Error(w, "Not permitted", http.StatusForbidden)
 565                         return
 566                 }
 567                 h.logUploadOrDownload(r, sess.arvadosclient, nil, strings.Join(targetPath, "/"), collection, tokenUser)
 568
 569                 http.ServeContent(w, r, basename, stat.ModTime(), f)
 570                 if wrote := int64(w.WroteBodyBytes()); wrote != stat.Size() && w.WroteStatus() == http.StatusOK {
 571                         // If we wrote fewer bytes than expected, it's
 572                         // too late to change the real response code
 573                         // or send an error message to the client, but
 574                         // at least we can try to put some useful
 575                         // debugging info in the logs.
 576                         n, err := f.Read(make([]byte, 1024))
 577                         ctxlog.FromContext(r.Context()).Errorf("stat.Size()==%d but only wrote %d bytes; read(1024) returns %d, %v", stat.Size(), wrote, n, err)
 578                 }
 579         }
 580 }
 581
 582 func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosClient, kc *keepclient.KeepClient, client *arvados.Client, release func(), err error) {
 583         arv = h.clientPool.Get()
 584         if arv == nil {
 585                 err = h.clientPool.Err()
 586                 return
 587         }
 588         release = func() { h.clientPool.Put(arv) }
 589         arv.ApiToken = token
 590         kc, err = keepclient.MakeKeepClient(arv)
 591         if err != nil {
 592                 release()
 593                 return
 594         }
 595         kc.RequestID = reqID
 596         client = (&arvados.Client{
 597                 APIHost:   arv.ApiServer,
 598                 AuthToken: arv.ApiToken,
 599                 Insecure:  arv.ApiInsecure,
 600         }).WithRequestID(reqID)
 601         return
 602 }
 603
 604 func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
 605         if len(tokens) == 0 {
 606                 w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 607                 http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
 608                 return
 609         }
 610         if writeMethod[r.Method] {
 611                 http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
 612                 return
 613         }
 614
 615         fs, sess, err := h.Cache.GetSession(tokens[0])
 616         if err != nil {
 617                 http.Error(w, err.Error(), http.StatusInternalServerError)
 618                 return
 619         }
 620         fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution)
 621         f, err := fs.Open(r.URL.Path)
 622         if os.IsNotExist(err) {
 623                 http.Error(w, err.Error(), http.StatusNotFound)
 624                 return
 625         } else if err != nil {
 626                 http.Error(w, err.Error(), http.StatusInternalServerError)
 627                 return
 628         }
 629         defer f.Close()
 630         if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
 631                 if !strings.HasSuffix(r.URL.Path, "/") {
 632                         h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
 633                 } else {
 634                         h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
 635                 }
 636                 return
 637         }
 638
 639         tokenUser, err := h.Cache.GetTokenUser(tokens[0])
 640         if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
 641                 http.Error(w, "Not permitted", http.StatusForbidden)
 642                 return
 643         }
 644         h.logUploadOrDownload(r, sess.arvadosclient, fs, r.URL.Path, nil, tokenUser)
 645
 646         if r.Method == "GET" {
 647                 _, basename := filepath.Split(r.URL.Path)
 648                 applyContentDispositionHdr(w, r, basename, attachment)
 649         }
 650         wh := webdav.Handler{
 651                 Prefix: "/",
 652                 FileSystem: &webdavFS{
 653                         collfs:        fs,
 654                         writing:       writeMethod[r.Method],
 655                         alwaysReadEOF: r.Method == "PROPFIND",
 656                 },
 657                 LockSystem: h.webdavLS,
 658                 Logger: func(_ *http.Request, err error) {
 659                         if err != nil {
 660                                 ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler")
 661                         }
 662                 },
 663         }
 664         wh.ServeHTTP(w, r)
 665 }
 666
 667 var dirListingTemplate = `<!DOCTYPE HTML>
 668 <HTML><HEAD>
 669   <META name="robots" content="NOINDEX">
 670   <TITLE>{{ .CollectionName }}</TITLE>
 671   <STYLE type="text/css">
 672     body {
 673       margin: 1.5em;
 674     }
 675     pre {
 676       background-color: #D9EDF7;
 677       border-radius: .25em;
 678       padding: .75em;
 679       overflow: auto;
 680     }
 681     .footer p {
 682       font-size: 82%;
 683     }
 684     ul {
 685       padding: 0;
 686     }
 687     ul li {
 688       font-family: monospace;
 689       list-style: none;
 690     }
 691   </STYLE>
 692 </HEAD>
 693 <BODY>
 694
 695 <H1>{{ .CollectionName }}</H1>
 696
 697 <P>This collection of data files is being shared with you through
 698 Arvados.  You can download individual files listed below.  To download
 699 the entire directory tree with wget, try:</P>
 700
 701 <PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
 702
 703 <H2>File Listing</H2>
 704
 705 {{if .Files}}
 706 <UL>
 707 {{range .Files}}
 708 {{if .IsDir }}
 709   <LI>{{" " | printf "%15s  " | nbsp}}<A href="{{print "./" .Name}}/">{{.Name}}/</A></LI>
 710 {{else}}
 711   <LI>{{.Size | printf "%15d  " | nbsp}}<A href="{{print "./" .Name}}">{{.Name}}</A></LI>
 712 {{end}}
 713 {{end}}
 714 </UL>
 715 {{else}}
 716 <P>(No files; this collection is empty.)</P>
 717 {{end}}
 718
 719 <HR noshade>
 720 <DIV class="footer">
 721   <P>
 722     About Arvados:
 723     Arvados is a free and open source software bioinformatics platform.
 724     To learn more, visit arvados.org.
 725     Arvados is not responsible for the files listed on this page.
 726   </P>
 727 </DIV>
 728
 729 </BODY>
 730 `
 731
 732 type fileListEnt struct {
 733         Name  string
 734         Size  int64
 735         IsDir bool
 736 }
 737
 738 func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
 739         var files []fileListEnt
 740         var walk func(string) error
 741         if !strings.HasSuffix(base, "/") {
 742                 base = base + "/"
 743         }
 744         walk = func(path string) error {
 745                 dirname := base + path
 746                 if dirname != "/" {
 747                         dirname = strings.TrimSuffix(dirname, "/")
 748                 }
 749                 d, err := fs.Open(dirname)
 750                 if err != nil {
 751                         return err
 752                 }
 753                 ents, err := d.Readdir(-1)
 754                 if err != nil {
 755                         return err
 756                 }
 757                 for _, ent := range ents {
 758                         if recurse && ent.IsDir() {
 759                                 err = walk(path + ent.Name() + "/")
 760                                 if err != nil {
 761                                         return err
 762                                 }
 763                         } else {
 764                                 files = append(files, fileListEnt{
 765                                         Name:  path + ent.Name(),
 766                                         Size:  ent.Size(),
 767                                         IsDir: ent.IsDir(),
 768                                 })
 769                         }
 770                 }
 771                 return nil
 772         }
 773         if err := walk(""); err != nil {
 774                 http.Error(w, "error getting directory listing: "+err.Error(), http.StatusInternalServerError)
 775                 return
 776         }
 777
 778         funcs := template.FuncMap{
 779                 "nbsp": func(s string) template.HTML {
 780                         return template.HTML(strings.Replace(s, " ", "&nbsp;", -1))
 781                 },
 782         }
 783         tmpl, err := template.New("dir").Funcs(funcs).Parse(dirListingTemplate)
 784         if err != nil {
 785                 http.Error(w, "error parsing template: "+err.Error(), http.StatusInternalServerError)
 786                 return
 787         }
 788         sort.Slice(files, func(i, j int) bool {
 789                 return files[i].Name < files[j].Name
 790         })
 791         w.WriteHeader(http.StatusOK)
 792         tmpl.Execute(w, map[string]interface{}{
 793                 "CollectionName": collectionName,
 794                 "Files":          files,
 795                 "Request":        r,
 796                 "StripParts":     strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
 797         })
 798 }
 799
 800 func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
 801         disposition := "inline"
 802         if isAttachment {
 803                 disposition = "attachment"
 804         }
 805         if strings.ContainsRune(r.RequestURI, '?') {
 806                 // Help the UA realize that the filename is just
 807                 // "filename.txt", not
 808                 // "filename.txt?disposition=attachment".
 809                 //
 810                 // TODO(TC): Follow advice at RFC 6266 appendix D
 811                 disposition += "; filename=" + strconv.QuoteToASCII(filename)
 812         }
 813         if disposition != "inline" {
 814                 w.Header().Set("Content-Disposition", disposition)
 815         }
 816 }
 817
 818 func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
 819         if formToken := r.FormValue("api_token"); formToken != "" {
 820                 if !credentialsOK {
 821                         // It is not safe to copy the provided token
 822                         // into a cookie unless the current vhost
 823                         // (origin) serves only a single collection or
 824                         // we are in TrustAllContent mode.
 825                         http.Error(w, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)", http.StatusBadRequest)
 826                         return
 827                 }
 828
 829                 // The HttpOnly flag is necessary to prevent
 830                 // JavaScript code (included in, or loaded by, a page
 831                 // in the collection being served) from employing the
 832                 // user's token beyond reading other files in the same
 833                 // domain, i.e., same collection.
 834                 //
 835                 // The 303 redirect is necessary in the case of a GET
 836                 // request to avoid exposing the token in the Location
 837                 // bar, and in the case of a POST request to avoid
 838                 // raising warnings when the user refreshes the
 839                 // resulting page.
 840                 http.SetCookie(w, &http.Cookie{
 841                         Name:     "arvados_api_token",
 842                         Value:    auth.EncodeTokenCookie([]byte(formToken)),
 843                         Path:     "/",
 844                         HttpOnly: true,
 845                         SameSite: http.SameSiteLaxMode,
 846                 })
 847         }
 848
 849         // Propagate query parameters (except api_token) from
 850         // the original request.
 851         redirQuery := r.URL.Query()
 852         redirQuery.Del("api_token")
 853
 854         u := r.URL
 855         if location != "" {
 856                 newu, err := u.Parse(location)
 857                 if err != nil {
 858                         http.Error(w, "error resolving redirect target: "+err.Error(), http.StatusInternalServerError)
 859                         return
 860                 }
 861                 u = newu
 862         }
 863         redir := (&url.URL{
 864                 Scheme:   r.URL.Scheme,
 865                 Host:     r.Host,
 866                 Path:     u.Path,
 867                 RawQuery: redirQuery.Encode(),
 868         }).String()
 869
 870         w.Header().Add("Location", redir)
 871         w.WriteHeader(http.StatusSeeOther)
 872         io.WriteString(w, `<A href="`)
 873         io.WriteString(w, html.EscapeString(redir))
 874         io.WriteString(w, `">Continue</A>`)
 875 }
 876
 877 func (h *handler) userPermittedToUploadOrDownload(method string, tokenUser *arvados.User) bool {
 878         var permitDownload bool
 879         var permitUpload bool
 880         if tokenUser != nil && tokenUser.IsAdmin {
 881                 permitUpload = h.Cluster.Collections.WebDAVPermission.Admin.Upload
 882                 permitDownload = h.Cluster.Collections.WebDAVPermission.Admin.Download
 883         } else {
 884                 permitUpload = h.Cluster.Collections.WebDAVPermission.User.Upload
 885                 permitDownload = h.Cluster.Collections.WebDAVPermission.User.Download
 886         }
 887         if (method == "PUT" || method == "POST") && !permitUpload {
 888                 // Disallow operations that upload new files.
 889                 // Permit webdav operations that move existing files around.
 890                 return false
 891         } else if method == "GET" && !permitDownload {
 892                 // Disallow downloading file contents.
 893                 // Permit webdav operations like PROPFIND that retrieve metadata
 894                 // but not file contents.
 895                 return false
 896         }
 897         return true
 898 }
 899
 900 func (h *handler) logUploadOrDownload(
 901         r *http.Request,
 902         client *arvadosclient.ArvadosClient,
 903         fs arvados.CustomFileSystem,
 904         filepath string,
 905         collection *arvados.Collection,
 906         user *arvados.User) {
 907
 908         log := ctxlog.FromContext(r.Context())
 909         props := make(map[string]string)
 910         props["reqPath"] = r.URL.Path
 911         var useruuid string
 912         if user != nil {
 913                 log = log.WithField("user_uuid", user.UUID).
 914                         WithField("user_full_name", user.FullName)
 915                 useruuid = user.UUID
 916         } else {
 917                 useruuid = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Cluster.ClusterID)
 918         }
 919         if collection == nil && fs != nil {
 920                 collection, filepath = h.determineCollection(fs, filepath)
 921         }
 922         if collection != nil {
 923                 log = log.WithField("collection_file_path", filepath)
 924                 props["collection_file_path"] = filepath
 925                 // h.determineCollection populates the collection_uuid
 926                 // prop with the PDH, if this collection is being
 927                 // accessed via PDH. For logging, we use a different
 928                 // field depending on whether it's a UUID or PDH.
 929                 if len(collection.UUID) > 32 {
 930                         log = log.WithField("portable_data_hash", collection.UUID)
 931                         props["portable_data_hash"] = collection.UUID
 932                 } else {
 933                         log = log.WithField("collection_uuid", collection.UUID)
 934                         props["collection_uuid"] = collection.UUID
 935                 }
 936         }
 937         if r.Method == "PUT" || r.Method == "POST" {
 938                 log.Info("File upload")
 939                 if h.Cluster.Collections.WebDAVLogEvents {
 940                         go func() {
 941                                 lr := arvadosclient.Dict{"log": arvadosclient.Dict{
 942                                         "object_uuid": useruuid,
 943                                         "event_type":  "file_upload",
 944                                         "properties":  props}}
 945                                 err := client.Create("logs", lr, nil)
 946                                 if err != nil {
 947                                         log.WithError(err).Error("Failed to create upload log event on API server")
 948                                 }
 949                         }()
 950                 }
 951         } else if r.Method == "GET" {
 952                 if collection != nil && collection.PortableDataHash != "" {
 953                         log = log.WithField("portable_data_hash", collection.PortableDataHash)
 954                         props["portable_data_hash"] = collection.PortableDataHash
 955                 }
 956                 log.Info("File download")
 957                 if h.Cluster.Collections.WebDAVLogEvents {
 958                         go func() {
 959                                 lr := arvadosclient.Dict{"log": arvadosclient.Dict{
 960                                         "object_uuid": useruuid,
 961                                         "event_type":  "file_download",
 962                                         "properties":  props}}
 963                                 err := client.Create("logs", lr, nil)
 964                                 if err != nil {
 965                                         log.WithError(err).Error("Failed to create download log event on API server")
 966                                 }
 967                         }()
 968                 }
 969         }
 970 }
 971
 972 func (h *handler) determineCollection(fs arvados.CustomFileSystem, path string) (*arvados.Collection, string) {
 973         target := strings.TrimSuffix(path, "/")
 974         for {
 975                 fi, err := fs.Stat(target)
 976                 if err != nil {
 977                         return nil, ""
 978                 }
 979                 switch src := fi.Sys().(type) {
 980                 case *arvados.Collection:
 981                         return src, strings.TrimPrefix(path[len(target):], "/")
 982                 case *arvados.Group:
 983                         return nil, ""
 984                 default:
 985                         if _, ok := src.(error); ok {
 986                                 return nil, ""
 987                         }
 988                 }
 989                 // Try parent
 990                 cut := strings.LastIndexByte(target, '/')
 991                 if cut < 0 {
 992                         return nil, ""
 993                 }
 994                 target = target[:cut]
 995         }
 996 }