services/keep-web/handler.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "html"
  12         "html/template"
  13         "io"
  14         "net"
  15         "net/http"
  16         "net/url"
  17         "os"
  18         "path"
  19         "slices"
  20         "sort"
  21         "strconv"
  22         "strings"
  23         "sync"
  24         "time"
  25
  26         "git.arvados.org/arvados.git/lib/cmd"
  27         "git.arvados.org/arvados.git/lib/webdavfs"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  30         "git.arvados.org/arvados.git/sdk/go/auth"
  31         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  32         "git.arvados.org/arvados.git/sdk/go/httpserver"
  33         "github.com/gotd/contrib/http_range"
  34         "github.com/sirupsen/logrus"
  35         "golang.org/x/net/webdav"
  36 )
  37
  38 type handler struct {
  39         Cache   cache
  40         Cluster *arvados.Cluster
  41         metrics *metrics
  42
  43         fileEventLogs         map[fileEventLog]time.Time
  44         fileEventLogsMtx      sync.Mutex
  45         fileEventLogsNextTidy time.Time
  46
  47         s3SecretCache         map[string]*cachedS3Secret
  48         s3SecretCacheMtx      sync.Mutex
  49         s3SecretCacheNextTidy time.Time
  50 }
  51
  52 var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
  53
  54 var notFoundMessage = "Not Found"
  55 var unauthorizedMessage = "401 Unauthorized\n\nA valid Arvados token must be provided to access this resource."
  56
  57 // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
  58 // PDH (even if it is a PDH with "+" replaced by " " or "-");
  59 // otherwise "".
  60 func parseCollectionIDFromURL(s string) string {
  61         if arvadosclient.UUIDMatch(s) {
  62                 return s
  63         }
  64         if pdh := urlPDHDecoder.Replace(s); arvadosclient.PDHMatch(pdh) {
  65                 return pdh
  66         }
  67         return ""
  68 }
  69
  70 func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
  71         json.NewEncoder(w).Encode(struct{ Version string }{cmd.Version.String()})
  72 }
  73
  74 type errorWithHTTPStatus interface {
  75         HTTPStatus() int
  76 }
  77
  78 // updateOnSuccess wraps httpserver.ResponseWriter. If the handler
  79 // sends an HTTP header indicating success, updateOnSuccess first
  80 // calls the provided update func. If the update func fails, an error
  81 // response is sent (using the error's HTTP status or 500 if none),
  82 // and the status code and body sent by the handler are ignored (all
  83 // response writes return the update error).
  84 type updateOnSuccess struct {
  85         httpserver.ResponseWriter
  86         logger     logrus.FieldLogger
  87         update     func() error
  88         sentHeader bool
  89         err        error
  90 }
  91
  92 func (uos *updateOnSuccess) Write(p []byte) (int, error) {
  93         if !uos.sentHeader {
  94                 uos.WriteHeader(http.StatusOK)
  95         }
  96         if uos.err != nil {
  97                 return 0, uos.err
  98         }
  99         return uos.ResponseWriter.Write(p)
 100 }
 101
 102 func (uos *updateOnSuccess) WriteHeader(code int) {
 103         if !uos.sentHeader {
 104                 uos.sentHeader = true
 105                 if code >= 200 && code < 400 {
 106                         if uos.err = uos.update(); uos.err != nil {
 107                                 code := http.StatusInternalServerError
 108                                 if he := errorWithHTTPStatus(nil); errors.As(uos.err, &he) {
 109                                         code = he.HTTPStatus()
 110                                 }
 111                                 uos.logger.WithError(uos.err).Errorf("update() returned %T error, changing response to HTTP %d", uos.err, code)
 112                                 http.Error(uos.ResponseWriter, uos.err.Error(), code)
 113                                 return
 114                         }
 115                 }
 116         }
 117         uos.ResponseWriter.WriteHeader(code)
 118 }
 119
 120 var (
 121         corsAllowHeadersHeader = strings.Join([]string{
 122                 "Authorization", "Content-Type", "Range",
 123                 // WebDAV request headers:
 124                 "Depth", "Destination", "If", "Lock-Token", "Overwrite", "Timeout", "Cache-Control",
 125         }, ", ")
 126         writeMethod = map[string]bool{
 127                 "COPY":      true,
 128                 "DELETE":    true,
 129                 "LOCK":      true,
 130                 "MKCOL":     true,
 131                 "MOVE":      true,
 132                 "PROPPATCH": true,
 133                 "PUT":       true,
 134                 "UNLOCK":    true,
 135         }
 136         webdavMethod = map[string]bool{
 137                 "COPY":      true,
 138                 "DELETE":    true,
 139                 "LOCK":      true,
 140                 "MKCOL":     true,
 141                 "MOVE":      true,
 142                 "OPTIONS":   true,
 143                 "PROPFIND":  true,
 144                 "PROPPATCH": true,
 145                 "PUT":       true,
 146                 "RMCOL":     true,
 147                 "UNLOCK":    true,
 148         }
 149         browserMethod = map[string]bool{
 150                 "GET":  true,
 151                 "HEAD": true,
 152                 "POST": true,
 153         }
 154         // top-level dirs to serve with siteFS
 155         siteFSDir = map[string]bool{
 156                 "":      true, // root directory
 157                 "by_id": true,
 158                 "users": true,
 159         }
 160 )
 161
 162 func stripDefaultPort(host string) string {
 163         // Will consider port 80 and port 443 to be the same vhost.  I think that's fine.
 164         u := &url.URL{Host: host}
 165         if p := u.Port(); p == "80" || p == "443" {
 166                 return strings.ToLower(u.Hostname())
 167         } else {
 168                 return strings.ToLower(host)
 169         }
 170 }
 171
 172 // CheckHealth implements service.Handler.
 173 func (h *handler) CheckHealth() error {
 174         return nil
 175 }
 176
 177 // Done implements service.Handler.
 178 func (h *handler) Done() <-chan struct{} {
 179         return nil
 180 }
 181
 182 // ServeHTTP implements http.Handler.
 183 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 184         if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" {
 185                 r.URL.Scheme = xfp
 186         }
 187
 188         wbuffer := newWriteBuffer(wOrig, int(h.Cluster.Collections.WebDAVOutputBuffer))
 189         defer wbuffer.Close()
 190         w := httpserver.WrapResponseWriter(responseWriter{
 191                 Writer:         wbuffer,
 192                 ResponseWriter: wOrig,
 193         })
 194
 195         if r.Method == "OPTIONS" && ServeCORSPreflight(w, r.Header) {
 196                 return
 197         }
 198
 199         if !browserMethod[r.Method] && !webdavMethod[r.Method] {
 200                 w.WriteHeader(http.StatusMethodNotAllowed)
 201                 return
 202         }
 203
 204         if r.Header.Get("Origin") != "" {
 205                 // Allow simple cross-origin requests without user
 206                 // credentials ("user credentials" as defined by CORS,
 207                 // i.e., cookies, HTTP authentication, and client-side
 208                 // SSL certificates. See
 209                 // http://www.w3.org/TR/cors/#user-credentials).
 210                 w.Header().Set("Access-Control-Allow-Origin", "*")
 211                 w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
 212         }
 213
 214         if h.serveS3(w, r) {
 215                 return
 216         }
 217
 218         // webdavPrefix is the leading portion of r.URL.Path that
 219         // should be ignored by the webdav handler, if any.
 220         //
 221         // req "/c={id}/..." -> webdavPrefix "/c={id}"
 222         // req "/by_id/..." -> webdavPrefix ""
 223         //
 224         // Note: in the code immediately below, we set webdavPrefix
 225         // only if it was explicitly set by the client. Otherwise, it
 226         // gets set later, after checking the request path for cases
 227         // like "/c={id}/...".
 228         webdavPrefix := ""
 229         arvPath := r.URL.Path
 230         if prefix := r.Header.Get("X-Webdav-Prefix"); prefix != "" {
 231                 // Enable a proxy (e.g., container log handler in
 232                 // controller) to satisfy a request for path
 233                 // "/foo/bar/baz.txt" using content from
 234                 // "//abc123-4.internal/bar/baz.txt", by adding a
 235                 // request header "X-Webdav-Prefix: /foo"
 236                 if !strings.HasPrefix(arvPath, prefix) {
 237                         http.Error(w, "X-Webdav-Prefix header is not a prefix of the requested path", http.StatusBadRequest)
 238                         return
 239                 }
 240                 arvPath = r.URL.Path[len(prefix):]
 241                 if arvPath == "" {
 242                         arvPath = "/"
 243                 }
 244                 w.Header().Set("Vary", "X-Webdav-Prefix, "+w.Header().Get("Vary"))
 245                 webdavPrefix = prefix
 246         }
 247         pathParts := strings.Split(arvPath[1:], "/")
 248
 249         var stripParts int
 250         var collectionID string
 251         var tokens []string
 252         var reqTokens []string
 253         var pathToken bool
 254         var attachment bool
 255         var useSiteFS bool
 256         credentialsOK := h.Cluster.Collections.TrustAllContent
 257         reasonNotAcceptingCredentials := ""
 258
 259         if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Cluster.Services.WebDAVDownload.ExternalURL.Host) {
 260                 credentialsOK = true
 261                 attachment = true
 262         } else if r.FormValue("disposition") == "attachment" {
 263                 attachment = true
 264         }
 265
 266         if !credentialsOK {
 267                 reasonNotAcceptingCredentials = fmt.Sprintf("vhost %q does not specify a single collection ID or match Services.WebDAVDownload.ExternalURL %q, and Collections.TrustAllContent is false",
 268                         r.Host, h.Cluster.Services.WebDAVDownload.ExternalURL)
 269         }
 270
 271         if collectionID = arvados.CollectionIDFromDNSName(r.Host); collectionID != "" {
 272                 // http://ID.collections.example/PATH...
 273                 credentialsOK = true
 274         } else if r.URL.Path == "/status.json" {
 275                 h.serveStatus(w, r)
 276                 return
 277         } else if siteFSDir[pathParts[0]] {
 278                 useSiteFS = true
 279         } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
 280                 // /c=ID[/PATH...]
 281                 collectionID = parseCollectionIDFromURL(pathParts[0][2:])
 282                 stripParts = 1
 283         } else if len(pathParts) >= 2 && pathParts[0] == "collections" {
 284                 if len(pathParts) >= 4 && pathParts[1] == "download" {
 285                         // /collections/download/ID/TOKEN/PATH...
 286                         collectionID = parseCollectionIDFromURL(pathParts[2])
 287                         tokens = []string{pathParts[3]}
 288                         stripParts = 4
 289                         pathToken = true
 290                 } else {
 291                         // /collections/ID/PATH...
 292                         collectionID = parseCollectionIDFromURL(pathParts[1])
 293                         stripParts = 2
 294                         // This path is only meant to work for public
 295                         // data. Tokens provided with the request are
 296                         // ignored.
 297                         credentialsOK = false
 298                         reasonNotAcceptingCredentials = "the '/collections/UUID/PATH' form only works for public data"
 299                 }
 300         }
 301
 302         forceReload := false
 303         if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
 304                 forceReload = true
 305         }
 306
 307         if credentialsOK {
 308                 reqTokens = auth.CredentialsFromRequest(r).Tokens
 309         }
 310
 311         r.ParseForm()
 312         origin := r.Header.Get("Origin")
 313         cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
 314         safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
 315         // Important distinction: safeAttachment checks whether api_token exists
 316         // as a query parameter. haveFormTokens checks whether api_token exists
 317         // as request form data *or* a query parameter. Different checks are
 318         // necessary because both the request disposition and the location of
 319         // the API token affect whether or not the request needs to be
 320         // redirected. The different branch comments below explain further.
 321         safeAttachment := attachment && !r.URL.Query().Has("api_token")
 322         if formTokens, haveFormTokens := r.Form["api_token"]; !haveFormTokens {
 323                 // No token to use or redact.
 324         } else if safeAjax || safeAttachment {
 325                 // If this is a cross-origin request, the URL won't
 326                 // appear in the browser's address bar, so
 327                 // substituting a clipboard-safe URL is pointless.
 328                 // Redirect-with-cookie wouldn't work anyway, because
 329                 // it's not safe to allow third-party use of our
 330                 // cookie.
 331                 //
 332                 // If we're supplying an attachment, we don't need to
 333                 // convert POST to GET to avoid the "really resubmit
 334                 // form?" problem, so provided the token isn't
 335                 // embedded in the URL, there's no reason to do
 336                 // redirect-with-cookie in this case either.
 337                 for _, tok := range formTokens {
 338                         reqTokens = append(reqTokens, tok)
 339                 }
 340         } else if browserMethod[r.Method] {
 341                 // If this is a page view, and the client provided a
 342                 // token via query string or POST body, we must put
 343                 // the token in an HttpOnly cookie, and redirect to an
 344                 // equivalent URL with the query param redacted and
 345                 // method = GET.
 346                 h.seeOtherWithCookie(w, r, "", credentialsOK)
 347                 return
 348         }
 349
 350         targetPath := pathParts[stripParts:]
 351         if tokens == nil && len(targetPath) > 0 && strings.HasPrefix(targetPath[0], "t=") {
 352                 // http://ID.example/t=TOKEN/PATH...
 353                 // /c=ID/t=TOKEN/PATH...
 354                 //
 355                 // This form must only be used to pass scoped tokens
 356                 // that give permission for a single collection. See
 357                 // FormValue case above.
 358                 tokens = []string{targetPath[0][2:]}
 359                 pathToken = true
 360                 targetPath = targetPath[1:]
 361                 stripParts++
 362         }
 363
 364         // fsprefix is the path from sitefs root to the sitefs
 365         // directory (implicitly or explicitly) indicated by the
 366         // leading / in the request path.
 367         //
 368         // Request "/by_id/..." -> fsprefix ""
 369         // Request "/c={id}/..." -> fsprefix "/by_id/{id}/"
 370         fsprefix := ""
 371         if useSiteFS {
 372                 if writeMethod[r.Method] {
 373                         http.Error(w, webdavfs.ErrReadOnly.Error(), http.StatusMethodNotAllowed)
 374                         return
 375                 }
 376                 if len(reqTokens) == 0 {
 377                         w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 378                         http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
 379                         return
 380                 }
 381                 tokens = reqTokens
 382         } else if collectionID == "" {
 383                 http.Error(w, notFoundMessage, http.StatusNotFound)
 384                 return
 385         } else {
 386                 fsprefix = "by_id/" + collectionID + "/"
 387         }
 388
 389         if src := r.Header.Get("X-Webdav-Source"); strings.HasPrefix(src, "/") && !strings.Contains(src, "//") && !strings.Contains(src, "/../") {
 390                 // Clients (specifically, the container log gateway)
 391                 // use X-Webdav-Source to specify that although the
 392                 // request path (and other webdav fields in the
 393                 // request) refer to target "/abc", the intended
 394                 // target is actually
 395                 // "{x-webdav-source-value}/abc".
 396                 //
 397                 // This, combined with X-Webdav-Prefix, enables the
 398                 // container log gateway to effectively alter the
 399                 // target path when proxying a request, without
 400                 // needing to rewrite all the other webdav
 401                 // request/response fields that might mention the
 402                 // target path.
 403                 fsprefix += src[1:]
 404         }
 405
 406         if tokens == nil {
 407                 tokens = reqTokens
 408                 if h.Cluster.Users.AnonymousUserToken != "" {
 409                         tokens = append(tokens, h.Cluster.Users.AnonymousUserToken)
 410                 }
 411         }
 412
 413         if len(targetPath) > 0 && targetPath[0] == "_" {
 414                 // If a collection has a directory called "t=foo" or
 415                 // "_", it can be served at
 416                 // //collections.example/_/t=foo/ or
 417                 // //collections.example/_/_/ respectively:
 418                 // //collections.example/t=foo/ won't work because
 419                 // t=foo will be interpreted as a token "foo".
 420                 targetPath = targetPath[1:]
 421                 stripParts++
 422         }
 423
 424         dirOpenMode := os.O_RDONLY
 425         if writeMethod[r.Method] {
 426                 dirOpenMode = os.O_RDWR
 427         }
 428
 429         var tokenValid bool
 430         var tokenScopeProblem bool
 431         var token string
 432         var tokenUser *arvados.User
 433         var sessionFS arvados.CustomFileSystem
 434         var targetFS arvados.FileSystem
 435         var session *cachedSession
 436         var collectionDir arvados.File
 437         for _, token = range tokens {
 438                 var statusErr errorWithHTTPStatus
 439                 fs, sess, user, err := h.Cache.GetSession(token)
 440                 if errors.As(err, &statusErr) && statusErr.HTTPStatus() == http.StatusUnauthorized {
 441                         // bad token
 442                         continue
 443                 } else if err != nil {
 444                         http.Error(w, "cache error: "+err.Error(), http.StatusInternalServerError)
 445                         return
 446                 }
 447                 if token != h.Cluster.Users.AnonymousUserToken {
 448                         tokenValid = true
 449                 }
 450                 f, err := fs.OpenFile(fsprefix, dirOpenMode, 0)
 451                 if errors.As(err, &statusErr) &&
 452                         statusErr.HTTPStatus() == http.StatusForbidden &&
 453                         token != h.Cluster.Users.AnonymousUserToken {
 454                         // collection id is outside scope of supplied
 455                         // token
 456                         tokenScopeProblem = true
 457                         sess.Release()
 458                         continue
 459                 } else if os.IsNotExist(err) {
 460                         // collection does not exist or is not
 461                         // readable using this token
 462                         sess.Release()
 463                         continue
 464                 } else if err != nil {
 465                         http.Error(w, err.Error(), http.StatusInternalServerError)
 466                         sess.Release()
 467                         return
 468                 }
 469                 defer f.Close()
 470
 471                 collectionDir, sessionFS, session, tokenUser = f, fs, sess, user
 472                 break
 473         }
 474
 475         // releaseSession() is equivalent to session.Release() except
 476         // that it's a no-op if (1) session is nil, or (2) it has
 477         // already been called.
 478         //
 479         // This way, we can do a defer call here to ensure it gets
 480         // called in all code paths, and also call it inline (see
 481         // below) in the cases where we want to release the lock
 482         // before returning.
 483         releaseSession := func() {}
 484         if session != nil {
 485                 var releaseSessionOnce sync.Once
 486                 releaseSession = func() { releaseSessionOnce.Do(func() { session.Release() }) }
 487         }
 488         defer releaseSession()
 489
 490         if forceReload && collectionDir != nil {
 491                 err := collectionDir.Sync()
 492                 if err != nil {
 493                         if he := errorWithHTTPStatus(nil); errors.As(err, &he) {
 494                                 http.Error(w, err.Error(), he.HTTPStatus())
 495                         } else {
 496                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 497                         }
 498                         return
 499                 }
 500         }
 501         if session == nil {
 502                 if pathToken {
 503                         // The URL is a "secret sharing link" that
 504                         // didn't work out.  Asking the client for
 505                         // additional credentials would just be
 506                         // confusing.
 507                         http.Error(w, notFoundMessage, http.StatusNotFound)
 508                         return
 509                 }
 510                 if tokenValid {
 511                         // The client provided valid token(s), but the
 512                         // collection was not found.
 513                         http.Error(w, notFoundMessage, http.StatusNotFound)
 514                         return
 515                 }
 516                 if tokenScopeProblem {
 517                         // The client provided a valid token but
 518                         // fetching a collection returned 401, which
 519                         // means the token scope doesn't permit
 520                         // fetching that collection.
 521                         http.Error(w, notFoundMessage, http.StatusForbidden)
 522                         return
 523                 }
 524                 // The client's token was invalid (e.g., expired), or
 525                 // the client didn't even provide one.  Redirect to
 526                 // workbench2's login-and-redirect-to-download url if
 527                 // this is a browser navigation request. (The redirect
 528                 // flow can't preserve the original method if it's not
 529                 // GET, and doesn't make sense if the UA is a
 530                 // command-line tool, is trying to load an inline
 531                 // image, etc.; in these cases, there's nothing we can
 532                 // do, so return 401 unauthorized.)
 533                 //
 534                 // Note Sec-Fetch-Mode is sent by all non-EOL
 535                 // browsers, except Safari.
 536                 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Mode
 537                 //
 538                 // TODO(TC): This response would be confusing to
 539                 // someone trying (anonymously) to download public
 540                 // data that has been deleted.  Allow a referrer to
 541                 // provide this context somehow?
 542                 if r.Method == http.MethodGet && r.Header.Get("Sec-Fetch-Mode") == "navigate" {
 543                         target := url.URL(h.Cluster.Services.Workbench2.ExternalURL)
 544                         redirkey := "redirectToPreview"
 545                         if attachment {
 546                                 redirkey = "redirectToDownload"
 547                         }
 548                         callback := "/c=" + collectionID + "/" + strings.Join(targetPath, "/")
 549                         query := url.Values{redirkey: {callback}}
 550                         queryString := query.Encode()
 551                         // Note: Encode (and QueryEscape function) turns space
 552                         // into plus sign (+) rather than %20 (the plus sign
 553                         // becomes %2B); that is the rule for web forms data
 554                         // sent in URL query part via GET, but we're not
 555                         // emulating forms here. Client JS APIs
 556                         // (URLSearchParam#get, decodeURIComponent) will
 557                         // decode %20, but while the former also expects the
 558                         // form-specific encoding, the latter doesn't.
 559                         // Encode() almost encodes everything; RFC 3986 3.4
 560                         // says "it is sometimes better for usability" to not
 561                         // encode / and ? when passing URI reference in query.
 562                         // This is also legal according to WHATWG URL spec and
 563                         // can be desirable for debugging webapp.
 564                         // We can let slash / appear in the encoded query, and
 565                         // equality-sign = too, but exempting ? is not very
 566                         // useful.
 567                         // Plus-sign, hash, and ampersand are never exempt.
 568                         r := strings.NewReplacer("+", "%20", "%2F", "/", "%3D", "=")
 569                         target.RawQuery = r.Replace(queryString)
 570                         w.Header().Add("Location", target.String())
 571                         w.WriteHeader(http.StatusSeeOther)
 572                         return
 573                 }
 574                 if !credentialsOK {
 575                         http.Error(w, fmt.Sprintf("Authorization tokens are not accepted here: %v, and no anonymous user token is configured.", reasonNotAcceptingCredentials), http.StatusUnauthorized)
 576                         return
 577                 }
 578                 // If none of the above cases apply, suggest the
 579                 // user-agent (which is either a non-browser agent
 580                 // like wget, or a browser that can't redirect through
 581                 // a login flow) prompt the user for credentials.
 582                 w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
 583                 http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
 584                 return
 585         }
 586
 587         if r.Method == http.MethodGet || r.Method == http.MethodHead {
 588                 targetfnm := fsprefix + strings.Join(pathParts[stripParts:], "/")
 589                 if fi, err := sessionFS.Stat(targetfnm); err == nil && fi.IsDir() {
 590                         releaseSession() // because we won't be writing anything
 591                         if !strings.HasSuffix(r.URL.Path, "/") {
 592                                 h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
 593                         } else {
 594                                 h.serveDirectory(w, r, fi.Name(), sessionFS, targetfnm, !useSiteFS)
 595                         }
 596                         return
 597                 }
 598         }
 599
 600         var basename string
 601         if len(targetPath) > 0 {
 602                 basename = targetPath[len(targetPath)-1]
 603         }
 604         if arvadosclient.PDHMatch(collectionID) && writeMethod[r.Method] {
 605                 http.Error(w, webdavfs.ErrReadOnly.Error(), http.StatusMethodNotAllowed)
 606                 return
 607         }
 608         if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
 609                 http.Error(w, "Not permitted", http.StatusForbidden)
 610                 return
 611         }
 612         fstarget := fsprefix + strings.Join(targetPath, "/")
 613         h.logUploadOrDownload(r, session.arvadosclient, sessionFS, fstarget, nil, tokenUser)
 614
 615         if webdavPrefix == "" && stripParts > 0 {
 616                 webdavPrefix = "/" + strings.Join(pathParts[:stripParts], "/")
 617         }
 618
 619         writing := writeMethod[r.Method]
 620         if writing {
 621                 // We implement write operations by writing to a
 622                 // temporary collection, then applying the change to
 623                 // the real collection using the replace_files option
 624                 // in a collection update request.  This lets us do
 625                 // the slow part (i.e., receive the file data from the
 626                 // client and write it to Keep) without worrying about
 627                 // side effects of other read/write operations.
 628                 //
 629                 // Collection update requests for a given collection
 630                 // are serialized by the controller, so we don't need
 631                 // to do any locking for that part either.
 632
 633                 // collprefix is the subdirectory in the target
 634                 // collection which (according to X-Webdav-Source) we
 635                 // should pretend is "/" for this request.
 636                 collprefix := strings.TrimPrefix(fsprefix, "by_id/"+collectionID+"/")
 637                 if len(collprefix) == len(fsprefix) {
 638                         http.Error(w, "internal error: writing to anything other than /by_id/{collectionID}", http.StatusInternalServerError)
 639                         return
 640                 }
 641                 colltarget := strings.Join(pathParts[stripParts:], "/")
 642                 colltarget = strings.TrimSuffix(colltarget, "/")
 643
 644                 // Create a temporary collection filesystem for webdav
 645                 // to operate on.
 646                 var tmpcoll arvados.Collection
 647                 client := session.client.WithRequestID(r.Header.Get("X-Request-Id"))
 648                 tmpfs, err := tmpcoll.FileSystem(client, session.keepclient)
 649                 if err != nil {
 650                         http.Error(w, err.Error(), http.StatusInternalServerError)
 651                         return
 652                 }
 653                 snap, err := arvados.Snapshot(sessionFS, "by_id/"+collectionID+"/")
 654                 if err != nil {
 655                         http.Error(w, "snapshot: "+err.Error(), http.StatusInternalServerError)
 656                         return
 657                 }
 658                 err = arvados.Splice(tmpfs, "/", snap)
 659                 if err != nil {
 660                         http.Error(w, "splice: "+err.Error(), http.StatusInternalServerError)
 661                         return
 662                 }
 663
 664                 targetFS = tmpfs
 665                 fsprefix = collprefix
 666                 replace := make(map[string]string)
 667
 668                 switch r.Method {
 669                 case "COPY", "MOVE":
 670                         dsturl, err := url.Parse(r.Header.Get("Destination"))
 671                         if err != nil {
 672                                 http.Error(w, err.Error(), http.StatusBadRequest)
 673                                 return
 674                         }
 675                         if dsturl.Host != "" && dsturl.Host != r.Host {
 676                                 http.Error(w, "destination host mismatch", http.StatusBadGateway)
 677                                 return
 678                         }
 679                         var dsttarget string
 680                         if webdavPrefix == "" {
 681                                 dsttarget = dsturl.Path
 682                         } else {
 683                                 dsttarget = strings.TrimPrefix(dsturl.Path, webdavPrefix)
 684                                 if len(dsttarget) == len(dsturl.Path) {
 685                                         http.Error(w, "destination path not supported", http.StatusBadRequest)
 686                                         return
 687                                 }
 688                         }
 689
 690                         srcspec := "current/" + colltarget
 691                         // RFC 4918 9.8.3: A COPY of "Depth: 0" only
 692                         // instructs that the collection and its
 693                         // properties, but not resources identified by
 694                         // its internal member URLs, are to be copied.
 695                         //
 696                         // ...meaning we will be creating an empty
 697                         // directory.
 698                         //
 699                         // RFC 4918 9.9.2: A client MUST NOT submit a
 700                         // Depth header on a MOVE on a collection with
 701                         // any value but "infinity".
 702                         //
 703                         // ...meaning we only need to consider this
 704                         // case for COPY, not for MOVE.
 705                         if fi, err := tmpfs.Stat(colltarget); err == nil && fi.IsDir() && r.Method == "COPY" && r.Header.Get("Depth") == "0" {
 706                                 srcspec = "manifest_text/"
 707                         }
 708
 709                         replace[strings.TrimSuffix(dsttarget, "/")] = srcspec
 710                         if r.Method == "MOVE" {
 711                                 replace["/"+colltarget] = ""
 712                         }
 713                 case "MKCOL":
 714                         replace["/"+colltarget] = "manifest_text/"
 715                 case "DELETE":
 716                         if depth := r.Header.Get("Depth"); depth != "" && depth != "infinity" {
 717                                 http.Error(w, "invalid depth header, see RFC 4918 9.6.1", http.StatusBadRequest)
 718                                 return
 719                         }
 720                         replace["/"+colltarget] = ""
 721                 case "PUT":
 722                         // changes will be applied by updateOnSuccess
 723                         // update func below
 724                 case "LOCK", "UNLOCK", "PROPPATCH":
 725                         // no changes
 726                 default:
 727                         http.Error(w, "method missing", http.StatusInternalServerError)
 728                         return
 729                 }
 730
 731                 // Save the collection only if/when all
 732                 // webdav->filesystem operations succeed using our
 733                 // temporary collection -- and send a 500 error if the
 734                 // updates can't be saved.
 735                 logger := ctxlog.FromContext(r.Context())
 736                 w = &updateOnSuccess{
 737                         ResponseWriter: w,
 738                         logger:         logger,
 739                         update: func() error {
 740                                 var manifest string
 741                                 var snap *arvados.Subtree
 742                                 var err error
 743                                 if r.Method == "PUT" {
 744                                         snap, err = arvados.Snapshot(tmpfs, colltarget)
 745                                         if err != nil {
 746                                                 return fmt.Errorf("snapshot tmpfs: %w", err)
 747                                         }
 748                                         tmpfs, err = (&arvados.Collection{}).FileSystem(client, session.keepclient)
 749                                         err = arvados.Splice(tmpfs, "file", snap)
 750                                         if err != nil {
 751                                                 return fmt.Errorf("splice tmpfs: %w", err)
 752                                         }
 753                                         manifest, err = tmpfs.MarshalManifest(".")
 754                                         if err != nil {
 755                                                 return fmt.Errorf("marshal tmpfs: %w", err)
 756                                         }
 757                                         replace["/"+colltarget] = "manifest_text/file"
 758                                 } else if len(replace) == 0 {
 759                                         return nil
 760                                 }
 761                                 err = client.RequestAndDecode(nil, "PATCH", "arvados/v1/collections/"+collectionID, nil, map[string]interface{}{
 762                                         "replace_files": replace,
 763                                         "collection":    map[string]interface{}{"manifest_text": manifest}})
 764                                 var te arvados.TransactionError
 765                                 if errors.As(err, &te) {
 766                                         err = te
 767                                 }
 768                                 if err != nil {
 769                                         return err
 770                                 }
 771
 772                                 // We want subsequent reqs to account
 773                                 // for this change.  Sync() can be
 774                                 // expensive on a large collection, so
 775                                 // in the simple (and common) case of
 776                                 // uploading a single file, we just
 777                                 // splice the new file into sessionFS
 778                                 // instead.
 779                                 if r.Method == "PUT" {
 780                                         err = arvados.Splice(sessionFS, fstarget, snap)
 781                                         if err != nil {
 782                                                 logger.WithError(err).Warn("splice failed")
 783                                                 collectionDir.Sync()
 784                                         }
 785                                 } else {
 786                                         collectionDir.Sync()
 787                                 }
 788                                 return nil
 789                         }}
 790         } else {
 791                 // When writing, we need to block session renewal
 792                 // until we're finished, in order to guarantee the
 793                 // effect of the write is visible in future responses.
 794                 // But if we're not writing, we can release the lock
 795                 // early.  This enables us to keep renewing sessions
 796                 // and processing more requests even if a slow client
 797                 // takes a long time to download a large file.
 798                 releaseSession()
 799                 targetFS = sessionFS
 800         }
 801         if r.Method == http.MethodGet {
 802                 applyContentDispositionHdr(w, r, basename, attachment)
 803         }
 804         wh := &webdav.Handler{
 805                 Prefix: webdavPrefix,
 806                 FileSystem: &webdavfs.FS{
 807                         FileSystem:    targetFS,
 808                         Prefix:        fsprefix,
 809                         Writing:       writeMethod[r.Method],
 810                         AlwaysReadEOF: r.Method == "PROPFIND",
 811                 },
 812                 LockSystem: webdavfs.NoLockSystem,
 813                 Logger: func(r *http.Request, err error) {
 814                         if err != nil && !os.IsNotExist(err) {
 815                                 ctxlog.FromContext(r.Context()).WithError(err).Error("error reported by webdav handler")
 816                         }
 817                 },
 818         }
 819         h.metrics.track(wh, w, r)
 820         if r.Method == http.MethodGet && w.WroteStatus() == http.StatusOK {
 821                 wrote := int64(w.WroteBodyBytes())
 822                 fnm := strings.Join(pathParts[stripParts:], "/")
 823                 fi, err := wh.FileSystem.Stat(r.Context(), fnm)
 824                 if err == nil && fi.Size() != wrote {
 825                         var n int
 826                         f, err := wh.FileSystem.OpenFile(r.Context(), fnm, os.O_RDONLY, 0)
 827                         if err == nil {
 828                                 n, err = f.Read(make([]byte, 1024))
 829                                 f.Close()
 830                         }
 831                         ctxlog.FromContext(r.Context()).Errorf("stat.Size()==%d but only wrote %d bytes; read(1024) returns %d, %v", fi.Size(), wrote, n, err)
 832                 }
 833         }
 834 }
 835
 836 var dirListingTemplate = `<!DOCTYPE HTML>
 837 <HTML><HEAD>
 838   <META name="robots" content="NOINDEX">
 839   <TITLE>{{ .CollectionName }}</TITLE>
 840   <STYLE type="text/css">
 841     body {
 842       margin: 1.5em;
 843     }
 844     pre {
 845       background-color: #D9EDF7;
 846       border-radius: .25em;
 847       padding: .75em;
 848       overflow: auto;
 849     }
 850     .footer p {
 851       font-size: 82%;
 852     }
 853     hr {
 854       border: 1px solid #808080;
 855     }
 856     ul {
 857       padding: 0;
 858     }
 859     ul li {
 860       font-family: monospace;
 861       list-style: none;
 862     }
 863   </STYLE>
 864 </HEAD>
 865 <BODY>
 866
 867 <H1>{{ .CollectionName }}</H1>
 868
 869 <P>This collection of data files is being shared with you through
 870 Arvados.  You can download individual files listed below.  To download
 871 the entire directory tree with <CODE>wget</CODE>, try:</P>
 872
 873 <PRE id="wget-example">$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} {{ .QuotedUrlForWget }}</PRE>
 874
 875 <H2>File Listing</H2>
 876
 877 {{if .Files}}
 878 <UL>
 879 {{range .Files}}
 880 {{if .IsDir }}
 881   <LI>{{" " | printf "%15s  " | nbsp}}<A class="item" href="{{ .Href }}/">{{ .Name }}/</A></LI>
 882 {{else}}
 883   <LI>{{.Size | printf "%15d  " | nbsp}}<A class="item" href="{{ .Href }}">{{ .Name }}</A></LI>
 884 {{end}}
 885 {{end}}
 886 </UL>
 887 {{else}}
 888 <P>(No files; this collection is empty.)</P>
 889 {{end}}
 890
 891 <HR>
 892 <DIV class="footer">
 893   <P>
 894     About Arvados:
 895     Arvados is a free and open source software bioinformatics platform.
 896     To learn more, visit arvados.org.
 897     Arvados is not responsible for the files listed on this page.
 898   </P>
 899 </DIV>
 900
 901 </BODY>
 902 </HTML>
 903 `
 904
 905 type fileListEnt struct {
 906         Name  string
 907         Href  string
 908         Size  int64
 909         IsDir bool
 910 }
 911
 912 // Given a filesystem path like `foo/"bar baz"`, return an escaped
 913 // (percent-encoded) relative path like `./foo/%22bar%20%baz%22`.
 914 //
 915 // Note the result may contain html-unsafe characters like '&'. These
 916 // will be handled separately by the HTML templating engine as needed.
 917 func relativeHref(path string) string {
 918         u := &url.URL{Path: path}
 919         return "./" + u.EscapedPath()
 920 }
 921
 922 // Return a shell-quoted URL suitable for pasting to a command line
 923 // ("wget ...") to repeat the given HTTP request.
 924 func makeQuotedUrlForWget(r *http.Request) string {
 925         scheme := r.Header.Get("X-Forwarded-Proto")
 926         if scheme == "http" || scheme == "https" {
 927                 // use protocol reported by load balancer / proxy
 928         } else if r.TLS != nil {
 929                 scheme = "https"
 930         } else {
 931                 scheme = "http"
 932         }
 933         p := r.URL.EscapedPath()
 934         // An escaped path may still contain single quote chars, which
 935         // would interfere with our shell quoting. Avoid this by
 936         // escaping them as %27.
 937         return fmt.Sprintf("'%s://%s%s'", scheme, r.Host, strings.Replace(p, "'", "%27", -1))
 938 }
 939
 940 func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
 941         var files []fileListEnt
 942         var walk func(string) error
 943         if !strings.HasSuffix(base, "/") {
 944                 base = base + "/"
 945         }
 946         walk = func(path string) error {
 947                 dirname := base + path
 948                 if dirname != "/" {
 949                         dirname = strings.TrimSuffix(dirname, "/")
 950                 }
 951                 d, err := fs.Open(dirname)
 952                 if err != nil {
 953                         return err
 954                 }
 955                 ents, err := d.Readdir(-1)
 956                 if err != nil {
 957                         return err
 958                 }
 959                 for _, ent := range ents {
 960                         if recurse && ent.IsDir() {
 961                                 err = walk(path + ent.Name() + "/")
 962                                 if err != nil {
 963                                         return err
 964                                 }
 965                         } else {
 966                                 listingName := path + ent.Name()
 967                                 files = append(files, fileListEnt{
 968                                         Name:  listingName,
 969                                         Href:  relativeHref(listingName),
 970                                         Size:  ent.Size(),
 971                                         IsDir: ent.IsDir(),
 972                                 })
 973                         }
 974                 }
 975                 return nil
 976         }
 977         if err := walk(""); err != nil {
 978                 http.Error(w, "error getting directory listing: "+err.Error(), http.StatusInternalServerError)
 979                 return
 980         }
 981
 982         funcs := template.FuncMap{
 983                 "nbsp": func(s string) template.HTML {
 984                         return template.HTML(strings.Replace(s, " ", "&nbsp;", -1))
 985                 },
 986         }
 987         tmpl, err := template.New("dir").Funcs(funcs).Parse(dirListingTemplate)
 988         if err != nil {
 989                 http.Error(w, "error parsing template: "+err.Error(), http.StatusInternalServerError)
 990                 return
 991         }
 992         sort.Slice(files, func(i, j int) bool {
 993                 return files[i].Name < files[j].Name
 994         })
 995         w.WriteHeader(http.StatusOK)
 996         tmpl.Execute(w, map[string]interface{}{
 997                 "CollectionName":   collectionName,
 998                 "Files":            files,
 999                 "Request":          r,
1000                 "StripParts":       strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
1001                 "QuotedUrlForWget": makeQuotedUrlForWget(r),
1002         })
1003 }
1004
1005 func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename string, isAttachment bool) {
1006         disposition := "inline"
1007         if isAttachment {
1008                 disposition = "attachment"
1009         }
1010         if strings.ContainsRune(r.RequestURI, '?') {
1011                 // Help the UA realize that the filename is just
1012                 // "filename.txt", not
1013                 // "filename.txt?disposition=attachment".
1014                 //
1015                 // TODO(TC): Follow advice at RFC 6266 appendix D
1016                 disposition += "; filename=" + strconv.QuoteToASCII(filename)
1017         }
1018         if disposition != "inline" {
1019                 w.Header().Set("Content-Disposition", disposition)
1020         }
1021 }
1022
1023 func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
1024         if formTokens, haveFormTokens := r.Form["api_token"]; haveFormTokens {
1025                 if !credentialsOK {
1026                         // It is not safe to copy the provided token
1027                         // into a cookie unless the current vhost
1028                         // (origin) serves only a single collection or
1029                         // we are in TrustAllContent mode.
1030                         http.Error(w, "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)", http.StatusBadRequest)
1031                         return
1032                 }
1033
1034                 // The HttpOnly flag is necessary to prevent
1035                 // JavaScript code (included in, or loaded by, a page
1036                 // in the collection being served) from employing the
1037                 // user's token beyond reading other files in the same
1038                 // domain, i.e., same collection.
1039                 //
1040                 // The 303 redirect is necessary in the case of a GET
1041                 // request to avoid exposing the token in the Location
1042                 // bar, and in the case of a POST request to avoid
1043                 // raising warnings when the user refreshes the
1044                 // resulting page.
1045                 for _, tok := range formTokens {
1046                         if tok == "" {
1047                                 continue
1048                         }
1049                         http.SetCookie(w, &http.Cookie{
1050                                 Name:     "arvados_api_token",
1051                                 Value:    auth.EncodeTokenCookie([]byte(tok)),
1052                                 Path:     "/",
1053                                 HttpOnly: true,
1054                                 SameSite: http.SameSiteLaxMode,
1055                         })
1056                         break
1057                 }
1058         }
1059
1060         // Propagate query parameters (except api_token) from
1061         // the original request.
1062         redirQuery := r.URL.Query()
1063         redirQuery.Del("api_token")
1064
1065         u := r.URL
1066         if location != "" {
1067                 newu, err := u.Parse(location)
1068                 if err != nil {
1069                         http.Error(w, "error resolving redirect target: "+err.Error(), http.StatusInternalServerError)
1070                         return
1071                 }
1072                 u = newu
1073         }
1074         redir := (&url.URL{
1075                 Scheme:   r.URL.Scheme,
1076                 Host:     r.Host,
1077                 Path:     u.Path,
1078                 RawQuery: redirQuery.Encode(),
1079         }).String()
1080
1081         w.Header().Add("Location", redir)
1082         w.WriteHeader(http.StatusSeeOther)
1083         io.WriteString(w, `<A href="`)
1084         io.WriteString(w, html.EscapeString(redir))
1085         io.WriteString(w, `">Continue</A>`)
1086 }
1087
1088 func (h *handler) userPermittedToUploadOrDownload(method string, tokenUser *arvados.User) bool {
1089         var permitDownload bool
1090         var permitUpload bool
1091         if tokenUser != nil && tokenUser.IsAdmin {
1092                 permitUpload = h.Cluster.Collections.WebDAVPermission.Admin.Upload
1093                 permitDownload = h.Cluster.Collections.WebDAVPermission.Admin.Download
1094         } else {
1095                 permitUpload = h.Cluster.Collections.WebDAVPermission.User.Upload
1096                 permitDownload = h.Cluster.Collections.WebDAVPermission.User.Download
1097         }
1098         if (method == "PUT" || method == "POST") && !permitUpload {
1099                 // Disallow operations that upload new files.
1100                 // Permit webdav operations that move existing files around.
1101                 return false
1102         } else if method == "GET" && !permitDownload {
1103                 // Disallow downloading file contents.
1104                 // Permit webdav operations like PROPFIND that retrieve metadata
1105                 // but not file contents.
1106                 return false
1107         }
1108         return true
1109 }
1110
1111 type fileEventLog struct {
1112         requestPath  string
1113         eventType    string
1114         userUUID     string
1115         userFullName string
1116         collUUID     string
1117         collPDH      string
1118         collFilePath string
1119         clientAddr   string
1120         clientToken  string
1121 }
1122
1123 func newFileEventLog(
1124         h *handler,
1125         r *http.Request,
1126         filepath string,
1127         collection *arvados.Collection,
1128         user *arvados.User,
1129         token string,
1130 ) *fileEventLog {
1131         var eventType string
1132         switch r.Method {
1133         case "POST", "PUT":
1134                 eventType = "file_upload"
1135         case "GET":
1136                 eventType = "file_download"
1137         default:
1138                 return nil
1139         }
1140
1141         // We want to log the address of the proxy closest to keep-web—the last
1142         // value in the X-Forwarded-For list—or the client address if there is no
1143         // valid proxy.
1144         var clientAddr string
1145         // 1. Build a slice of proxy addresses from X-Forwarded-For.
1146         xff := strings.Join(r.Header.Values("X-Forwarded-For"), ",")
1147         addrs := strings.Split(xff, ",")
1148         // 2. Reverse the slice so it's in our most preferred order for logging.
1149         slices.Reverse(addrs)
1150         // 3. Append the client address to that slice.
1151         if addr, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
1152                 addrs = append(addrs, addr)
1153         }
1154         // 4. Use the first valid address in the slice.
1155         for _, addr := range addrs {
1156                 if ip := net.ParseIP(strings.TrimSpace(addr)); ip != nil {
1157                         clientAddr = ip.String()
1158                         break
1159                 }
1160         }
1161
1162         ev := &fileEventLog{
1163                 requestPath: r.URL.Path,
1164                 eventType:   eventType,
1165                 clientAddr:  clientAddr,
1166                 clientToken: token,
1167         }
1168
1169         if user != nil {
1170                 ev.userUUID = user.UUID
1171                 ev.userFullName = user.FullName
1172         } else {
1173                 ev.userUUID = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Cluster.ClusterID)
1174         }
1175
1176         if collection != nil {
1177                 ev.collFilePath = filepath
1178                 // h.determineCollection populates the collection_uuid
1179                 // prop with the PDH, if this collection is being
1180                 // accessed via PDH. For logging, we use a different
1181                 // field depending on whether it's a UUID or PDH.
1182                 if len(collection.UUID) > 32 {
1183                         ev.collPDH = collection.UUID
1184                 } else {
1185                         ev.collPDH = collection.PortableDataHash
1186                         ev.collUUID = collection.UUID
1187                 }
1188         }
1189
1190         return ev
1191 }
1192
1193 func (ev *fileEventLog) shouldLogPDH() bool {
1194         return ev.eventType == "file_download" && ev.collPDH != ""
1195 }
1196
1197 func (ev *fileEventLog) asDict() arvadosclient.Dict {
1198         props := arvadosclient.Dict{
1199                 "reqPath":              ev.requestPath,
1200                 "collection_uuid":      ev.collUUID,
1201                 "collection_file_path": ev.collFilePath,
1202         }
1203         if ev.shouldLogPDH() {
1204                 props["portable_data_hash"] = ev.collPDH
1205         }
1206         return arvadosclient.Dict{
1207                 "object_uuid": ev.userUUID,
1208                 "event_type":  ev.eventType,
1209                 "properties":  props,
1210         }
1211 }
1212
1213 func (ev *fileEventLog) asFields() logrus.Fields {
1214         fields := logrus.Fields{
1215                 "collection_file_path": ev.collFilePath,
1216                 "collection_uuid":      ev.collUUID,
1217                 "user_uuid":            ev.userUUID,
1218         }
1219         if ev.shouldLogPDH() {
1220                 fields["portable_data_hash"] = ev.collPDH
1221         }
1222         if !strings.HasSuffix(ev.userUUID, "-tpzed-anonymouspublic") {
1223                 fields["user_full_name"] = ev.userFullName
1224         }
1225         return fields
1226 }
1227
1228 func (h *handler) shouldLogEvent(
1229         event *fileEventLog,
1230         req *http.Request,
1231         fileInfo os.FileInfo,
1232         t time.Time,
1233 ) bool {
1234         if event == nil {
1235                 return false
1236         } else if event.eventType != "file_download" ||
1237                 h.Cluster.Collections.WebDAVLogDownloadInterval == 0 ||
1238                 fileInfo == nil {
1239                 return true
1240         }
1241         td := h.Cluster.Collections.WebDAVLogDownloadInterval.Duration()
1242         cutoff := t.Add(-td)
1243         ev := *event
1244         h.fileEventLogsMtx.Lock()
1245         defer h.fileEventLogsMtx.Unlock()
1246         if h.fileEventLogs == nil {
1247                 h.fileEventLogs = make(map[fileEventLog]time.Time)
1248         }
1249         shouldLog := h.fileEventLogs[ev].Before(cutoff)
1250         if !shouldLog {
1251                 // Go's http fs server evaluates http.Request.Header.Get("Range")
1252                 // (as of Go 1.22) so we should do the same.
1253                 // Don't worry about merging multiple headers, etc.
1254                 ranges, err := http_range.ParseRange(req.Header.Get("Range"), fileInfo.Size())
1255                 if ranges == nil || err != nil {
1256                         // The Range header was either empty or malformed.
1257                         // Err on the side of logging.
1258                         shouldLog = true
1259                 } else {
1260                         // Log this request only if it requested the first byte
1261                         // (our heuristic for "starting a new download").
1262                         for _, reqRange := range ranges {
1263                                 if reqRange.Start == 0 {
1264                                         shouldLog = true
1265                                         break
1266                                 }
1267                         }
1268                 }
1269         }
1270         if shouldLog {
1271                 h.fileEventLogs[ev] = t
1272         }
1273         if t.After(h.fileEventLogsNextTidy) {
1274                 for key, logTime := range h.fileEventLogs {
1275                         if logTime.Before(cutoff) {
1276                                 delete(h.fileEventLogs, key)
1277                         }
1278                 }
1279                 h.fileEventLogsNextTidy = t.Add(td)
1280         }
1281         return shouldLog
1282 }
1283
1284 func (h *handler) logUploadOrDownload(
1285         r *http.Request,
1286         client *arvadosclient.ArvadosClient,
1287         fs arvados.CustomFileSystem,
1288         filepath string,
1289         collection *arvados.Collection,
1290         user *arvados.User,
1291 ) {
1292         var fileInfo os.FileInfo
1293         if fs != nil {
1294                 if collection == nil {
1295                         collection, filepath = h.determineCollection(fs, filepath)
1296                 }
1297                 if collection != nil {
1298                         // It's okay to ignore this error because shouldLogEvent will
1299                         // always return true if fileInfo == nil.
1300                         fileInfo, _ = fs.Stat(path.Join("by_id", collection.UUID, filepath))
1301                 }
1302         }
1303         event := newFileEventLog(h, r, filepath, collection, user, client.ApiToken)
1304         if !h.shouldLogEvent(event, r, fileInfo, time.Now()) {
1305                 return
1306         }
1307         log := ctxlog.FromContext(r.Context()).WithFields(event.asFields())
1308         log.Info(strings.Replace(event.eventType, "file_", "File ", 1))
1309         if h.Cluster.Collections.WebDAVLogEvents {
1310                 go func() {
1311                         logReq := arvadosclient.Dict{"log": event.asDict()}
1312                         err := client.Create("logs", logReq, nil)
1313                         if err != nil {
1314                                 log.WithError(err).Errorf("Failed to create %s log event on API server", event.eventType)
1315                         }
1316                 }()
1317         }
1318 }
1319
1320 func (h *handler) determineCollection(fs arvados.CustomFileSystem, path string) (*arvados.Collection, string) {
1321         target := strings.TrimSuffix(path, "/")
1322         for cut := len(target); cut >= 0; cut = strings.LastIndexByte(target, '/') {
1323                 target = target[:cut]
1324                 fi, err := fs.Stat(target)
1325                 if os.IsNotExist(err) {
1326                         // creating a new file/dir, or download
1327                         // destined to fail
1328                         continue
1329                 } else if err != nil {
1330                         return nil, ""
1331                 }
1332                 switch src := fi.Sys().(type) {
1333                 case *arvados.Collection:
1334                         return src, strings.TrimPrefix(path[len(target):], "/")
1335                 case *arvados.Group:
1336                         return nil, ""
1337                 default:
1338                         if _, ok := src.(error); ok {
1339                                 return nil, ""
1340                         }
1341                 }
1342         }
1343         return nil, ""
1344 }
1345
1346 func ServeCORSPreflight(w http.ResponseWriter, header http.Header) bool {
1347         method := header.Get("Access-Control-Request-Method")
1348         if method == "" {
1349                 return false
1350         }
1351         if !browserMethod[method] && !webdavMethod[method] {
1352                 w.WriteHeader(http.StatusMethodNotAllowed)
1353                 return true
1354         }
1355         w.Header().Set("Access-Control-Allow-Headers", corsAllowHeadersHeader)
1356         w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
1357         w.Header().Set("Access-Control-Allow-Origin", "*")
1358         w.Header().Set("Access-Control-Max-Age", "86400")
1359         return true
1360 }