13111: Expose read-only /users/ tree.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 2d6dd2b5a6bf38b71fa5006e753aa7a13ae0096d..6a7ce2a68d1e7b8597387ad3cef4b0e71cd5c094 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -5,7 +5,6 @@
 package main
 
 import (
-       _ "encoding/json"
        "fmt"
        "io"
        "io/ioutil"

diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 5f5f69a118d526ced6a2546079f4adb2abf8b9c2..843a8e89d3a1f13a08d0c517902621df14fb0eac 100644 (file)
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -27,7 +27,7 @@ func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
                w := "/c=" + newCollection.UUID + "/"
                pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
                return r, w, pdh
-       })
+       }, nil)
 }
 
 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
@@ -36,19 +36,23 @@ func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
                w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
                pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
                return r, w, pdh
-       })
+       }, nil)
 }
 
 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
+       rpath := "/users/active/foo_file_in_dir/"
        s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
-               r := "/users/active/foo_file_in_dir/"
-               w := "/users/active/" + newCollection.Name
+               wpath := "/users/active/" + newCollection.Name
                pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
-               return r, w, pdh
+               return rpath, wpath, pdh
+       }, func(path string) bool {
+               // Skip tests that rely on writes, because /users/
+               // tree is read-only.
+               return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
        })
 }
 
-func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string)) {
+func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
        testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
 
        tempdir, err := ioutil.TempDir("", "keep-web-test-")
@@ -239,6 +243,10 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
                },
        } {
                c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+               if skip != nil && skip(trial.path) {
+                       c.Log("(skip)")
+                       continue
+               }
 
                os.Remove(checkfile.Name())
 

diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 5ab4f70cdea33405b74d4f8762dc9c6d7e3aa725..00af0f4eab86d3614170ef1e9f3af087ac333483 100644 (file)
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -226,13 +226,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
        }
 
-       arv := h.clientPool.Get()
-       if arv == nil {
-               statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
-               return
-       }
-       defer h.clientPool.Put(arv)
-
        pathParts := strings.Split(r.URL.Path[1:], "/")
 
        var stripParts int
@@ -241,6 +234,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        var reqTokens []string
        var pathToken bool
        var attachment bool
+       var useSiteFS bool
        credentialsOK := h.Config.TrustAllContent
 
        if r.Host != "" && r.Host == h.Config.AttachmentOnlyHost {
@@ -256,6 +250,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        } else if r.URL.Path == "/status.json" {
                h.serveStatus(w, r)
                return
+       } else if len(pathParts) >= 1 && pathParts[0] == "users" {
+               useSiteFS = true
        } else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
                // /c=ID[/PATH...]
                collectionID = parseCollectionIDFromURL(pathParts[0][2:])
@@ -275,6 +271,16 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                }
        }
 
+       if collectionID == "" && !useSiteFS {
+               statusCode = http.StatusNotFound
+               return
+       }
+
+       forceReload := false
+       if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
+               forceReload = true
+       }
+
        formToken := r.FormValue("api_token")
        if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
                // The client provided an explicit token in the POST
@@ -322,6 +328,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                tokens = append(reqTokens, h.Config.AnonymousTokens...)
        }
 
+       if useSiteFS {
+               h.serveSiteFS(w, r, tokens)
+               return
+       }
+
        if len(targetPath) > 0 && targetPath[0] == "_" {
                // If a collection has a directory called "t=foo" or
                // "_", it can be served at
@@ -333,10 +344,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                stripParts++
        }
 
-       forceReload := false
-       if cc := r.Header.Get("Cache-Control"); strings.Contains(cc, "no-cache") || strings.Contains(cc, "must-revalidate") {
-               forceReload = true
+       arv := h.clientPool.Get()
+       if arv == nil {
+               statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+h.clientPool.Err().Error()
+               return
        }
+       defer h.clientPool.Put(arv)
 
        var collection *arvados.Collection
        tokenResult := make(map[string]int)
@@ -410,12 +423,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                Insecure:  arv.ApiInsecure,
        }
 
-       var fs arvados.FileSystem
-       if collectionID == "" {
-               fs = client.SiteFileSystem(kc)
-       } else {
-               fs, err = collection.FileSystem(client, kc)
-       }
+       fs, err := collection.FileSystem(client, kc)
        if err != nil {
                statusCode, statusText = http.StatusInternalServerError, err.Error()
                return
@@ -475,7 +483,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
                // "dirname/fnm".
                h.seeOtherWithCookie(w, r, r.URL.Path+"/", credentialsOK)
        } else if stat.IsDir() {
-               h.serveDirectory(w, r, collection.Name, fs, openPath, stripParts)
+               h.serveDirectory(w, r, collection.Name, fs, openPath, true)
        } else {
                http.ServeContent(w, r, basename, stat.ModTime(), f)
                if r.Header.Get("Range") == "" && int64(w.WroteBodyBytes()) != stat.Size() {
@@ -491,10 +499,69 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
        }
 }
 
+func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string) {
+       if len(tokens) == 0 {
+               w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
+               http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+               return
+       }
+       if writeMethod[r.Method] {
+               http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
+               return
+       }
+       arv := h.clientPool.Get()
+       if arv == nil {
+               http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+               return
+       }
+       defer h.clientPool.Put(arv)
+       arv.ApiToken = tokens[0]
+
+       kc, err := keepclient.MakeKeepClient(arv)
+       if err != nil {
+               http.Error(w, err.Error(), http.StatusInternalServerError)
+               return
+       }
+       client := &arvados.Client{
+               APIHost:   arv.ApiServer,
+               AuthToken: arv.ApiToken,
+               Insecure:  arv.ApiInsecure,
+       }
+       fs := client.SiteFileSystem(kc)
+       if f, err := fs.Open(r.URL.Path); os.IsNotExist(err) {
+               http.Error(w, err.Error(), http.StatusNotFound)
+               return
+       } else if err != nil {
+               http.Error(w, err.Error(), http.StatusInternalServerError)
+               return
+       } else if fi, err := f.Stat(); err == nil && fi.IsDir() && r.Method == "GET" {
+
+               h.serveDirectory(w, r, fi.Name(), fs, r.URL.Path, false)
+               return
+       } else {
+               f.Close()
+       }
+       wh := webdav.Handler{
+               Prefix: "/",
+               FileSystem: &webdavFS{
+                       collfs:        fs,
+                       writing:       writeMethod[r.Method],
+                       alwaysReadEOF: r.Method == "PROPFIND",
+               },
+               LockSystem: h.webdavLS,
+               Logger: func(_ *http.Request, err error) {
+                       if err != nil {
+                               log.Printf("error from webdav handler: %q", err)
+                       }
+               },
+       }
+       wh.ServeHTTP(w, r)
+}
+
 var dirListingTemplate = `<!DOCTYPE HTML>
 <HTML><HEAD>
   <META name="robots" content="NOINDEX">
-  <TITLE>{{ .Collection.Name }}</TITLE>
+  <TITLE>{{ .CollectionName }}</TITLE>
   <STYLE type="text/css">
     body {
       margin: 1.5em;
@@ -518,19 +585,26 @@ var dirListingTemplate = `<!DOCTYPE HTML>
   </STYLE>
 </HEAD>
 <BODY>
+
 <H1>{{ .CollectionName }}</H1>
 
 <P>This collection of data files is being shared with you through
 Arvados.  You can download individual files listed below.  To download
-the entire collection with wget, try:</P>
+the entire directory tree with wget, try:</P>
 
-<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL }}</PRE>
+<PRE>$ wget --mirror --no-parent --no-host --cut-dirs={{ .StripParts }} https://{{ .Request.Host }}{{ .Request.URL.Path }}</PRE>
 
 <H2>File Listing</H2>
 
 {{if .Files}}
 <UL>
-{{range .Files}}  <LI>{{.Size | printf "%15d  " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>{{end}}
+{{range .Files}}
+{{if .IsDir }}
+  <LI>{{" " | printf "%15s  " | nbsp}}<A href="{{.Name}}/">{{.Name}}/</A></LI>
+{{else}}
+  <LI>{{.Size | printf "%15d  " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>
+{{end}}
+{{end}}
 </UL>
 {{else}}
 <P>(No files; this collection is empty.)</P>
@@ -550,11 +624,12 @@ the entire collection with wget, try:</P>
 `
 
 type fileListEnt struct {
-       Name string
-       Size int64
+       Name  string
+       Size  int64
+       IsDir bool
 }
 
-func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, stripParts int) {
+func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collectionName string, fs http.FileSystem, base string, recurse bool) {
        var files []fileListEnt
        var walk func(string) error
        if !strings.HasSuffix(base, "/") {
@@ -574,15 +649,16 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
                        return err
                }
                for _, ent := range ents {
-                       if ent.IsDir() {
+                       if recurse && ent.IsDir() {
                                err = walk(path + ent.Name() + "/")
                                if err != nil {
                                        return err
                                }
                        } else {
                                files = append(files, fileListEnt{
-                                       Name: path + ent.Name(),
-                                       Size: ent.Size(),
+                                       Name:  path + ent.Name(),
+                                       Size:  ent.Size(),
+                                       IsDir: ent.IsDir(),
                                })
                        }
                }
@@ -611,7 +687,7 @@ func (h *handler) serveDirectory(w http.ResponseWriter, r *http.Request, collect
                "CollectionName": collectionName,
                "Files":          files,
                "Request":        r,
-               "StripParts":     stripParts,
+               "StripParts":     strings.Count(strings.TrimRight(r.URL.Path, "/"), "/"),
        })
 }
 

diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 3e7ae5fbd8728a3cb32ee9bfa4f9740ac741b8b1..7fed6fbd628f42b76210bd44f28608b56b7e9c6b 100644 (file)
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -508,7 +508,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
                        header:  authHeader,
                        expect:  []string{"foo", "bar"},
-                       cutDirs: 0,
+                       cutDirs: 1,
                },
                {
                        uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
@@ -517,11 +517,29 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        cutDirs: 2,
                },
                {
-                       uri:     "download.example.com/users/active/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+                       uri:     "download.example.com/users/active/foo_file_in_dir/",
                        header:  authHeader,
-                       expect:  []string{"dir1/foo", "dir1/bar"},
+                       expect:  []string{"dir1/"},
                        cutDirs: 3,
                },
+               {
+                       uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
+                       header:  authHeader,
+                       expect:  []string{"bar"},
+                       cutDirs: 4,
+               },
+               {
+                       uri:     "download.example.com/users/",
+                       header:  authHeader,
+                       expect:  []string{"active/"},
+                       cutDirs: 1,
+               },
+               {
+                       uri:     "download.example.com/users/active/",
+                       header:  authHeader,
+                       expect:  []string{"foo_file_in_dir/"},
+                       cutDirs: 2,
+               },
                {
                        uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
                        header:  nil,
@@ -550,19 +568,19 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
                        header:  authHeader,
                        expect:  []string{"foo", "bar"},
-                       cutDirs: 1,
+                       cutDirs: 2,
                },
                {
                        uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
                        header:  authHeader,
                        expect:  []string{"foo", "bar"},
-                       cutDirs: 2,
+                       cutDirs: 3,
                },
                {
                        uri:     arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
                        header:  authHeader,
                        expect:  []string{"foo", "bar"},
-                       cutDirs: 0,
+                       cutDirs: 1,
                },
                {
                        uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",

diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 941090a75d6f96c8bc0ad3bbc3ae81f572463894..3e62b19bbe296f35a514543121d3b29111133db1 100644 (file)
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -47,6 +47,9 @@ type webdavFS struct {
 }
 
 func (fs *webdavFS) makeparents(name string) {
+       if !fs.writing {
+               return
+       }
        dir, name := path.Split(name)
        if dir == "" || dir == "/" {
                return
@@ -66,7 +69,7 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
 }
 
 func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (f webdav.File, err error) {
-       writing := flag&(os.O_WRONLY|os.O_RDWR) != 0
+       writing := flag&(os.O_WRONLY|os.O_RDWR|os.O_TRUNC) != 0
        if writing {
                fs.makeparents(name)
        }
@@ -75,8 +78,13 @@ func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os
                // webdav module returns 404 on all OpenFile errors,
                // but returns 405 Method Not Allowed if OpenFile()
                // succeeds but Write() or Close() fails. We'd rather
-               // have 405.
-               f = writeFailer{File: f, err: errReadOnly}
+               // have 405. writeFailer ensures Close() fails if the
+               // file is opened for writing *or* Write() is called.
+               var err error
+               if writing {
+                       err = errReadOnly
+               }
+               f = writeFailer{File: f, err: err}
        }
        if fs.alwaysReadEOF {
                f = readEOF{File: f}
@@ -109,10 +117,15 @@ type writeFailer struct {
 }
 
 func (wf writeFailer) Write([]byte) (int, error) {
+       wf.err = errReadOnly
        return 0, wf.err
 }
 
 func (wf writeFailer) Close() error {
+       err := wf.File.Close()
+       if err != nil {
+               wf.err = err
+       }
        return wf.err
 }