+
+func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
+ s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
+ s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+ s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ authHeader := http.Header{
+ "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
+ }
+ for _, trial := range []struct {
+ uri string
+ header http.Header
+ expect []string
+ redirect string
+ cutDirs int
+ }{
+ {
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 0,
+ },
+ {
+ uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
+ },
+ {
+ // URLs of this form ignore authHeader, and
+ // FooAndBarFilesInDirUUID isn't public, so
+ // this returns 404.
+ uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+ header: authHeader,
+ expect: nil,
+ },
+ {
+ uri: "download.example.com/users/active/foo_file_in_dir/",
+ header: authHeader,
+ expect: []string{"dir1/"},
+ cutDirs: 3,
+ },
+ {
+ uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
+ header: authHeader,
+ expect: []string{"bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "download.example.com/",
+ header: authHeader,
+ expect: []string{"users/"},
+ cutDirs: 0,
+ },
+ {
+ uri: "download.example.com/users",
+ header: authHeader,
+ redirect: "/users/",
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/",
+ header: authHeader,
+ expect: []string{"active/"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/users/active",
+ header: authHeader,
+ redirect: "/users/active/",
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
+ uri: "download.example.com/users/active/",
+ header: authHeader,
+ expect: []string{"foo_file_in_dir/"},
+ cutDirs: 2,
+ },
+ {
+ uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 4,
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 2,
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
+ header: nil,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 2,
+ },
+ {
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
+ header: authHeader,
+ expect: []string{"dir1/foo", "dir1/bar"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
+ header: authHeader,
+ redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 2,
+ },
+ {
+ uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
+ header: authHeader,
+ expect: []string{"foo", "bar"},
+ cutDirs: 3,
+ },
+ {
+ uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
+ header: authHeader,
+ redirect: "/dir1/",
+ expect: []string{"foo", "bar"},
+ cutDirs: 1,
+ },
+ {
+ uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
+ header: authHeader,
+ expect: nil,
+ },
+ {
+ uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
+ header: authHeader,
+ expect: []string{"waz"},
+ cutDirs: 1,
+ },
+ {
+ uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
+ header: authHeader,
+ expect: []string{"waz"},
+ cutDirs: 2,
+ },
+ } {
+ comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
+ resp := httptest.NewRecorder()
+ u := mustParseURL("//" + trial.uri)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: copyHeader(trial.header),
+ }
+ s.testServer.Handler.ServeHTTP(resp, req)
+ var cookies []*http.Cookie
+ for resp.Code == http.StatusSeeOther {
+ u, _ := req.URL.Parse(resp.Header().Get("Location"))
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: copyHeader(trial.header),
+ }
+ cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
+ for _, c := range cookies {
+ req.AddCookie(c)
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ }
+ if trial.redirect != "" {
+ c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
+ }
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK, comment)
+ for _, e := range trial.expect {
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
+ }
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
+ }
+
+ comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
+ req = &http.Request{
+ Method: "OPTIONS",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: copyHeader(trial.header),
+ Body: ioutil.NopCloser(&bytes.Buffer{}),
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK, comment)
+ }
+
+ req = &http.Request{
+ Method: "PROPFIND",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: copyHeader(trial.header),
+ Body: ioutil.NopCloser(&bytes.Buffer{}),
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ if trial.expect == nil {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
+ for _, e := range trial.expect {
+ if strings.HasSuffix(e, "/") {
+ e = filepath.Join(u.Path, e) + "/"
+ } else {
+ e = filepath.Join(u.Path, e)
+ }
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
+ }
+ }
+ }
+}
+
+func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
+ arv := arvados.NewClientFromEnv()
+ var newCollection arvados.Collection
+ err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "owner_uuid": arvadostest.ActiveUserUUID,
+ "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
+ "name": "keep-web test collection",
+ },
+ "ensure_unique_name": true,
+ })
+ c.Assert(err, check.IsNil)
+ defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
+
+ var updated arvados.Collection
+ for _, fnm := range []string{"foo.txt", "bar.txt"} {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
+ u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
+ req := &http.Request{
+ Method: "DELETE",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusNoContent)
+
+ updated = arvados.Collection{}
+ err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
+ c.Check(err, check.IsNil)
+ c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
+ c.Logf("updated manifest_text %q", updated.ManifestText)
+ }
+ c.Check(updated.ManifestText, check.Equals, "")
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+ s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
+ authHeader := http.Header{
+ "Authorization": {"Bearer " + arvadostest.ManagementToken},
+ }
+
+ resp := httptest.NewRecorder()
+ u := mustParseURL("http://download.example.com/_health/ping")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: authHeader,
+ }
+ s.testServer.Handler.ServeHTTP(resp, req)
+
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
+}
+
+func (s *IntegrationSuite) TestFileContentType(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ arv, err := arvadosclient.New(&client)
+ c.Assert(err, check.Equals, nil)
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+ c.Assert(err, check.IsNil)
+
+ trials := []struct {
+ filename string
+ content string
+ contentType string
+ }{
+ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
+ {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
+ {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
+ {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
+ {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
+ }
+ for _, trial := range trials {
+ f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte(trial.content))
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ for _, trial := range trials {
+ u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
+ c.Check(resp.Body.String(), check.Equals, trial.content)
+ }
+}
+
+func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
+ s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
+}
+
+func copyHeader(h http.Header) http.Header {
+ hc := http.Header{}
+ for k, v := range h {
+ hc[k] = append([]string(nil), v...)
+ }
+ return hc
+}