"path/filepath"
"regexp"
"strings"
+ "sync"
"time"
"git.arvados.org/arvados.git/lib/config"
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
- c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
// Check preflight for a disallowed request
resp = httptest.NewRecorder()
c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
}
+func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
+ for _, trial := range []struct {
+ method string
+ path string
+ prefix string
+ source string
+ notFound bool
+ seeOther bool
+ }{
+ {
+ method: "PROPFIND",
+ path: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "/dir1",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix",
+ source: "",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/dir1/foo",
+ prefix: "/prefix/",
+ source: "/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix/foo",
+ prefix: "/prefix/",
+ source: "/dir1/",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix/",
+ prefix: "/prefix",
+ source: "/dir1",
+ },
+ {
+ method: "PROPFIND",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1/",
+ },
+ {
+ method: "GET",
+ path: "/prefix",
+ prefix: "/prefix",
+ source: "/dir1",
+ seeOther: true,
+ },
+ {
+ method: "PROPFIND",
+ path: "/dir1/foo",
+ prefix: "",
+ source: "/dir1",
+ notFound: true,
+ },
+ } {
+ c.Logf("trial %+v", trial)
+ u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
+ req := &http.Request{
+ Method: trial.method,
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveTokenV2},
+ "X-Webdav-Prefix": {trial.prefix},
+ "X-Webdav-Source": {trial.source},
+ },
+ Body: ioutil.NopCloser(bytes.NewReader(nil)),
+ }
+
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ if trial.notFound {
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ } else if trial.method == "PROPFIND" {
+ c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
+ } else if trial.seeOther {
+ c.Check(resp.Code, check.Equals, http.StatusSeeOther)
+ } else {
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ }
+ }
+}
+
func (s *UnitSuite) TestEmptyResponse(c *check.C) {
for _, trial := range []struct {
dataExists bool
}
}
+func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
+ u := mustParseURL(urlstring)
+ if hdr == nil && token != "" {
+ hdr = http.Header{"Authorization": {"Bearer " + token}}
+ } else if hdr == nil {
+ hdr = http.Header{}
+ } else if token != "" {
+ panic("must not pass both token and hdr")
+ }
+ return s.doReq(&http.Request{
+ Method: method,
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: hdr,
+ })
+}
+
func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
)
}
+func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
+ "",
+ nil,
+ "",
+ http.StatusOK,
+ "foo",
+ )
+ // Same valid sharing token, but requesting a different collection
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
+ "",
+ nil,
+ "",
+ http.StatusNotFound,
+ regexp.QuoteMeta(notFoundMessage+"\n"),
+ )
+}
+
// Bad token in URL is 404 Not Found because it doesn't make sense to
// retry the same URL with different authorization.
func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
nil,
"",
http.StatusNotFound,
- notFoundMessage+"\n",
+ regexp.QuoteMeta(notFoundMessage+"\n"),
)
}
http.Header{"Sec-Fetch-Mode": {"cors"}},
"",
http.StatusUnauthorized,
- unauthorizedMessage+"\n",
+ regexp.QuoteMeta(unauthorizedMessage+"\n"),
)
s.testVhostRedirectTokenToCookie(c, "GET",
s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
nil,
"",
http.StatusUnauthorized,
- unauthorizedMessage+"\n",
+ regexp.QuoteMeta(unauthorizedMessage+"\n"),
)
}
+func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token=thisisabogustoken",
+ http.Header{
+ "Sec-Fetch-Mode": {"navigate"},
+ "Cache-Control": {"no-cache"},
+ },
+ "",
+ http.StatusSeeOther,
+ "",
+ )
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %s", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+}
+
+func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
+ for _, trial := range []struct {
+ anonToken bool
+ cacheControl string
+ }{
+ {},
+ {cacheControl: "no-cache"},
+ {anonToken: true},
+ {anonToken: true, cacheControl: "no-cache"},
+ } {
+ c.Logf("trial: %+v", trial)
+
+ if trial.anonToken {
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ } else {
+ s.handler.Cluster.Users.AnonymousUserToken = ""
+ }
+ req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Sec-Fetch-Mode", "navigate")
+ if trial.cacheControl != "" {
+ req.Header.Set("Cache-Control", trial.cacheControl)
+ }
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusSeeOther)
+ u, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Logf("redirected to %q", u)
+ c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
+ c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
+ }
+}
+
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
nil,
"",
http.StatusBadRequest,
- "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
+ regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
)
}
nil,
"",
http.StatusBadRequest,
- "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
+ regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
)
resp := s.testVhostRedirectTokenToCookie(c, "GET",
http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- notFoundMessage+"\n",
+ regexp.QuoteMeta(notFoundMessage+"\n"),
)
}
"",
nil,
"",
- http.StatusNotFound,
- notFoundMessage+"\n",
+ http.StatusUnauthorized,
+ "Authorization tokens are not accepted here: .*\n",
)
}
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
-func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
if reqHeader == nil {
reqHeader = http.Header{}
}
resp := httptest.NewRecorder()
defer func() {
c.Check(resp.Code, check.Equals, expectStatus)
- c.Check(resp.Body.String(), check.Equals, expectRespBody)
+ c.Check(resp.Body.String(), check.Matches, matchRespBody)
}()
s.handler.ServeHTTP(resp, req)
c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
}
if trial.expect == nil {
- if s.handler.Cluster.Users.AnonymousUserToken == "" {
- c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
- } else {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
- }
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK, comment)
for _, e := range trial.expect {
resp = httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.handler.Cluster.Users.AnonymousUserToken == "" {
- c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
- } else {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
- }
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusOK, comment)
}
resp = httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.handler.Cluster.Users.AnonymousUserToken == "" {
- c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
- } else {
- c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
- }
+ c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
for _, e := range trial.expect {
}
func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
- successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
+ successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.AdminToken
c.Check(err, check.IsNil)
c.Check(logentries.Items, check.HasLen, 1)
lastLogId := logentries.Items[0].ID
+ c.Logf("lastLogId: %d", lastLogId)
var logbuf bytes.Buffer
logger := logrus.New()
deadline := time.Now().Add(time.Second)
for {
c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
+ logentries = arvados.LogList{}
err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
arvados.ResourceListParams{
Filters: []arvados.Filter{
logentries.Items[0].ID > lastLogId &&
logentries.Items[0].ObjectUUID == userUuid &&
logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
+ (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
logentries.Items[0].Properties["collection_file_path"] == filepath {
break
}
},
}
s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
- arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
+ arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
// Test user permission
req = &http.Request{
},
}
s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
- arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
+ arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
}
}
},
}
s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
- arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
+ arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
}
u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
},
}
s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
- arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
+ arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
}
func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
- arvadostest.AdminUserUUID, coll.UUID, "bar")
+ arvadostest.AdminUserUUID, coll.UUID, "", "bar")
// Test user permission
req = &http.Request{
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
- arvadostest.ActiveUserUUID, coll.UUID, "bar")
+ arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
+ }
+ }
+}
+
+func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
+ s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
+ lockTidyInterval = time.Second
+ client := arvados.NewClientFromEnv()
+ client.AuthToken = arvadostest.ActiveTokenV2
+ // Start small, and increase concurrency (2^2, 4^2, ...)
+ // only until hitting failure. Avoids unnecessarily long
+ // failure reports.
+ for n := 2; n < 16 && !c.Failed(); n = n * 2 {
+ c.Logf("%s: n=%d", c.TestName(), n)
+
+ var coll arvados.Collection
+ err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
+ c.Assert(err, check.IsNil)
+ defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
+
+ var wg sync.WaitGroup
+ for i := 0; i < n && !c.Failed(); i++ {
+ i := i
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("MKCOL", u.String(), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusCreated)
+ for j := 0; j < n && !c.Failed(); j++ {
+ j := j
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ content := fmt.Sprintf("i=%d/j=%d", i, j)
+ u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
+
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusCreated)
+
+ time.Sleep(time.Second)
+ resp = httptest.NewRecorder()
+ req, err = http.NewRequest("GET", u.String(), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, content)
+ }()
+ }
+ }()
+ }
+ wg.Wait()
+ for i := 0; i < n; i++ {
+ u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
+ resp := httptest.NewRecorder()
+ req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+client.AuthToken)
+ s.handler.ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
}
}
}
projects / arvados.git / blobdiff