1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/lib/config"
21 "git.arvados.org/arvados.git/sdk/go/arvados"
22 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
24 "git.arvados.org/arvados.git/sdk/go/auth"
25 "git.arvados.org/arvados.git/sdk/go/ctxlog"
26 "git.arvados.org/arvados.git/sdk/go/keepclient"
27 check "gopkg.in/check.v1"
30 var _ = check.Suite(&UnitSuite{})
32 type UnitSuite struct {
33 Config *arvados.Config
36 func (s *UnitSuite) SetUpTest(c *check.C) {
37 ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
39 cfg, err := ldr.Load()
40 c.Assert(err, check.IsNil)
44 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
45 h := handler{Config: newConfig(s.Config)}
46 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
51 RequestURI: u.RequestURI(),
53 "Origin": {"https://workbench.example"},
54 "Access-Control-Request-Method": {"POST"},
58 // Check preflight for an allowed request
59 resp := httptest.NewRecorder()
60 h.ServeHTTP(resp, req)
61 c.Check(resp.Code, check.Equals, http.StatusOK)
62 c.Check(resp.Body.String(), check.Equals, "")
63 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
64 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
65 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
67 // Check preflight for a disallowed request
68 resp = httptest.NewRecorder()
69 req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
70 h.ServeHTTP(resp, req)
71 c.Check(resp.Body.String(), check.Equals, "")
72 c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
75 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
76 bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
77 token := arvadostest.ActiveToken
78 for _, trial := range []string{
79 "http://keep-web/c=" + bogusID + "/foo",
80 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
81 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
82 "http://keep-web/collections/" + bogusID + "/foo",
83 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
84 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
87 u := mustParseURL(trial)
92 RequestURI: u.RequestURI(),
94 resp := httptest.NewRecorder()
95 cfg := newConfig(s.Config)
96 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
97 h := handler{Config: cfg}
98 h.ServeHTTP(resp, req)
99 c.Check(resp.Code, check.Equals, http.StatusNotFound)
103 func mustParseURL(s string) *url.URL {
104 r, err := url.Parse(s)
106 panic("parse URL: " + s)
111 func (s *IntegrationSuite) TestVhost404(c *check.C) {
112 for _, testURL := range []string{
113 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
114 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
116 resp := httptest.NewRecorder()
117 u := mustParseURL(testURL)
118 req := &http.Request{
121 RequestURI: u.RequestURI(),
123 s.testServer.Handler.ServeHTTP(resp, req)
124 c.Check(resp.Code, check.Equals, http.StatusNotFound)
125 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
129 // An authorizer modifies an HTTP request to make use of the given
130 // token -- by adding it to a header, cookie, query param, or whatever
131 // -- and returns the HTTP status code we should expect from keep-web if
132 // the token is invalid.
133 type authorizer func(*http.Request, string) int
135 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
136 s.doVhostRequests(c, authzViaAuthzHeader)
138 func authzViaAuthzHeader(r *http.Request, tok string) int {
139 r.Header.Add("Authorization", "OAuth2 "+tok)
140 return http.StatusUnauthorized
143 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
144 s.doVhostRequests(c, authzViaCookieValue)
146 func authzViaCookieValue(r *http.Request, tok string) int {
147 r.AddCookie(&http.Cookie{
148 Name: "arvados_api_token",
149 Value: auth.EncodeTokenCookie([]byte(tok)),
151 return http.StatusUnauthorized
154 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
155 s.doVhostRequests(c, authzViaPath)
157 func authzViaPath(r *http.Request, tok string) int {
158 r.URL.Path = "/t=" + tok + r.URL.Path
159 return http.StatusNotFound
162 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
163 s.doVhostRequests(c, authzViaQueryString)
165 func authzViaQueryString(r *http.Request, tok string) int {
166 r.URL.RawQuery = "api_token=" + tok
167 return http.StatusUnauthorized
170 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
171 s.doVhostRequests(c, authzViaPOST)
173 func authzViaPOST(r *http.Request, tok string) int {
175 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
176 r.Body = ioutil.NopCloser(strings.NewReader(
177 url.Values{"api_token": {tok}}.Encode()))
178 return http.StatusUnauthorized
181 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
182 s.doVhostRequests(c, authzViaPOST)
184 func authzViaXHRPOST(r *http.Request, tok string) int {
186 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
187 r.Header.Add("Origin", "https://origin.example")
188 r.Body = ioutil.NopCloser(strings.NewReader(
191 "disposition": {"attachment"},
193 return http.StatusUnauthorized
196 // Try some combinations of {url, token} using the given authorization
197 // mechanism, and verify the result is correct.
198 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
199 for _, hostPath := range []string{
200 arvadostest.FooCollection + ".example.com/foo",
201 arvadostest.FooCollection + "--collections.example.com/foo",
202 arvadostest.FooCollection + "--collections.example.com/_/foo",
203 arvadostest.FooCollectionPDH + ".example.com/foo",
204 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
205 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
207 c.Log("doRequests: ", hostPath)
208 s.doVhostRequestsWithHostPath(c, authz, hostPath)
212 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
213 for _, tok := range []string{
214 arvadostest.ActiveToken,
215 arvadostest.ActiveToken[:15],
216 arvadostest.SpectatorToken,
220 u := mustParseURL("http://" + hostPath)
221 req := &http.Request{
225 RequestURI: u.RequestURI(),
226 Header: http.Header{},
228 failCode := authz(req, tok)
229 req, resp := s.doReq(req)
230 code, body := resp.Code, resp.Body.String()
232 // If the initial request had a (non-empty) token
233 // showing in the query string, we should have been
234 // redirected in order to hide it in a cookie.
235 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
237 if tok == arvadostest.ActiveToken {
238 c.Check(code, check.Equals, http.StatusOK)
239 c.Check(body, check.Equals, "foo")
242 c.Check(code >= 400, check.Equals, true)
243 c.Check(code < 500, check.Equals, true)
244 if tok == arvadostest.SpectatorToken {
245 // Valid token never offers to retry
246 // with different credentials.
247 c.Check(code, check.Equals, http.StatusNotFound)
249 // Invalid token can ask to retry
250 // depending on the authz method.
251 c.Check(code, check.Equals, failCode)
254 c.Check(body, check.Equals, notFoundMessage+"\n")
256 c.Check(body, check.Equals, unauthorizedMessage+"\n")
262 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
263 resp := httptest.NewRecorder()
264 s.testServer.Handler.ServeHTTP(resp, req)
265 if resp.Code != http.StatusSeeOther {
268 cookies := (&http.Response{Header: resp.Header()}).Cookies()
269 u, _ := req.URL.Parse(resp.Header().Get("Location"))
274 RequestURI: u.RequestURI(),
275 Header: http.Header{},
277 for _, c := range cookies {
283 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
284 s.testVhostRedirectTokenToCookie(c, "GET",
285 arvadostest.FooCollection+".example.com/foo",
286 "?api_token="+arvadostest.ActiveToken,
294 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
295 s.testVhostRedirectTokenToCookie(c, "GET",
296 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
305 // Bad token in URL is 404 Not Found because it doesn't make sense to
306 // retry the same URL with different authorization.
307 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
308 s.testVhostRedirectTokenToCookie(c, "GET",
309 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
314 notFoundMessage+"\n",
318 // Bad token in a cookie (even if it got there via our own
319 // query-string-to-cookie redirect) is, in principle, retryable at the
320 // same URL so it's 401 Unauthorized.
321 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
322 s.testVhostRedirectTokenToCookie(c, "GET",
323 arvadostest.FooCollection+".example.com/foo",
324 "?api_token=thisisabogustoken",
327 http.StatusUnauthorized,
328 unauthorizedMessage+"\n",
332 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
333 s.testVhostRedirectTokenToCookie(c, "GET",
334 "example.com/c="+arvadostest.FooCollection+"/foo",
335 "?api_token="+arvadostest.ActiveToken,
338 http.StatusBadRequest,
339 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
343 // If client requests an attachment by putting ?disposition=attachment
344 // in the query string, and gets redirected, the redirect target
345 // should respond with an attachment.
346 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
347 resp := s.testVhostRedirectTokenToCookie(c, "GET",
348 arvadostest.FooCollection+".example.com/foo",
349 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
355 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
358 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
359 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
360 resp := s.testVhostRedirectTokenToCookie(c, "GET",
361 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
362 "?api_token="+arvadostest.ActiveToken,
368 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
371 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
372 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
373 resp := s.testVhostRedirectTokenToCookie(c, "GET",
374 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
375 "?api_token="+arvadostest.ActiveToken,
381 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
382 resp = s.testVhostRedirectTokenToCookie(c, "GET",
383 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
384 "?api_token="+arvadostest.ActiveToken,
390 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
393 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
394 s.testServer.Config.cluster.Collections.TrustAllContent = true
395 s.testVhostRedirectTokenToCookie(c, "GET",
396 "example.com/c="+arvadostest.FooCollection+"/foo",
397 "?api_token="+arvadostest.ActiveToken,
405 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
406 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
408 s.testVhostRedirectTokenToCookie(c, "GET",
409 "example.com/c="+arvadostest.FooCollection+"/foo",
410 "?api_token="+arvadostest.ActiveToken,
413 http.StatusBadRequest,
414 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
417 resp := s.testVhostRedirectTokenToCookie(c, "GET",
418 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
419 "?api_token="+arvadostest.ActiveToken,
425 c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
428 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
429 s.testVhostRedirectTokenToCookie(c, "POST",
430 arvadostest.FooCollection+".example.com/foo",
432 "application/x-www-form-urlencoded",
433 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
439 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
440 s.testVhostRedirectTokenToCookie(c, "POST",
441 arvadostest.FooCollection+".example.com/foo",
443 "application/x-www-form-urlencoded",
444 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
446 notFoundMessage+"\n",
450 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
451 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
452 s.testVhostRedirectTokenToCookie(c, "GET",
453 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
462 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
463 s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
464 s.testVhostRedirectTokenToCookie(c, "GET",
465 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
470 notFoundMessage+"\n",
474 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
475 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
477 client := s.testServer.Config.Client
478 client.AuthToken = arvadostest.ActiveToken
479 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
480 c.Assert(err, check.IsNil)
481 f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
482 c.Assert(err, check.IsNil)
484 mtxt, err := fs.MarshalManifest(".")
485 c.Assert(err, check.IsNil)
486 var coll arvados.Collection
487 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
488 "collection": map[string]string{
489 "manifest_text": mtxt,
492 c.Assert(err, check.IsNil)
494 u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
495 req := &http.Request{
499 RequestURI: u.RequestURI(),
501 "Authorization": {"Bearer " + client.AuthToken},
504 resp := httptest.NewRecorder()
505 s.testServer.Handler.ServeHTTP(resp, req)
506 c.Check(resp.Code, check.Equals, http.StatusOK)
507 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
510 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
511 arv := arvados.NewClientFromEnv()
512 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
513 s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
514 name := "foo/bar/baz"
515 nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
516 nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
518 client := s.testServer.Config.Client
519 client.AuthToken = arvadostest.ActiveToken
520 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
521 c.Assert(err, check.IsNil)
522 f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
523 c.Assert(err, check.IsNil)
525 mtxt, err := fs.MarshalManifest(".")
526 c.Assert(err, check.IsNil)
527 var coll arvados.Collection
528 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
529 "collection": map[string]string{
530 "manifest_text": mtxt,
532 "owner_uuid": arvadostest.AProjectUUID,
535 c.Assert(err, check.IsNil)
536 defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
538 base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
539 for tryURL, expectRegexp := range map[string]string{
540 base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
541 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
543 u, _ := url.Parse(tryURL)
544 req := &http.Request{
548 RequestURI: u.RequestURI(),
550 "Authorization": {"Bearer " + client.AuthToken},
553 resp := httptest.NewRecorder()
554 s.testServer.Handler.ServeHTTP(resp, req)
555 c.Check(resp.Code, check.Equals, http.StatusOK)
556 c.Check(resp.Body.String(), check.Matches, expectRegexp)
560 // XHRs can't follow redirect-with-cookie so they rely on method=POST
561 // and disposition=attachment (telling us it's acceptable to respond
562 // with content instead of a redirect) and an Origin header that gets
563 // added automatically by the browser (telling us it's desirable to do
565 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
566 u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
567 req := &http.Request{
571 RequestURI: u.RequestURI(),
573 "Origin": {"https://origin.example"},
574 "Content-Type": {"application/x-www-form-urlencoded"},
576 Body: ioutil.NopCloser(strings.NewReader(url.Values{
577 "api_token": {arvadostest.ActiveToken},
578 "disposition": {"attachment"},
581 resp := httptest.NewRecorder()
582 s.testServer.Handler.ServeHTTP(resp, req)
583 c.Check(resp.Code, check.Equals, http.StatusOK)
584 c.Check(resp.Body.String(), check.Equals, "foo")
585 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
588 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
589 u, _ := url.Parse(`http://` + hostPath + queryString)
590 req := &http.Request{
594 RequestURI: u.RequestURI(),
595 Header: http.Header{"Content-Type": {contentType}},
596 Body: ioutil.NopCloser(strings.NewReader(reqBody)),
599 resp := httptest.NewRecorder()
601 c.Check(resp.Code, check.Equals, expectStatus)
602 c.Check(resp.Body.String(), check.Equals, expectRespBody)
605 s.testServer.Handler.ServeHTTP(resp, req)
606 if resp.Code != http.StatusSeeOther {
609 c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
610 cookies := (&http.Response{Header: resp.Header()}).Cookies()
612 u, _ = u.Parse(resp.Header().Get("Location"))
617 RequestURI: u.RequestURI(),
618 Header: http.Header{},
620 for _, c := range cookies {
624 resp = httptest.NewRecorder()
625 s.testServer.Handler.ServeHTTP(resp, req)
626 c.Check(resp.Header().Get("Location"), check.Equals, "")
630 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
631 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
632 s.testDirectoryListing(c)
635 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
636 s.testServer.Config.cluster.Users.AnonymousUserToken = ""
637 s.testDirectoryListing(c)
640 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
641 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
642 authHeader := http.Header{
643 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
645 for _, trial := range []struct {
653 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
655 expect: []string{"dir1/foo", "dir1/bar"},
659 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
661 expect: []string{"foo", "bar"},
665 // URLs of this form ignore authHeader, and
666 // FooAndBarFilesInDirUUID isn't public, so
668 uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
673 uri: "download.example.com/users/active/foo_file_in_dir/",
675 expect: []string{"dir1/"},
679 uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
681 expect: []string{"bar"},
685 uri: "download.example.com/",
687 expect: []string{"users/"},
691 uri: "download.example.com/users",
694 expect: []string{"active/"},
698 uri: "download.example.com/users/",
700 expect: []string{"active/"},
704 uri: "download.example.com/users/active",
706 redirect: "/users/active/",
707 expect: []string{"foo_file_in_dir/"},
711 uri: "download.example.com/users/active/",
713 expect: []string{"foo_file_in_dir/"},
717 uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
719 expect: []string{"dir1/foo", "dir1/bar"},
723 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
725 expect: []string{"dir1/foo", "dir1/bar"},
729 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
731 expect: []string{"dir1/foo", "dir1/bar"},
735 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
737 expect: []string{"dir1/foo", "dir1/bar"},
741 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
743 redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
744 expect: []string{"foo", "bar"},
748 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
750 expect: []string{"foo", "bar"},
754 uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
757 expect: []string{"foo", "bar"},
761 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
766 uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
768 expect: []string{"waz"},
772 uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
774 expect: []string{"waz"},
778 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
779 resp := httptest.NewRecorder()
780 u := mustParseURL("//" + trial.uri)
781 req := &http.Request{
785 RequestURI: u.RequestURI(),
786 Header: copyHeader(trial.header),
788 s.testServer.Handler.ServeHTTP(resp, req)
789 var cookies []*http.Cookie
790 for resp.Code == http.StatusSeeOther {
791 u, _ := req.URL.Parse(resp.Header().Get("Location"))
796 RequestURI: u.RequestURI(),
797 Header: copyHeader(trial.header),
799 cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
800 for _, c := range cookies {
803 resp = httptest.NewRecorder()
804 s.testServer.Handler.ServeHTTP(resp, req)
806 if trial.redirect != "" {
807 c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
809 if trial.expect == nil {
810 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
812 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
813 for _, e := range trial.expect {
814 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
816 c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
819 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
824 RequestURI: u.RequestURI(),
825 Header: copyHeader(trial.header),
826 Body: ioutil.NopCloser(&bytes.Buffer{}),
828 resp = httptest.NewRecorder()
829 s.testServer.Handler.ServeHTTP(resp, req)
830 if trial.expect == nil {
831 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
833 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
840 RequestURI: u.RequestURI(),
841 Header: copyHeader(trial.header),
842 Body: ioutil.NopCloser(&bytes.Buffer{}),
844 resp = httptest.NewRecorder()
845 s.testServer.Handler.ServeHTTP(resp, req)
846 if trial.expect == nil {
847 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
849 c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
850 for _, e := range trial.expect {
851 if strings.HasSuffix(e, "/") {
852 e = filepath.Join(u.Path, e) + "/"
854 e = filepath.Join(u.Path, e)
856 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
862 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
863 arv := arvados.NewClientFromEnv()
864 var newCollection arvados.Collection
865 err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
866 "collection": map[string]string{
867 "owner_uuid": arvadostest.ActiveUserUUID,
868 "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
869 "name": "keep-web test collection",
871 "ensure_unique_name": true,
873 c.Assert(err, check.IsNil)
874 defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
876 var updated arvados.Collection
877 for _, fnm := range []string{"foo.txt", "bar.txt"} {
878 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
879 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
880 req := &http.Request{
884 RequestURI: u.RequestURI(),
886 "Authorization": {"Bearer " + arvadostest.ActiveToken},
889 resp := httptest.NewRecorder()
890 s.testServer.Handler.ServeHTTP(resp, req)
891 c.Check(resp.Code, check.Equals, http.StatusNoContent)
893 updated = arvados.Collection{}
894 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
895 c.Check(err, check.IsNil)
896 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
897 c.Logf("updated manifest_text %q", updated.ManifestText)
899 c.Check(updated.ManifestText, check.Equals, "")
902 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
903 s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
904 authHeader := http.Header{
905 "Authorization": {"Bearer " + arvadostest.ManagementToken},
908 resp := httptest.NewRecorder()
909 u := mustParseURL("http://download.example.com/_health/ping")
910 req := &http.Request{
914 RequestURI: u.RequestURI(),
917 s.testServer.Handler.ServeHTTP(resp, req)
919 c.Check(resp.Code, check.Equals, http.StatusOK)
920 c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
923 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
924 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
926 client := s.testServer.Config.Client
927 client.AuthToken = arvadostest.ActiveToken
928 arv, err := arvadosclient.New(&client)
929 c.Assert(err, check.Equals, nil)
930 kc, err := keepclient.MakeKeepClient(arv)
931 c.Assert(err, check.Equals, nil)
933 fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
934 c.Assert(err, check.IsNil)
941 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
942 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
943 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
944 {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
945 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
947 for _, trial := range trials {
948 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
949 c.Assert(err, check.IsNil)
950 _, err = f.Write([]byte(trial.content))
951 c.Assert(err, check.IsNil)
952 c.Assert(f.Close(), check.IsNil)
954 mtxt, err := fs.MarshalManifest(".")
955 c.Assert(err, check.IsNil)
956 var coll arvados.Collection
957 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
958 "collection": map[string]string{
959 "manifest_text": mtxt,
962 c.Assert(err, check.IsNil)
964 for _, trial := range trials {
965 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
966 req := &http.Request{
970 RequestURI: u.RequestURI(),
972 "Authorization": {"Bearer " + client.AuthToken},
975 resp := httptest.NewRecorder()
976 s.testServer.Handler.ServeHTTP(resp, req)
977 c.Check(resp.Code, check.Equals, http.StatusOK)
978 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
979 c.Check(resp.Body.String(), check.Equals, trial.content)
983 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
984 s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
985 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
986 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
987 req := &http.Request{
991 RequestURI: u.RequestURI(),
993 resp := httptest.NewRecorder()
994 s.testServer.Handler.ServeHTTP(resp, req)
995 c.Check(resp.Code, check.Equals, http.StatusOK)
996 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
999 func copyHeader(h http.Header) http.Header {
1001 for k, v := range h {
1002 hc[k] = append([]string(nil), v...)