1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/lib/config"
21 "git.arvados.org/arvados.git/sdk/go/arvados"
22 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
24 "git.arvados.org/arvados.git/sdk/go/auth"
25 "git.arvados.org/arvados.git/sdk/go/ctxlog"
26 "git.arvados.org/arvados.git/sdk/go/keepclient"
27 check "gopkg.in/check.v1"
30 var _ = check.Suite(&UnitSuite{})
32 type UnitSuite struct {
33 Config *arvados.Config
36 func (s *UnitSuite) SetUpTest(c *check.C) {
37 ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
39 cfg, err := ldr.Load()
40 c.Assert(err, check.IsNil)
44 func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
45 cfg := newConfig(s.Config)
46 cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
47 h := handler{Config: cfg}
48 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
49 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
54 RequestURI: u.RequestURI(),
56 resp := httptest.NewRecorder()
57 h.ServeHTTP(resp, req)
58 c.Check(resp.Code, check.Equals, http.StatusOK)
59 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
62 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
63 h := handler{Config: newConfig(s.Config)}
64 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
69 RequestURI: u.RequestURI(),
71 "Origin": {"https://workbench.example"},
72 "Access-Control-Request-Method": {"POST"},
76 // Check preflight for an allowed request
77 resp := httptest.NewRecorder()
78 h.ServeHTTP(resp, req)
79 c.Check(resp.Code, check.Equals, http.StatusOK)
80 c.Check(resp.Body.String(), check.Equals, "")
81 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
82 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
83 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
85 // Check preflight for a disallowed request
86 resp = httptest.NewRecorder()
87 req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
88 h.ServeHTTP(resp, req)
89 c.Check(resp.Body.String(), check.Equals, "")
90 c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
93 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
94 bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
95 token := arvadostest.ActiveToken
96 for _, trial := range []string{
97 "http://keep-web/c=" + bogusID + "/foo",
98 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
99 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
100 "http://keep-web/collections/" + bogusID + "/foo",
101 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
102 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
105 u := mustParseURL(trial)
106 req := &http.Request{
110 RequestURI: u.RequestURI(),
112 resp := httptest.NewRecorder()
113 cfg := newConfig(s.Config)
114 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
115 h := handler{Config: cfg}
116 h.ServeHTTP(resp, req)
117 c.Check(resp.Code, check.Equals, http.StatusNotFound)
121 func mustParseURL(s string) *url.URL {
122 r, err := url.Parse(s)
124 panic("parse URL: " + s)
129 func (s *IntegrationSuite) TestVhost404(c *check.C) {
130 for _, testURL := range []string{
131 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
132 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
134 resp := httptest.NewRecorder()
135 u := mustParseURL(testURL)
136 req := &http.Request{
139 RequestURI: u.RequestURI(),
141 s.testServer.Handler.ServeHTTP(resp, req)
142 c.Check(resp.Code, check.Equals, http.StatusNotFound)
143 c.Check(resp.Body.String(), check.Equals, "")
147 // An authorizer modifies an HTTP request to make use of the given
148 // token -- by adding it to a header, cookie, query param, or whatever
149 // -- and returns the HTTP status code we should expect from keep-web if
150 // the token is invalid.
151 type authorizer func(*http.Request, string) int
153 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
154 s.doVhostRequests(c, authzViaAuthzHeader)
156 func authzViaAuthzHeader(r *http.Request, tok string) int {
157 r.Header.Add("Authorization", "OAuth2 "+tok)
158 return http.StatusUnauthorized
161 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
162 s.doVhostRequests(c, authzViaCookieValue)
164 func authzViaCookieValue(r *http.Request, tok string) int {
165 r.AddCookie(&http.Cookie{
166 Name: "arvados_api_token",
167 Value: auth.EncodeTokenCookie([]byte(tok)),
169 return http.StatusUnauthorized
172 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
173 s.doVhostRequests(c, authzViaPath)
175 func authzViaPath(r *http.Request, tok string) int {
176 r.URL.Path = "/t=" + tok + r.URL.Path
177 return http.StatusNotFound
180 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
181 s.doVhostRequests(c, authzViaQueryString)
183 func authzViaQueryString(r *http.Request, tok string) int {
184 r.URL.RawQuery = "api_token=" + tok
185 return http.StatusUnauthorized
188 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
189 s.doVhostRequests(c, authzViaPOST)
191 func authzViaPOST(r *http.Request, tok string) int {
193 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
194 r.Body = ioutil.NopCloser(strings.NewReader(
195 url.Values{"api_token": {tok}}.Encode()))
196 return http.StatusUnauthorized
199 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
200 s.doVhostRequests(c, authzViaPOST)
202 func authzViaXHRPOST(r *http.Request, tok string) int {
204 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
205 r.Header.Add("Origin", "https://origin.example")
206 r.Body = ioutil.NopCloser(strings.NewReader(
209 "disposition": {"attachment"},
211 return http.StatusUnauthorized
214 // Try some combinations of {url, token} using the given authorization
215 // mechanism, and verify the result is correct.
216 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
217 for _, hostPath := range []string{
218 arvadostest.FooCollection + ".example.com/foo",
219 arvadostest.FooCollection + "--collections.example.com/foo",
220 arvadostest.FooCollection + "--collections.example.com/_/foo",
221 arvadostest.FooCollectionPDH + ".example.com/foo",
222 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
223 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
225 c.Log("doRequests: ", hostPath)
226 s.doVhostRequestsWithHostPath(c, authz, hostPath)
230 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
231 for _, tok := range []string{
232 arvadostest.ActiveToken,
233 arvadostest.ActiveToken[:15],
234 arvadostest.SpectatorToken,
238 u := mustParseURL("http://" + hostPath)
239 req := &http.Request{
243 RequestURI: u.RequestURI(),
244 Header: http.Header{},
246 failCode := authz(req, tok)
247 req, resp := s.doReq(req)
248 code, body := resp.Code, resp.Body.String()
250 // If the initial request had a (non-empty) token
251 // showing in the query string, we should have been
252 // redirected in order to hide it in a cookie.
253 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
255 if tok == arvadostest.ActiveToken {
256 c.Check(code, check.Equals, http.StatusOK)
257 c.Check(body, check.Equals, "foo")
260 c.Check(code >= 400, check.Equals, true)
261 c.Check(code < 500, check.Equals, true)
262 if tok == arvadostest.SpectatorToken {
263 // Valid token never offers to retry
264 // with different credentials.
265 c.Check(code, check.Equals, http.StatusNotFound)
267 // Invalid token can ask to retry
268 // depending on the authz method.
269 c.Check(code, check.Equals, failCode)
271 c.Check(body, check.Equals, "")
276 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
277 resp := httptest.NewRecorder()
278 s.testServer.Handler.ServeHTTP(resp, req)
279 if resp.Code != http.StatusSeeOther {
282 cookies := (&http.Response{Header: resp.Header()}).Cookies()
283 u, _ := req.URL.Parse(resp.Header().Get("Location"))
288 RequestURI: u.RequestURI(),
289 Header: http.Header{},
291 for _, c := range cookies {
297 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
298 s.testVhostRedirectTokenToCookie(c, "GET",
299 arvadostest.FooCollection+".example.com/foo",
300 "?api_token="+arvadostest.ActiveToken,
308 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
309 s.testVhostRedirectTokenToCookie(c, "GET",
310 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
319 // Bad token in URL is 404 Not Found because it doesn't make sense to
320 // retry the same URL with different authorization.
321 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
322 s.testVhostRedirectTokenToCookie(c, "GET",
323 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
332 // Bad token in a cookie (even if it got there via our own
333 // query-string-to-cookie redirect) is, in principle, retryable at the
334 // same URL so it's 401 Unauthorized.
335 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
336 s.testVhostRedirectTokenToCookie(c, "GET",
337 arvadostest.FooCollection+".example.com/foo",
338 "?api_token=thisisabogustoken",
341 http.StatusUnauthorized,
346 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
347 s.testVhostRedirectTokenToCookie(c, "GET",
348 "example.com/c="+arvadostest.FooCollection+"/foo",
349 "?api_token="+arvadostest.ActiveToken,
352 http.StatusBadRequest,
353 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
357 // If client requests an attachment by putting ?disposition=attachment
358 // in the query string, and gets redirected, the redirect target
359 // should respond with an attachment.
360 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
361 resp := s.testVhostRedirectTokenToCookie(c, "GET",
362 arvadostest.FooCollection+".example.com/foo",
363 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
369 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
372 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
373 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
374 resp := s.testVhostRedirectTokenToCookie(c, "GET",
375 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
376 "?api_token="+arvadostest.ActiveToken,
382 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
385 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
386 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
387 resp := s.testVhostRedirectTokenToCookie(c, "GET",
388 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
389 "?api_token="+arvadostest.ActiveToken,
395 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
396 resp = s.testVhostRedirectTokenToCookie(c, "GET",
397 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
398 "?api_token="+arvadostest.ActiveToken,
404 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
407 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
408 s.testServer.Config.cluster.Collections.TrustAllContent = true
409 s.testVhostRedirectTokenToCookie(c, "GET",
410 "example.com/c="+arvadostest.FooCollection+"/foo",
411 "?api_token="+arvadostest.ActiveToken,
419 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
420 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
422 s.testVhostRedirectTokenToCookie(c, "GET",
423 "example.com/c="+arvadostest.FooCollection+"/foo",
424 "?api_token="+arvadostest.ActiveToken,
427 http.StatusBadRequest,
428 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
431 resp := s.testVhostRedirectTokenToCookie(c, "GET",
432 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
433 "?api_token="+arvadostest.ActiveToken,
439 c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
442 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
443 s.testVhostRedirectTokenToCookie(c, "POST",
444 arvadostest.FooCollection+".example.com/foo",
446 "application/x-www-form-urlencoded",
447 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
453 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
454 s.testVhostRedirectTokenToCookie(c, "POST",
455 arvadostest.FooCollection+".example.com/foo",
457 "application/x-www-form-urlencoded",
458 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
464 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
465 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
466 s.testVhostRedirectTokenToCookie(c, "GET",
467 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
476 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
477 s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
478 s.testVhostRedirectTokenToCookie(c, "GET",
479 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
488 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
489 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
491 client := s.testServer.Config.Client
492 client.AuthToken = arvadostest.ActiveToken
493 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
494 c.Assert(err, check.IsNil)
495 f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
496 c.Assert(err, check.IsNil)
498 mtxt, err := fs.MarshalManifest(".")
499 c.Assert(err, check.IsNil)
500 var coll arvados.Collection
501 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
502 "collection": map[string]string{
503 "manifest_text": mtxt,
506 c.Assert(err, check.IsNil)
508 u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
509 req := &http.Request{
513 RequestURI: u.RequestURI(),
515 "Authorization": {"Bearer " + client.AuthToken},
518 resp := httptest.NewRecorder()
519 s.testServer.Handler.ServeHTTP(resp, req)
520 c.Check(resp.Code, check.Equals, http.StatusOK)
521 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
524 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
525 arv := arvados.NewClientFromEnv()
526 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
527 s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
528 name := "foo/bar/baz"
529 nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
530 nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
532 client := s.testServer.Config.Client
533 client.AuthToken = arvadostest.ActiveToken
534 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
535 c.Assert(err, check.IsNil)
536 f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
537 c.Assert(err, check.IsNil)
539 mtxt, err := fs.MarshalManifest(".")
540 c.Assert(err, check.IsNil)
541 var coll arvados.Collection
542 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
543 "collection": map[string]string{
544 "manifest_text": mtxt,
546 "owner_uuid": arvadostest.AProjectUUID,
549 c.Assert(err, check.IsNil)
550 defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
552 base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
553 for tryURL, expectRegexp := range map[string]string{
554 base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
555 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
557 u, _ := url.Parse(tryURL)
558 req := &http.Request{
562 RequestURI: u.RequestURI(),
564 "Authorization": {"Bearer " + client.AuthToken},
567 resp := httptest.NewRecorder()
568 s.testServer.Handler.ServeHTTP(resp, req)
569 c.Check(resp.Code, check.Equals, http.StatusOK)
570 c.Check(resp.Body.String(), check.Matches, expectRegexp)
574 // XHRs can't follow redirect-with-cookie so they rely on method=POST
575 // and disposition=attachment (telling us it's acceptable to respond
576 // with content instead of a redirect) and an Origin header that gets
577 // added automatically by the browser (telling us it's desirable to do
579 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
580 u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
581 req := &http.Request{
585 RequestURI: u.RequestURI(),
587 "Origin": {"https://origin.example"},
588 "Content-Type": {"application/x-www-form-urlencoded"},
590 Body: ioutil.NopCloser(strings.NewReader(url.Values{
591 "api_token": {arvadostest.ActiveToken},
592 "disposition": {"attachment"},
595 resp := httptest.NewRecorder()
596 s.testServer.Handler.ServeHTTP(resp, req)
597 c.Check(resp.Code, check.Equals, http.StatusOK)
598 c.Check(resp.Body.String(), check.Equals, "foo")
599 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
602 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
603 u, _ := url.Parse(`http://` + hostPath + queryString)
604 req := &http.Request{
608 RequestURI: u.RequestURI(),
609 Header: http.Header{"Content-Type": {contentType}},
610 Body: ioutil.NopCloser(strings.NewReader(reqBody)),
613 resp := httptest.NewRecorder()
615 c.Check(resp.Code, check.Equals, expectStatus)
616 c.Check(resp.Body.String(), check.Equals, expectRespBody)
619 s.testServer.Handler.ServeHTTP(resp, req)
620 if resp.Code != http.StatusSeeOther {
623 c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
624 cookies := (&http.Response{Header: resp.Header()}).Cookies()
626 u, _ = u.Parse(resp.Header().Get("Location"))
631 RequestURI: u.RequestURI(),
632 Header: http.Header{},
634 for _, c := range cookies {
638 resp = httptest.NewRecorder()
639 s.testServer.Handler.ServeHTTP(resp, req)
640 c.Check(resp.Header().Get("Location"), check.Equals, "")
644 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
645 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
646 s.testDirectoryListing(c)
649 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
650 s.testServer.Config.cluster.Users.AnonymousUserToken = ""
651 s.testDirectoryListing(c)
654 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
655 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
656 authHeader := http.Header{
657 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
659 for _, trial := range []struct {
667 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
669 expect: []string{"dir1/foo", "dir1/bar"},
673 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
675 expect: []string{"foo", "bar"},
679 // URLs of this form ignore authHeader, and
680 // FooAndBarFilesInDirUUID isn't public, so
682 uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
687 uri: "download.example.com/users/active/foo_file_in_dir/",
689 expect: []string{"dir1/"},
693 uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
695 expect: []string{"bar"},
699 uri: "download.example.com/",
701 expect: []string{"users/"},
705 uri: "download.example.com/users",
708 expect: []string{"active/"},
712 uri: "download.example.com/users/",
714 expect: []string{"active/"},
718 uri: "download.example.com/users/active",
720 redirect: "/users/active/",
721 expect: []string{"foo_file_in_dir/"},
725 uri: "download.example.com/users/active/",
727 expect: []string{"foo_file_in_dir/"},
731 uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
733 expect: []string{"dir1/foo", "dir1/bar"},
737 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
739 expect: []string{"dir1/foo", "dir1/bar"},
743 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
745 expect: []string{"dir1/foo", "dir1/bar"},
749 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
751 expect: []string{"dir1/foo", "dir1/bar"},
755 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
757 redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
758 expect: []string{"foo", "bar"},
762 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
764 expect: []string{"foo", "bar"},
768 uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
771 expect: []string{"foo", "bar"},
775 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
780 uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
782 expect: []string{"waz"},
786 uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
788 expect: []string{"waz"},
792 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
793 resp := httptest.NewRecorder()
794 u := mustParseURL("//" + trial.uri)
795 req := &http.Request{
799 RequestURI: u.RequestURI(),
800 Header: copyHeader(trial.header),
802 s.testServer.Handler.ServeHTTP(resp, req)
803 var cookies []*http.Cookie
804 for resp.Code == http.StatusSeeOther {
805 u, _ := req.URL.Parse(resp.Header().Get("Location"))
810 RequestURI: u.RequestURI(),
811 Header: copyHeader(trial.header),
813 cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
814 for _, c := range cookies {
817 resp = httptest.NewRecorder()
818 s.testServer.Handler.ServeHTTP(resp, req)
820 if trial.redirect != "" {
821 c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
823 if trial.expect == nil {
824 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
826 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
827 for _, e := range trial.expect {
828 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
830 c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
833 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
838 RequestURI: u.RequestURI(),
839 Header: copyHeader(trial.header),
840 Body: ioutil.NopCloser(&bytes.Buffer{}),
842 resp = httptest.NewRecorder()
843 s.testServer.Handler.ServeHTTP(resp, req)
844 if trial.expect == nil {
845 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
847 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
854 RequestURI: u.RequestURI(),
855 Header: copyHeader(trial.header),
856 Body: ioutil.NopCloser(&bytes.Buffer{}),
858 resp = httptest.NewRecorder()
859 s.testServer.Handler.ServeHTTP(resp, req)
860 if trial.expect == nil {
861 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
863 c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
864 for _, e := range trial.expect {
865 if strings.HasSuffix(e, "/") {
866 e = filepath.Join(u.Path, e) + "/"
868 e = filepath.Join(u.Path, e)
870 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
876 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
877 arv := arvados.NewClientFromEnv()
878 var newCollection arvados.Collection
879 err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
880 "collection": map[string]string{
881 "owner_uuid": arvadostest.ActiveUserUUID,
882 "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
883 "name": "keep-web test collection",
885 "ensure_unique_name": true,
887 c.Assert(err, check.IsNil)
888 defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
890 var updated arvados.Collection
891 for _, fnm := range []string{"foo.txt", "bar.txt"} {
892 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
893 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
894 req := &http.Request{
898 RequestURI: u.RequestURI(),
900 "Authorization": {"Bearer " + arvadostest.ActiveToken},
903 resp := httptest.NewRecorder()
904 s.testServer.Handler.ServeHTTP(resp, req)
905 c.Check(resp.Code, check.Equals, http.StatusNoContent)
907 updated = arvados.Collection{}
908 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
909 c.Check(err, check.IsNil)
910 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
911 c.Logf("updated manifest_text %q", updated.ManifestText)
913 c.Check(updated.ManifestText, check.Equals, "")
916 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
917 s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
918 authHeader := http.Header{
919 "Authorization": {"Bearer " + arvadostest.ManagementToken},
922 resp := httptest.NewRecorder()
923 u := mustParseURL("http://download.example.com/_health/ping")
924 req := &http.Request{
928 RequestURI: u.RequestURI(),
931 s.testServer.Handler.ServeHTTP(resp, req)
933 c.Check(resp.Code, check.Equals, http.StatusOK)
934 c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
937 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
938 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
940 client := s.testServer.Config.Client
941 client.AuthToken = arvadostest.ActiveToken
942 arv, err := arvadosclient.New(&client)
943 c.Assert(err, check.Equals, nil)
944 kc, err := keepclient.MakeKeepClient(arv)
945 c.Assert(err, check.Equals, nil)
947 fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
948 c.Assert(err, check.IsNil)
955 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
956 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
957 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
958 {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
959 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
961 for _, trial := range trials {
962 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
963 c.Assert(err, check.IsNil)
964 _, err = f.Write([]byte(trial.content))
965 c.Assert(err, check.IsNil)
966 c.Assert(f.Close(), check.IsNil)
968 mtxt, err := fs.MarshalManifest(".")
969 c.Assert(err, check.IsNil)
970 var coll arvados.Collection
971 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
972 "collection": map[string]string{
973 "manifest_text": mtxt,
976 c.Assert(err, check.IsNil)
978 for _, trial := range trials {
979 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
980 req := &http.Request{
984 RequestURI: u.RequestURI(),
986 "Authorization": {"Bearer " + client.AuthToken},
989 resp := httptest.NewRecorder()
990 s.testServer.Handler.ServeHTTP(resp, req)
991 c.Check(resp.Code, check.Equals, http.StatusOK)
992 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
993 c.Check(resp.Body.String(), check.Equals, trial.content)
997 func copyHeader(h http.Header) http.Header {
999 for k, v := range h {
1000 hc[k] = append([]string(nil), v...)