1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/lib/config"
23 "git.arvados.org/arvados.git/sdk/go/arvados"
24 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
25 "git.arvados.org/arvados.git/sdk/go/arvadostest"
26 "git.arvados.org/arvados.git/sdk/go/auth"
27 "git.arvados.org/arvados.git/sdk/go/ctxlog"
28 "git.arvados.org/arvados.git/sdk/go/keepclient"
29 "github.com/sirupsen/logrus"
30 check "gopkg.in/check.v1"
33 var _ = check.Suite(&UnitSuite{})
36 arvados.DebugLocksPanicMode = true
39 type UnitSuite struct {
40 Config *arvados.Config
43 func (s *UnitSuite) SetUpTest(c *check.C) {
44 ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
46 cfg, err := ldr.Load()
47 c.Assert(err, check.IsNil)
51 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
52 h := handler{Config: newConfig(s.Config)}
53 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
58 RequestURI: u.RequestURI(),
60 "Origin": {"https://workbench.example"},
61 "Access-Control-Request-Method": {"POST"},
65 // Check preflight for an allowed request
66 resp := httptest.NewRecorder()
67 h.ServeHTTP(resp, req)
68 c.Check(resp.Code, check.Equals, http.StatusOK)
69 c.Check(resp.Body.String(), check.Equals, "")
70 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
71 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
72 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
74 // Check preflight for a disallowed request
75 resp = httptest.NewRecorder()
76 req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
77 h.ServeHTTP(resp, req)
78 c.Check(resp.Body.String(), check.Equals, "")
79 c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
82 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
83 for _, trial := range []struct {
89 // If we return no content due to a Keep read error,
90 // we should emit a log message.
91 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
93 // If we return no content because the client sent an
94 // If-Modified-Since header, our response should be
95 // 304, and we should not emit a log message.
96 {true, true, http.StatusNotModified, ``},
98 c.Logf("trial: %+v", trial)
99 arvadostest.StartKeep(2, true)
100 if trial.dataExists {
101 arv, err := arvadosclient.MakeArvadosClient()
102 c.Assert(err, check.IsNil)
103 arv.ApiToken = arvadostest.ActiveToken
104 kc, err := keepclient.MakeKeepClient(arv)
105 c.Assert(err, check.IsNil)
106 _, _, err = kc.PutB([]byte("foo"))
107 c.Assert(err, check.IsNil)
110 h := handler{Config: newConfig(s.Config)}
111 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
112 req := &http.Request{
116 RequestURI: u.RequestURI(),
118 "Authorization": {"Bearer " + arvadostest.ActiveToken},
121 if trial.sendIMSHeader {
122 req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
125 var logbuf bytes.Buffer
126 logger := logrus.New()
128 req = req.WithContext(ctxlog.Context(context.Background(), logger))
130 resp := httptest.NewRecorder()
131 h.ServeHTTP(resp, req)
132 c.Check(resp.Code, check.Equals, trial.expectStatus)
133 c.Check(resp.Body.String(), check.Equals, "")
135 c.Log(logbuf.String())
136 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
140 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
141 bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
142 token := arvadostest.ActiveToken
143 for _, trial := range []string{
144 "http://keep-web/c=" + bogusID + "/foo",
145 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
146 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
147 "http://keep-web/collections/" + bogusID + "/foo",
148 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
149 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
152 u := mustParseURL(trial)
153 req := &http.Request{
157 RequestURI: u.RequestURI(),
159 resp := httptest.NewRecorder()
160 cfg := newConfig(s.Config)
161 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
162 h := handler{Config: cfg}
163 h.ServeHTTP(resp, req)
164 c.Check(resp.Code, check.Equals, http.StatusNotFound)
168 func mustParseURL(s string) *url.URL {
169 r, err := url.Parse(s)
171 panic("parse URL: " + s)
176 func (s *IntegrationSuite) TestVhost404(c *check.C) {
177 for _, testURL := range []string{
178 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
179 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
181 resp := httptest.NewRecorder()
182 u := mustParseURL(testURL)
183 req := &http.Request{
186 RequestURI: u.RequestURI(),
188 s.testServer.Handler.ServeHTTP(resp, req)
189 c.Check(resp.Code, check.Equals, http.StatusNotFound)
190 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
194 // An authorizer modifies an HTTP request to make use of the given
195 // token -- by adding it to a header, cookie, query param, or whatever
196 // -- and returns the HTTP status code we should expect from keep-web if
197 // the token is invalid.
198 type authorizer func(*http.Request, string) int
200 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
201 s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
203 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
204 r.Header.Add("Authorization", "Bearer "+tok)
205 return http.StatusUnauthorized
207 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
208 s.doVhostRequests(c, authzViaAuthzHeaderBearer)
210 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
211 r.Header.Add("Authorization", "Bearer "+tok)
212 return http.StatusUnauthorized
215 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
216 s.doVhostRequests(c, authzViaCookieValue)
218 func authzViaCookieValue(r *http.Request, tok string) int {
219 r.AddCookie(&http.Cookie{
220 Name: "arvados_api_token",
221 Value: auth.EncodeTokenCookie([]byte(tok)),
223 return http.StatusUnauthorized
226 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
227 s.doVhostRequests(c, authzViaPath)
229 func authzViaPath(r *http.Request, tok string) int {
230 r.URL.Path = "/t=" + tok + r.URL.Path
231 return http.StatusNotFound
234 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
235 s.doVhostRequests(c, authzViaQueryString)
237 func authzViaQueryString(r *http.Request, tok string) int {
238 r.URL.RawQuery = "api_token=" + tok
239 return http.StatusUnauthorized
242 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
243 s.doVhostRequests(c, authzViaPOST)
245 func authzViaPOST(r *http.Request, tok string) int {
247 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
248 r.Body = ioutil.NopCloser(strings.NewReader(
249 url.Values{"api_token": {tok}}.Encode()))
250 return http.StatusUnauthorized
253 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
254 s.doVhostRequests(c, authzViaPOST)
256 func authzViaXHRPOST(r *http.Request, tok string) int {
258 r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
259 r.Header.Add("Origin", "https://origin.example")
260 r.Body = ioutil.NopCloser(strings.NewReader(
263 "disposition": {"attachment"},
265 return http.StatusUnauthorized
268 // Try some combinations of {url, token} using the given authorization
269 // mechanism, and verify the result is correct.
270 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
271 for _, hostPath := range []string{
272 arvadostest.FooCollection + ".example.com/foo",
273 arvadostest.FooCollection + "--collections.example.com/foo",
274 arvadostest.FooCollection + "--collections.example.com/_/foo",
275 arvadostest.FooCollectionPDH + ".example.com/foo",
276 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
277 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
279 c.Log("doRequests: ", hostPath)
280 s.doVhostRequestsWithHostPath(c, authz, hostPath)
284 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
285 for _, tok := range []string{
286 arvadostest.ActiveToken,
287 arvadostest.ActiveToken[:15],
288 arvadostest.SpectatorToken,
292 u := mustParseURL("http://" + hostPath)
293 req := &http.Request{
297 RequestURI: u.RequestURI(),
298 Header: http.Header{},
300 failCode := authz(req, tok)
301 req, resp := s.doReq(req)
302 code, body := resp.Code, resp.Body.String()
304 // If the initial request had a (non-empty) token
305 // showing in the query string, we should have been
306 // redirected in order to hide it in a cookie.
307 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
309 if tok == arvadostest.ActiveToken {
310 c.Check(code, check.Equals, http.StatusOK)
311 c.Check(body, check.Equals, "foo")
313 c.Check(code >= 400, check.Equals, true)
314 c.Check(code < 500, check.Equals, true)
315 if tok == arvadostest.SpectatorToken {
316 // Valid token never offers to retry
317 // with different credentials.
318 c.Check(code, check.Equals, http.StatusNotFound)
320 // Invalid token can ask to retry
321 // depending on the authz method.
322 c.Check(code, check.Equals, failCode)
325 c.Check(body, check.Equals, notFoundMessage+"\n")
327 c.Check(body, check.Equals, unauthorizedMessage+"\n")
333 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
334 for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
335 for _, port := range []string{"80", "443", "8000"} {
336 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
337 u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
338 req := &http.Request{
342 RequestURI: u.RequestURI(),
343 Header: http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
345 req, resp := s.doReq(req)
346 code, _ := resp.Code, resp.Body.String()
349 c.Check(code, check.Equals, 401)
351 c.Check(code, check.Equals, 200)
357 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
358 resp := httptest.NewRecorder()
359 s.testServer.Handler.ServeHTTP(resp, req)
360 if resp.Code != http.StatusSeeOther {
363 cookies := (&http.Response{Header: resp.Header()}).Cookies()
364 u, _ := req.URL.Parse(resp.Header().Get("Location"))
369 RequestURI: u.RequestURI(),
370 Header: http.Header{},
372 for _, c := range cookies {
378 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
379 s.testVhostRedirectTokenToCookie(c, "GET",
380 arvadostest.FooCollection+".example.com/foo",
381 "?api_token="+arvadostest.ActiveToken,
389 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
390 s.testVhostRedirectTokenToCookie(c, "GET",
391 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
400 // Bad token in URL is 404 Not Found because it doesn't make sense to
401 // retry the same URL with different authorization.
402 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
403 s.testVhostRedirectTokenToCookie(c, "GET",
404 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
409 notFoundMessage+"\n",
413 // Bad token in a cookie (even if it got there via our own
414 // query-string-to-cookie redirect) is, in principle, retryable at the
415 // same URL so it's 401 Unauthorized.
416 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
417 s.testVhostRedirectTokenToCookie(c, "GET",
418 arvadostest.FooCollection+".example.com/foo",
419 "?api_token=thisisabogustoken",
422 http.StatusUnauthorized,
423 unauthorizedMessage+"\n",
427 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
428 s.testVhostRedirectTokenToCookie(c, "GET",
429 "example.com/c="+arvadostest.FooCollection+"/foo",
430 "?api_token="+arvadostest.ActiveToken,
433 http.StatusBadRequest,
434 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
438 // If client requests an attachment by putting ?disposition=attachment
439 // in the query string, and gets redirected, the redirect target
440 // should respond with an attachment.
441 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
442 resp := s.testVhostRedirectTokenToCookie(c, "GET",
443 arvadostest.FooCollection+".example.com/foo",
444 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
450 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
453 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
454 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
455 resp := s.testVhostRedirectTokenToCookie(c, "GET",
456 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
457 "?api_token="+arvadostest.ActiveToken,
463 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
466 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
467 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
468 resp := s.testVhostRedirectTokenToCookie(c, "GET",
469 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
470 "?api_token="+arvadostest.ActiveToken,
476 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
477 resp = s.testVhostRedirectTokenToCookie(c, "GET",
478 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
479 "?api_token="+arvadostest.ActiveToken,
485 c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
488 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
489 s.testServer.Config.cluster.Collections.TrustAllContent = true
490 s.testVhostRedirectTokenToCookie(c, "GET",
491 "example.com/c="+arvadostest.FooCollection+"/foo",
492 "?api_token="+arvadostest.ActiveToken,
500 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
501 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
503 s.testVhostRedirectTokenToCookie(c, "GET",
504 "example.com/c="+arvadostest.FooCollection+"/foo",
505 "?api_token="+arvadostest.ActiveToken,
508 http.StatusBadRequest,
509 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
512 resp := s.testVhostRedirectTokenToCookie(c, "GET",
513 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
514 "?api_token="+arvadostest.ActiveToken,
520 c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
523 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
524 s.testVhostRedirectTokenToCookie(c, "POST",
525 arvadostest.FooCollection+".example.com/foo",
527 "application/x-www-form-urlencoded",
528 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
534 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
535 s.testVhostRedirectTokenToCookie(c, "POST",
536 arvadostest.FooCollection+".example.com/foo",
538 "application/x-www-form-urlencoded",
539 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
541 notFoundMessage+"\n",
545 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
546 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
547 s.testVhostRedirectTokenToCookie(c, "GET",
548 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
557 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
558 s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
559 s.testVhostRedirectTokenToCookie(c, "GET",
560 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
565 notFoundMessage+"\n",
569 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
570 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
572 client := s.testServer.Config.Client
573 client.AuthToken = arvadostest.ActiveToken
574 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
575 c.Assert(err, check.IsNil)
576 f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
577 c.Assert(err, check.IsNil)
579 mtxt, err := fs.MarshalManifest(".")
580 c.Assert(err, check.IsNil)
581 var coll arvados.Collection
582 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
583 "collection": map[string]string{
584 "manifest_text": mtxt,
587 c.Assert(err, check.IsNil)
589 u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
590 req := &http.Request{
594 RequestURI: u.RequestURI(),
596 "Authorization": {"Bearer " + client.AuthToken},
599 resp := httptest.NewRecorder()
600 s.testServer.Handler.ServeHTTP(resp, req)
601 c.Check(resp.Code, check.Equals, http.StatusOK)
602 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
605 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
606 arv := arvados.NewClientFromEnv()
607 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
608 s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
609 name := "foo/bar/baz"
610 nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
611 nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
613 client := s.testServer.Config.Client
614 client.AuthToken = arvadostest.ActiveToken
615 fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
616 c.Assert(err, check.IsNil)
617 f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
618 c.Assert(err, check.IsNil)
620 mtxt, err := fs.MarshalManifest(".")
621 c.Assert(err, check.IsNil)
622 var coll arvados.Collection
623 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
624 "collection": map[string]string{
625 "manifest_text": mtxt,
627 "owner_uuid": arvadostest.AProjectUUID,
630 c.Assert(err, check.IsNil)
631 defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
633 base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
634 for tryURL, expectRegexp := range map[string]string{
635 base: `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
636 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
638 u, _ := url.Parse(tryURL)
639 req := &http.Request{
643 RequestURI: u.RequestURI(),
645 "Authorization": {"Bearer " + client.AuthToken},
648 resp := httptest.NewRecorder()
649 s.testServer.Handler.ServeHTTP(resp, req)
650 c.Check(resp.Code, check.Equals, http.StatusOK)
651 c.Check(resp.Body.String(), check.Matches, expectRegexp)
655 // XHRs can't follow redirect-with-cookie so they rely on method=POST
656 // and disposition=attachment (telling us it's acceptable to respond
657 // with content instead of a redirect) and an Origin header that gets
658 // added automatically by the browser (telling us it's desirable to do
660 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
661 u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
662 req := &http.Request{
666 RequestURI: u.RequestURI(),
668 "Origin": {"https://origin.example"},
669 "Content-Type": {"application/x-www-form-urlencoded"},
671 Body: ioutil.NopCloser(strings.NewReader(url.Values{
672 "api_token": {arvadostest.ActiveToken},
673 "disposition": {"attachment"},
676 resp := httptest.NewRecorder()
677 s.testServer.Handler.ServeHTTP(resp, req)
678 c.Check(resp.Code, check.Equals, http.StatusOK)
679 c.Check(resp.Body.String(), check.Equals, "foo")
680 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
682 // GET + Origin header is representative of both AJAX GET
683 // requests and inline images via <IMG crossorigin="anonymous"
685 u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
690 RequestURI: u.RequestURI(),
692 "Origin": {"https://origin.example"},
695 resp = httptest.NewRecorder()
696 s.testServer.Handler.ServeHTTP(resp, req)
697 c.Check(resp.Code, check.Equals, http.StatusOK)
698 c.Check(resp.Body.String(), check.Equals, "foo")
699 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
702 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
703 u, _ := url.Parse(`http://` + hostPath + queryString)
704 req := &http.Request{
708 RequestURI: u.RequestURI(),
709 Header: http.Header{"Content-Type": {contentType}},
710 Body: ioutil.NopCloser(strings.NewReader(reqBody)),
713 resp := httptest.NewRecorder()
715 c.Check(resp.Code, check.Equals, expectStatus)
716 c.Check(resp.Body.String(), check.Equals, expectRespBody)
719 s.testServer.Handler.ServeHTTP(resp, req)
720 if resp.Code != http.StatusSeeOther {
723 c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
724 cookies := (&http.Response{Header: resp.Header()}).Cookies()
726 u, _ = u.Parse(resp.Header().Get("Location"))
731 RequestURI: u.RequestURI(),
732 Header: http.Header{},
734 for _, c := range cookies {
738 resp = httptest.NewRecorder()
739 s.testServer.Handler.ServeHTTP(resp, req)
740 c.Check(resp.Header().Get("Location"), check.Equals, "")
744 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
745 s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
746 s.testDirectoryListing(c)
749 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
750 s.testServer.Config.cluster.Users.AnonymousUserToken = ""
751 s.testDirectoryListing(c)
754 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
755 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
756 authHeader := http.Header{
757 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
759 for _, trial := range []struct {
767 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
769 expect: []string{"dir1/foo", "dir1/bar"},
773 uri: strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
775 expect: []string{"foo", "bar"},
779 // URLs of this form ignore authHeader, and
780 // FooAndBarFilesInDirUUID isn't public, so
782 uri: "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
787 uri: "download.example.com/users/active/foo_file_in_dir/",
789 expect: []string{"dir1/"},
793 uri: "download.example.com/users/active/foo_file_in_dir/dir1/",
795 expect: []string{"bar"},
799 uri: "download.example.com/",
801 expect: []string{"users/"},
805 uri: "download.example.com/users",
808 expect: []string{"active/"},
812 uri: "download.example.com/users/",
814 expect: []string{"active/"},
818 uri: "download.example.com/users/active",
820 redirect: "/users/active/",
821 expect: []string{"foo_file_in_dir/"},
825 uri: "download.example.com/users/active/",
827 expect: []string{"foo_file_in_dir/"},
831 uri: "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
833 expect: []string{"dir1/foo", "dir1/bar"},
837 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
839 expect: []string{"dir1/foo", "dir1/bar"},
843 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
845 expect: []string{"dir1/foo", "dir1/bar"},
849 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
851 expect: []string{"dir1/foo", "dir1/bar"},
855 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
857 redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
858 expect: []string{"foo", "bar"},
862 uri: "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
864 expect: []string{"foo", "bar"},
868 uri: arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
871 expect: []string{"foo", "bar"},
875 uri: "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
880 uri: "download.example.com/c=" + arvadostest.WazVersion1Collection,
882 expect: []string{"waz"},
886 uri: "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
888 expect: []string{"waz"},
892 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
893 resp := httptest.NewRecorder()
894 u := mustParseURL("//" + trial.uri)
895 req := &http.Request{
899 RequestURI: u.RequestURI(),
900 Header: copyHeader(trial.header),
902 s.testServer.Handler.ServeHTTP(resp, req)
903 var cookies []*http.Cookie
904 for resp.Code == http.StatusSeeOther {
905 u, _ := req.URL.Parse(resp.Header().Get("Location"))
910 RequestURI: u.RequestURI(),
911 Header: copyHeader(trial.header),
913 cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
914 for _, c := range cookies {
917 resp = httptest.NewRecorder()
918 s.testServer.Handler.ServeHTTP(resp, req)
920 if trial.redirect != "" {
921 c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
923 if trial.expect == nil {
924 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
925 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
927 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
930 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
931 for _, e := range trial.expect {
932 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
934 c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
937 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
942 RequestURI: u.RequestURI(),
943 Header: copyHeader(trial.header),
944 Body: ioutil.NopCloser(&bytes.Buffer{}),
946 resp = httptest.NewRecorder()
947 s.testServer.Handler.ServeHTTP(resp, req)
948 if trial.expect == nil {
949 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
950 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
952 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
955 c.Check(resp.Code, check.Equals, http.StatusOK, comment)
962 RequestURI: u.RequestURI(),
963 Header: copyHeader(trial.header),
964 Body: ioutil.NopCloser(&bytes.Buffer{}),
966 resp = httptest.NewRecorder()
967 s.testServer.Handler.ServeHTTP(resp, req)
968 if trial.expect == nil {
969 if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
970 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
972 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
975 c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
976 for _, e := range trial.expect {
977 if strings.HasSuffix(e, "/") {
978 e = filepath.Join(u.Path, e) + "/"
980 e = filepath.Join(u.Path, e)
982 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
988 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
989 arv := arvados.NewClientFromEnv()
990 var newCollection arvados.Collection
991 err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
992 "collection": map[string]string{
993 "owner_uuid": arvadostest.ActiveUserUUID,
994 "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
995 "name": "keep-web test collection",
997 "ensure_unique_name": true,
999 c.Assert(err, check.IsNil)
1000 defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1002 var updated arvados.Collection
1003 for _, fnm := range []string{"foo.txt", "bar.txt"} {
1004 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1005 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1006 req := &http.Request{
1010 RequestURI: u.RequestURI(),
1011 Header: http.Header{
1012 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1015 resp := httptest.NewRecorder()
1016 s.testServer.Handler.ServeHTTP(resp, req)
1017 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1019 updated = arvados.Collection{}
1020 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1021 c.Check(err, check.IsNil)
1022 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1023 c.Logf("updated manifest_text %q", updated.ManifestText)
1025 c.Check(updated.ManifestText, check.Equals, "")
1028 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1029 s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1030 authHeader := http.Header{
1031 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1034 resp := httptest.NewRecorder()
1035 u := mustParseURL("http://download.example.com/_health/ping")
1036 req := &http.Request{
1040 RequestURI: u.RequestURI(),
1043 s.testServer.Handler.ServeHTTP(resp, req)
1045 c.Check(resp.Code, check.Equals, http.StatusOK)
1046 c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1049 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1050 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1052 client := s.testServer.Config.Client
1053 client.AuthToken = arvadostest.ActiveToken
1054 arv, err := arvadosclient.New(&client)
1055 c.Assert(err, check.Equals, nil)
1056 kc, err := keepclient.MakeKeepClient(arv)
1057 c.Assert(err, check.Equals, nil)
1059 fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1060 c.Assert(err, check.IsNil)
1062 trials := []struct {
1067 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1068 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1069 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1070 {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
1071 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1073 for _, trial := range trials {
1074 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1075 c.Assert(err, check.IsNil)
1076 _, err = f.Write([]byte(trial.content))
1077 c.Assert(err, check.IsNil)
1078 c.Assert(f.Close(), check.IsNil)
1080 mtxt, err := fs.MarshalManifest(".")
1081 c.Assert(err, check.IsNil)
1082 var coll arvados.Collection
1083 err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1084 "collection": map[string]string{
1085 "manifest_text": mtxt,
1088 c.Assert(err, check.IsNil)
1090 for _, trial := range trials {
1091 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1092 req := &http.Request{
1096 RequestURI: u.RequestURI(),
1097 Header: http.Header{
1098 "Authorization": {"Bearer " + client.AuthToken},
1101 resp := httptest.NewRecorder()
1102 s.testServer.Handler.ServeHTTP(resp, req)
1103 c.Check(resp.Code, check.Equals, http.StatusOK)
1104 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1105 c.Check(resp.Body.String(), check.Equals, trial.content)
1109 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1110 s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1111 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1112 u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1113 req := &http.Request{
1117 RequestURI: u.RequestURI(),
1119 resp := httptest.NewRecorder()
1120 s.testServer.Handler.ServeHTTP(resp, req)
1121 c.Check(resp.Code, check.Equals, http.StatusOK)
1122 c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1125 // Writing to a collection shouldn't affect its entry in the
1126 // PDH-to-manifest cache.
1127 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1128 arv, err := arvadosclient.MakeArvadosClient()
1129 c.Assert(err, check.Equals, nil)
1130 arv.ApiToken = arvadostest.ActiveToken
1132 u := mustParseURL("http://x.example/testfile")
1133 req := &http.Request{
1137 RequestURI: u.RequestURI(),
1138 Header: http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1141 checkWithID := func(id string, status int) {
1142 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1143 req.Host = req.URL.Host
1144 resp := httptest.NewRecorder()
1145 s.testServer.Handler.ServeHTTP(resp, req)
1146 c.Check(resp.Code, check.Equals, status)
1149 var colls [2]arvados.Collection
1150 for i := range colls {
1151 err := arv.Create("collections",
1152 map[string]interface{}{
1153 "ensure_unique_name": true,
1154 "collection": map[string]interface{}{
1155 "name": "test collection",
1158 c.Assert(err, check.Equals, nil)
1161 // Populate cache with empty collection
1162 checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1164 // write a file to colls[0]
1166 reqPut.Method = "PUT"
1167 reqPut.URL.Host = colls[0].UUID + ".example"
1168 reqPut.Host = req.URL.Host
1169 reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1170 resp := httptest.NewRecorder()
1171 s.testServer.Handler.ServeHTTP(resp, &reqPut)
1172 c.Check(resp.Code, check.Equals, http.StatusCreated)
1174 // new file should not appear in colls[1]
1175 checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1176 checkWithID(colls[1].UUID, http.StatusNotFound)
1178 checkWithID(colls[0].UUID, http.StatusOK)
1181 func copyHeader(h http.Header) http.Header {
1183 for k, v := range h {
1184 hc[k] = append([]string(nil), v...)