Merge branch '15924-import-path'
[arvados.git] / services / keep-web / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "fmt"
10         "html"
11         "io/ioutil"
12         "net/http"
13         "net/http/httptest"
14         "net/url"
15         "os"
16         "path/filepath"
17         "regexp"
18         "strings"
19
20         "git.arvados.org/arvados.git/lib/config"
21         "git.arvados.org/arvados.git/sdk/go/arvados"
22         "git.arvados.org/arvados.git/sdk/go/arvadostest"
23         "git.arvados.org/arvados.git/sdk/go/auth"
24         "git.arvados.org/arvados.git/sdk/go/ctxlog"
25         "git.arvados.org/arvados.git/sdk/go/keepclient"
26         check "gopkg.in/check.v1"
27 )
28
29 var _ = check.Suite(&UnitSuite{})
30
31 type UnitSuite struct {
32         Config *arvados.Config
33 }
34
35 func (s *UnitSuite) SetUpTest(c *check.C) {
36         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
37         ldr.Path = "-"
38         cfg, err := ldr.Load()
39         c.Assert(err, check.IsNil)
40         s.Config = cfg
41 }
42
43 func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
44         cfg := newConfig(s.Config)
45         cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
46         h := handler{Config: cfg}
47         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
48         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
49         req := &http.Request{
50                 Method:     "GET",
51                 Host:       u.Host,
52                 URL:        u,
53                 RequestURI: u.RequestURI(),
54         }
55         resp := httptest.NewRecorder()
56         h.ServeHTTP(resp, req)
57         c.Check(resp.Code, check.Equals, http.StatusOK)
58         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
59 }
60
61 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
62         h := handler{Config: newConfig(s.Config)}
63         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
64         req := &http.Request{
65                 Method:     "OPTIONS",
66                 Host:       u.Host,
67                 URL:        u,
68                 RequestURI: u.RequestURI(),
69                 Header: http.Header{
70                         "Origin":                        {"https://workbench.example"},
71                         "Access-Control-Request-Method": {"POST"},
72                 },
73         }
74
75         // Check preflight for an allowed request
76         resp := httptest.NewRecorder()
77         h.ServeHTTP(resp, req)
78         c.Check(resp.Code, check.Equals, http.StatusOK)
79         c.Check(resp.Body.String(), check.Equals, "")
80         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
81         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
82         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
83
84         // Check preflight for a disallowed request
85         resp = httptest.NewRecorder()
86         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
87         h.ServeHTTP(resp, req)
88         c.Check(resp.Body.String(), check.Equals, "")
89         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
90 }
91
92 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
93         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
94         token := arvadostest.ActiveToken
95         for _, trial := range []string{
96                 "http://keep-web/c=" + bogusID + "/foo",
97                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
98                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
99                 "http://keep-web/collections/" + bogusID + "/foo",
100                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
101                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
102         } {
103                 c.Log(trial)
104                 u := mustParseURL(trial)
105                 req := &http.Request{
106                         Method:     "GET",
107                         Host:       u.Host,
108                         URL:        u,
109                         RequestURI: u.RequestURI(),
110                 }
111                 resp := httptest.NewRecorder()
112                 cfg := newConfig(s.Config)
113                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
114                 h := handler{Config: cfg}
115                 h.ServeHTTP(resp, req)
116                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
117         }
118 }
119
120 func mustParseURL(s string) *url.URL {
121         r, err := url.Parse(s)
122         if err != nil {
123                 panic("parse URL: " + s)
124         }
125         return r
126 }
127
128 func (s *IntegrationSuite) TestVhost404(c *check.C) {
129         for _, testURL := range []string{
130                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
131                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
132         } {
133                 resp := httptest.NewRecorder()
134                 u := mustParseURL(testURL)
135                 req := &http.Request{
136                         Method:     "GET",
137                         URL:        u,
138                         RequestURI: u.RequestURI(),
139                 }
140                 s.testServer.Handler.ServeHTTP(resp, req)
141                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
142                 c.Check(resp.Body.String(), check.Equals, "")
143         }
144 }
145
146 // An authorizer modifies an HTTP request to make use of the given
147 // token -- by adding it to a header, cookie, query param, or whatever
148 // -- and returns the HTTP status code we should expect from keep-web if
149 // the token is invalid.
150 type authorizer func(*http.Request, string) int
151
152 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
153         s.doVhostRequests(c, authzViaAuthzHeader)
154 }
155 func authzViaAuthzHeader(r *http.Request, tok string) int {
156         r.Header.Add("Authorization", "OAuth2 "+tok)
157         return http.StatusUnauthorized
158 }
159
160 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
161         s.doVhostRequests(c, authzViaCookieValue)
162 }
163 func authzViaCookieValue(r *http.Request, tok string) int {
164         r.AddCookie(&http.Cookie{
165                 Name:  "arvados_api_token",
166                 Value: auth.EncodeTokenCookie([]byte(tok)),
167         })
168         return http.StatusUnauthorized
169 }
170
171 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
172         s.doVhostRequests(c, authzViaPath)
173 }
174 func authzViaPath(r *http.Request, tok string) int {
175         r.URL.Path = "/t=" + tok + r.URL.Path
176         return http.StatusNotFound
177 }
178
179 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
180         s.doVhostRequests(c, authzViaQueryString)
181 }
182 func authzViaQueryString(r *http.Request, tok string) int {
183         r.URL.RawQuery = "api_token=" + tok
184         return http.StatusUnauthorized
185 }
186
187 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
188         s.doVhostRequests(c, authzViaPOST)
189 }
190 func authzViaPOST(r *http.Request, tok string) int {
191         r.Method = "POST"
192         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
193         r.Body = ioutil.NopCloser(strings.NewReader(
194                 url.Values{"api_token": {tok}}.Encode()))
195         return http.StatusUnauthorized
196 }
197
198 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
199         s.doVhostRequests(c, authzViaPOST)
200 }
201 func authzViaXHRPOST(r *http.Request, tok string) int {
202         r.Method = "POST"
203         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
204         r.Header.Add("Origin", "https://origin.example")
205         r.Body = ioutil.NopCloser(strings.NewReader(
206                 url.Values{
207                         "api_token":   {tok},
208                         "disposition": {"attachment"},
209                 }.Encode()))
210         return http.StatusUnauthorized
211 }
212
213 // Try some combinations of {url, token} using the given authorization
214 // mechanism, and verify the result is correct.
215 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
216         for _, hostPath := range []string{
217                 arvadostest.FooCollection + ".example.com/foo",
218                 arvadostest.FooCollection + "--collections.example.com/foo",
219                 arvadostest.FooCollection + "--collections.example.com/_/foo",
220                 arvadostest.FooCollectionPDH + ".example.com/foo",
221                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
222                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
223         } {
224                 c.Log("doRequests: ", hostPath)
225                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
226         }
227 }
228
229 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
230         for _, tok := range []string{
231                 arvadostest.ActiveToken,
232                 arvadostest.ActiveToken[:15],
233                 arvadostest.SpectatorToken,
234                 "bogus",
235                 "",
236         } {
237                 u := mustParseURL("http://" + hostPath)
238                 req := &http.Request{
239                         Method:     "GET",
240                         Host:       u.Host,
241                         URL:        u,
242                         RequestURI: u.RequestURI(),
243                         Header:     http.Header{},
244                 }
245                 failCode := authz(req, tok)
246                 req, resp := s.doReq(req)
247                 code, body := resp.Code, resp.Body.String()
248
249                 // If the initial request had a (non-empty) token
250                 // showing in the query string, we should have been
251                 // redirected in order to hide it in a cookie.
252                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
253
254                 if tok == arvadostest.ActiveToken {
255                         c.Check(code, check.Equals, http.StatusOK)
256                         c.Check(body, check.Equals, "foo")
257
258                 } else {
259                         c.Check(code >= 400, check.Equals, true)
260                         c.Check(code < 500, check.Equals, true)
261                         if tok == arvadostest.SpectatorToken {
262                                 // Valid token never offers to retry
263                                 // with different credentials.
264                                 c.Check(code, check.Equals, http.StatusNotFound)
265                         } else {
266                                 // Invalid token can ask to retry
267                                 // depending on the authz method.
268                                 c.Check(code, check.Equals, failCode)
269                         }
270                         c.Check(body, check.Equals, "")
271                 }
272         }
273 }
274
275 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
276         resp := httptest.NewRecorder()
277         s.testServer.Handler.ServeHTTP(resp, req)
278         if resp.Code != http.StatusSeeOther {
279                 return req, resp
280         }
281         cookies := (&http.Response{Header: resp.Header()}).Cookies()
282         u, _ := req.URL.Parse(resp.Header().Get("Location"))
283         req = &http.Request{
284                 Method:     "GET",
285                 Host:       u.Host,
286                 URL:        u,
287                 RequestURI: u.RequestURI(),
288                 Header:     http.Header{},
289         }
290         for _, c := range cookies {
291                 req.AddCookie(c)
292         }
293         return s.doReq(req)
294 }
295
296 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
297         s.testVhostRedirectTokenToCookie(c, "GET",
298                 arvadostest.FooCollection+".example.com/foo",
299                 "?api_token="+arvadostest.ActiveToken,
300                 "",
301                 "",
302                 http.StatusOK,
303                 "foo",
304         )
305 }
306
307 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
308         s.testVhostRedirectTokenToCookie(c, "GET",
309                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
310                 "",
311                 "",
312                 "",
313                 http.StatusOK,
314                 "foo",
315         )
316 }
317
318 // Bad token in URL is 404 Not Found because it doesn't make sense to
319 // retry the same URL with different authorization.
320 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
321         s.testVhostRedirectTokenToCookie(c, "GET",
322                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
323                 "",
324                 "",
325                 "",
326                 http.StatusNotFound,
327                 "",
328         )
329 }
330
331 // Bad token in a cookie (even if it got there via our own
332 // query-string-to-cookie redirect) is, in principle, retryable at the
333 // same URL so it's 401 Unauthorized.
334 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
335         s.testVhostRedirectTokenToCookie(c, "GET",
336                 arvadostest.FooCollection+".example.com/foo",
337                 "?api_token=thisisabogustoken",
338                 "",
339                 "",
340                 http.StatusUnauthorized,
341                 "",
342         )
343 }
344
345 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
346         s.testVhostRedirectTokenToCookie(c, "GET",
347                 "example.com/c="+arvadostest.FooCollection+"/foo",
348                 "?api_token="+arvadostest.ActiveToken,
349                 "",
350                 "",
351                 http.StatusBadRequest,
352                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
353         )
354 }
355
356 // If client requests an attachment by putting ?disposition=attachment
357 // in the query string, and gets redirected, the redirect target
358 // should respond with an attachment.
359 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
360         resp := s.testVhostRedirectTokenToCookie(c, "GET",
361                 arvadostest.FooCollection+".example.com/foo",
362                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
363                 "",
364                 "",
365                 http.StatusOK,
366                 "foo",
367         )
368         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
369 }
370
371 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
372         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
373         resp := s.testVhostRedirectTokenToCookie(c, "GET",
374                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
375                 "?api_token="+arvadostest.ActiveToken,
376                 "",
377                 "",
378                 http.StatusOK,
379                 "foo",
380         )
381         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
382 }
383
384 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
385         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
386         resp := s.testVhostRedirectTokenToCookie(c, "GET",
387                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
388                 "?api_token="+arvadostest.ActiveToken,
389                 "",
390                 "",
391                 http.StatusOK,
392                 "waz",
393         )
394         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
395         resp = s.testVhostRedirectTokenToCookie(c, "GET",
396                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
397                 "?api_token="+arvadostest.ActiveToken,
398                 "",
399                 "",
400                 http.StatusOK,
401                 "waz",
402         )
403         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
404 }
405
406 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
407         s.testServer.Config.cluster.Collections.TrustAllContent = true
408         s.testVhostRedirectTokenToCookie(c, "GET",
409                 "example.com/c="+arvadostest.FooCollection+"/foo",
410                 "?api_token="+arvadostest.ActiveToken,
411                 "",
412                 "",
413                 http.StatusOK,
414                 "foo",
415         )
416 }
417
418 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
419         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
420
421         s.testVhostRedirectTokenToCookie(c, "GET",
422                 "example.com/c="+arvadostest.FooCollection+"/foo",
423                 "?api_token="+arvadostest.ActiveToken,
424                 "",
425                 "",
426                 http.StatusBadRequest,
427                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
428         )
429
430         resp := s.testVhostRedirectTokenToCookie(c, "GET",
431                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
432                 "?api_token="+arvadostest.ActiveToken,
433                 "",
434                 "",
435                 http.StatusOK,
436                 "foo",
437         )
438         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
439 }
440
441 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
442         s.testVhostRedirectTokenToCookie(c, "POST",
443                 arvadostest.FooCollection+".example.com/foo",
444                 "",
445                 "application/x-www-form-urlencoded",
446                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
447                 http.StatusOK,
448                 "foo",
449         )
450 }
451
452 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
453         s.testVhostRedirectTokenToCookie(c, "POST",
454                 arvadostest.FooCollection+".example.com/foo",
455                 "",
456                 "application/x-www-form-urlencoded",
457                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
458                 http.StatusNotFound,
459                 "",
460         )
461 }
462
463 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
464         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
465         s.testVhostRedirectTokenToCookie(c, "GET",
466                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
467                 "",
468                 "",
469                 "",
470                 http.StatusOK,
471                 "Hello world\n",
472         )
473 }
474
475 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
476         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
477         s.testVhostRedirectTokenToCookie(c, "GET",
478                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
479                 "",
480                 "",
481                 "",
482                 http.StatusNotFound,
483                 "",
484         )
485 }
486
487 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
488         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
489
490         client := s.testServer.Config.Client
491         client.AuthToken = arvadostest.ActiveToken
492         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
493         c.Assert(err, check.IsNil)
494         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
495         c.Assert(err, check.IsNil)
496         f.Close()
497         mtxt, err := fs.MarshalManifest(".")
498         c.Assert(err, check.IsNil)
499         var coll arvados.Collection
500         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
501                 "collection": map[string]string{
502                         "manifest_text": mtxt,
503                 },
504         })
505         c.Assert(err, check.IsNil)
506
507         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
508         req := &http.Request{
509                 Method:     "GET",
510                 Host:       u.Host,
511                 URL:        u,
512                 RequestURI: u.RequestURI(),
513                 Header: http.Header{
514                         "Authorization": {"Bearer " + client.AuthToken},
515                 },
516         }
517         resp := httptest.NewRecorder()
518         s.testServer.Handler.ServeHTTP(resp, req)
519         c.Check(resp.Code, check.Equals, http.StatusOK)
520         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
521 }
522
523 // XHRs can't follow redirect-with-cookie so they rely on method=POST
524 // and disposition=attachment (telling us it's acceptable to respond
525 // with content instead of a redirect) and an Origin header that gets
526 // added automatically by the browser (telling us it's desirable to do
527 // so).
528 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
529         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
530         req := &http.Request{
531                 Method:     "POST",
532                 Host:       u.Host,
533                 URL:        u,
534                 RequestURI: u.RequestURI(),
535                 Header: http.Header{
536                         "Origin":       {"https://origin.example"},
537                         "Content-Type": {"application/x-www-form-urlencoded"},
538                 },
539                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
540                         "api_token":   {arvadostest.ActiveToken},
541                         "disposition": {"attachment"},
542                 }.Encode())),
543         }
544         resp := httptest.NewRecorder()
545         s.testServer.Handler.ServeHTTP(resp, req)
546         c.Check(resp.Code, check.Equals, http.StatusOK)
547         c.Check(resp.Body.String(), check.Equals, "foo")
548         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
549 }
550
551 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
552         u, _ := url.Parse(`http://` + hostPath + queryString)
553         req := &http.Request{
554                 Method:     method,
555                 Host:       u.Host,
556                 URL:        u,
557                 RequestURI: u.RequestURI(),
558                 Header:     http.Header{"Content-Type": {contentType}},
559                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
560         }
561
562         resp := httptest.NewRecorder()
563         defer func() {
564                 c.Check(resp.Code, check.Equals, expectStatus)
565                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
566         }()
567
568         s.testServer.Handler.ServeHTTP(resp, req)
569         if resp.Code != http.StatusSeeOther {
570                 return resp
571         }
572         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
573         cookies := (&http.Response{Header: resp.Header()}).Cookies()
574
575         u, _ = u.Parse(resp.Header().Get("Location"))
576         req = &http.Request{
577                 Method:     "GET",
578                 Host:       u.Host,
579                 URL:        u,
580                 RequestURI: u.RequestURI(),
581                 Header:     http.Header{},
582         }
583         for _, c := range cookies {
584                 req.AddCookie(c)
585         }
586
587         resp = httptest.NewRecorder()
588         s.testServer.Handler.ServeHTTP(resp, req)
589         c.Check(resp.Header().Get("Location"), check.Equals, "")
590         return resp
591 }
592
593 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
594         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
595         s.testDirectoryListing(c)
596 }
597
598 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
599         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
600         s.testDirectoryListing(c)
601 }
602
603 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
604         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
605         authHeader := http.Header{
606                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
607         }
608         for _, trial := range []struct {
609                 uri      string
610                 header   http.Header
611                 expect   []string
612                 redirect string
613                 cutDirs  int
614         }{
615                 {
616                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
617                         header:  authHeader,
618                         expect:  []string{"dir1/foo", "dir1/bar"},
619                         cutDirs: 0,
620                 },
621                 {
622                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
623                         header:  authHeader,
624                         expect:  []string{"foo", "bar"},
625                         cutDirs: 1,
626                 },
627                 {
628                         // URLs of this form ignore authHeader, and
629                         // FooAndBarFilesInDirUUID isn't public, so
630                         // this returns 404.
631                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
632                         header: authHeader,
633                         expect: nil,
634                 },
635                 {
636                         uri:     "download.example.com/users/active/foo_file_in_dir/",
637                         header:  authHeader,
638                         expect:  []string{"dir1/"},
639                         cutDirs: 3,
640                 },
641                 {
642                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
643                         header:  authHeader,
644                         expect:  []string{"bar"},
645                         cutDirs: 4,
646                 },
647                 {
648                         uri:     "download.example.com/",
649                         header:  authHeader,
650                         expect:  []string{"users/"},
651                         cutDirs: 0,
652                 },
653                 {
654                         uri:      "download.example.com/users",
655                         header:   authHeader,
656                         redirect: "/users/",
657                         expect:   []string{"active/"},
658                         cutDirs:  1,
659                 },
660                 {
661                         uri:     "download.example.com/users/",
662                         header:  authHeader,
663                         expect:  []string{"active/"},
664                         cutDirs: 1,
665                 },
666                 {
667                         uri:      "download.example.com/users/active",
668                         header:   authHeader,
669                         redirect: "/users/active/",
670                         expect:   []string{"foo_file_in_dir/"},
671                         cutDirs:  2,
672                 },
673                 {
674                         uri:     "download.example.com/users/active/",
675                         header:  authHeader,
676                         expect:  []string{"foo_file_in_dir/"},
677                         cutDirs: 2,
678                 },
679                 {
680                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
681                         header:  nil,
682                         expect:  []string{"dir1/foo", "dir1/bar"},
683                         cutDirs: 4,
684                 },
685                 {
686                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
687                         header:  nil,
688                         expect:  []string{"dir1/foo", "dir1/bar"},
689                         cutDirs: 2,
690                 },
691                 {
692                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
693                         header:  nil,
694                         expect:  []string{"dir1/foo", "dir1/bar"},
695                         cutDirs: 2,
696                 },
697                 {
698                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
699                         header:  authHeader,
700                         expect:  []string{"dir1/foo", "dir1/bar"},
701                         cutDirs: 1,
702                 },
703                 {
704                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
705                         header:   authHeader,
706                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
707                         expect:   []string{"foo", "bar"},
708                         cutDirs:  2,
709                 },
710                 {
711                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
712                         header:  authHeader,
713                         expect:  []string{"foo", "bar"},
714                         cutDirs: 3,
715                 },
716                 {
717                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
718                         header:   authHeader,
719                         redirect: "/dir1/",
720                         expect:   []string{"foo", "bar"},
721                         cutDirs:  1,
722                 },
723                 {
724                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
725                         header: authHeader,
726                         expect: nil,
727                 },
728                 {
729                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
730                         header:  authHeader,
731                         expect:  []string{"waz"},
732                         cutDirs: 1,
733                 },
734                 {
735                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
736                         header:  authHeader,
737                         expect:  []string{"waz"},
738                         cutDirs: 2,
739                 },
740         } {
741                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
742                 resp := httptest.NewRecorder()
743                 u := mustParseURL("//" + trial.uri)
744                 req := &http.Request{
745                         Method:     "GET",
746                         Host:       u.Host,
747                         URL:        u,
748                         RequestURI: u.RequestURI(),
749                         Header:     copyHeader(trial.header),
750                 }
751                 s.testServer.Handler.ServeHTTP(resp, req)
752                 var cookies []*http.Cookie
753                 for resp.Code == http.StatusSeeOther {
754                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
755                         req = &http.Request{
756                                 Method:     "GET",
757                                 Host:       u.Host,
758                                 URL:        u,
759                                 RequestURI: u.RequestURI(),
760                                 Header:     copyHeader(trial.header),
761                         }
762                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
763                         for _, c := range cookies {
764                                 req.AddCookie(c)
765                         }
766                         resp = httptest.NewRecorder()
767                         s.testServer.Handler.ServeHTTP(resp, req)
768                 }
769                 if trial.redirect != "" {
770                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
771                 }
772                 if trial.expect == nil {
773                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
774                 } else {
775                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
776                         for _, e := range trial.expect {
777                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
778                         }
779                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
780                 }
781
782                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
783                 req = &http.Request{
784                         Method:     "OPTIONS",
785                         Host:       u.Host,
786                         URL:        u,
787                         RequestURI: u.RequestURI(),
788                         Header:     copyHeader(trial.header),
789                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
790                 }
791                 resp = httptest.NewRecorder()
792                 s.testServer.Handler.ServeHTTP(resp, req)
793                 if trial.expect == nil {
794                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
795                 } else {
796                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
797                 }
798
799                 req = &http.Request{
800                         Method:     "PROPFIND",
801                         Host:       u.Host,
802                         URL:        u,
803                         RequestURI: u.RequestURI(),
804                         Header:     copyHeader(trial.header),
805                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
806                 }
807                 resp = httptest.NewRecorder()
808                 s.testServer.Handler.ServeHTTP(resp, req)
809                 if trial.expect == nil {
810                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
811                 } else {
812                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
813                         for _, e := range trial.expect {
814                                 if strings.HasSuffix(e, "/") {
815                                         e = filepath.Join(u.Path, e) + "/"
816                                 } else {
817                                         e = filepath.Join(u.Path, e)
818                                 }
819                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
820                         }
821                 }
822         }
823 }
824
825 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
826         arv := arvados.NewClientFromEnv()
827         var newCollection arvados.Collection
828         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
829                 "collection": map[string]string{
830                         "owner_uuid":    arvadostest.ActiveUserUUID,
831                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
832                         "name":          "keep-web test collection",
833                 },
834                 "ensure_unique_name": true,
835         })
836         c.Assert(err, check.IsNil)
837         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
838
839         var updated arvados.Collection
840         for _, fnm := range []string{"foo.txt", "bar.txt"} {
841                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
842                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
843                 req := &http.Request{
844                         Method:     "DELETE",
845                         Host:       u.Host,
846                         URL:        u,
847                         RequestURI: u.RequestURI(),
848                         Header: http.Header{
849                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
850                         },
851                 }
852                 resp := httptest.NewRecorder()
853                 s.testServer.Handler.ServeHTTP(resp, req)
854                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
855
856                 updated = arvados.Collection{}
857                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
858                 c.Check(err, check.IsNil)
859                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
860                 c.Logf("updated manifest_text %q", updated.ManifestText)
861         }
862         c.Check(updated.ManifestText, check.Equals, "")
863 }
864
865 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
866         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
867         authHeader := http.Header{
868                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
869         }
870
871         resp := httptest.NewRecorder()
872         u := mustParseURL("http://download.example.com/_health/ping")
873         req := &http.Request{
874                 Method:     "GET",
875                 Host:       u.Host,
876                 URL:        u,
877                 RequestURI: u.RequestURI(),
878                 Header:     authHeader,
879         }
880         s.testServer.Handler.ServeHTTP(resp, req)
881
882         c.Check(resp.Code, check.Equals, http.StatusOK)
883         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
884 }
885
886 func copyHeader(h http.Header) http.Header {
887         hc := http.Header{}
888         for k, v := range h {
889                 hc[k] = append([]string(nil), v...)
890         }
891         return hc
892 }