Merge branch '15028-cwl-v1.1' refs #15028
[arvados.git] / services / keep-web / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package main
6
7 import (
8         "bytes"
9         "fmt"
10         "html"
11         "io/ioutil"
12         "net/http"
13         "net/http/httptest"
14         "net/url"
15         "os"
16         "path/filepath"
17         "regexp"
18         "strings"
19
20         "git.curoverse.com/arvados.git/sdk/go/arvados"
21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
22         "git.curoverse.com/arvados.git/sdk/go/auth"
23         check "gopkg.in/check.v1"
24 )
25
26 var _ = check.Suite(&UnitSuite{})
27
28 type UnitSuite struct{}
29
30 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
31         h := handler{Config: DefaultConfig()}
32         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
33         req := &http.Request{
34                 Method:     "OPTIONS",
35                 Host:       u.Host,
36                 URL:        u,
37                 RequestURI: u.RequestURI(),
38                 Header: http.Header{
39                         "Origin":                        {"https://workbench.example"},
40                         "Access-Control-Request-Method": {"POST"},
41                 },
42         }
43
44         // Check preflight for an allowed request
45         resp := httptest.NewRecorder()
46         h.ServeHTTP(resp, req)
47         c.Check(resp.Code, check.Equals, http.StatusOK)
48         c.Check(resp.Body.String(), check.Equals, "")
49         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
50         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
51         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
52
53         // Check preflight for a disallowed request
54         resp = httptest.NewRecorder()
55         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
56         h.ServeHTTP(resp, req)
57         c.Check(resp.Body.String(), check.Equals, "")
58         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
59 }
60
61 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
62         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
63         token := arvadostest.ActiveToken
64         for _, trial := range []string{
65                 "http://keep-web/c=" + bogusID + "/foo",
66                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
67                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
68                 "http://keep-web/collections/" + bogusID + "/foo",
69                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
70                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
71         } {
72                 c.Log(trial)
73                 u := mustParseURL(trial)
74                 req := &http.Request{
75                         Method:     "GET",
76                         Host:       u.Host,
77                         URL:        u,
78                         RequestURI: u.RequestURI(),
79                 }
80                 resp := httptest.NewRecorder()
81                 cfg := DefaultConfig()
82                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
83                 h := handler{Config: cfg}
84                 h.ServeHTTP(resp, req)
85                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
86         }
87 }
88
89 func mustParseURL(s string) *url.URL {
90         r, err := url.Parse(s)
91         if err != nil {
92                 panic("parse URL: " + s)
93         }
94         return r
95 }
96
97 func (s *IntegrationSuite) TestVhost404(c *check.C) {
98         for _, testURL := range []string{
99                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
100                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
101         } {
102                 resp := httptest.NewRecorder()
103                 u := mustParseURL(testURL)
104                 req := &http.Request{
105                         Method:     "GET",
106                         URL:        u,
107                         RequestURI: u.RequestURI(),
108                 }
109                 s.testServer.Handler.ServeHTTP(resp, req)
110                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
111                 c.Check(resp.Body.String(), check.Equals, "")
112         }
113 }
114
115 // An authorizer modifies an HTTP request to make use of the given
116 // token -- by adding it to a header, cookie, query param, or whatever
117 // -- and returns the HTTP status code we should expect from keep-web if
118 // the token is invalid.
119 type authorizer func(*http.Request, string) int
120
121 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
122         s.doVhostRequests(c, authzViaAuthzHeader)
123 }
124 func authzViaAuthzHeader(r *http.Request, tok string) int {
125         r.Header.Add("Authorization", "OAuth2 "+tok)
126         return http.StatusUnauthorized
127 }
128
129 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
130         s.doVhostRequests(c, authzViaCookieValue)
131 }
132 func authzViaCookieValue(r *http.Request, tok string) int {
133         r.AddCookie(&http.Cookie{
134                 Name:  "arvados_api_token",
135                 Value: auth.EncodeTokenCookie([]byte(tok)),
136         })
137         return http.StatusUnauthorized
138 }
139
140 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
141         s.doVhostRequests(c, authzViaPath)
142 }
143 func authzViaPath(r *http.Request, tok string) int {
144         r.URL.Path = "/t=" + tok + r.URL.Path
145         return http.StatusNotFound
146 }
147
148 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
149         s.doVhostRequests(c, authzViaQueryString)
150 }
151 func authzViaQueryString(r *http.Request, tok string) int {
152         r.URL.RawQuery = "api_token=" + tok
153         return http.StatusUnauthorized
154 }
155
156 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
157         s.doVhostRequests(c, authzViaPOST)
158 }
159 func authzViaPOST(r *http.Request, tok string) int {
160         r.Method = "POST"
161         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
162         r.Body = ioutil.NopCloser(strings.NewReader(
163                 url.Values{"api_token": {tok}}.Encode()))
164         return http.StatusUnauthorized
165 }
166
167 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
168         s.doVhostRequests(c, authzViaPOST)
169 }
170 func authzViaXHRPOST(r *http.Request, tok string) int {
171         r.Method = "POST"
172         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
173         r.Header.Add("Origin", "https://origin.example")
174         r.Body = ioutil.NopCloser(strings.NewReader(
175                 url.Values{
176                         "api_token":   {tok},
177                         "disposition": {"attachment"},
178                 }.Encode()))
179         return http.StatusUnauthorized
180 }
181
182 // Try some combinations of {url, token} using the given authorization
183 // mechanism, and verify the result is correct.
184 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
185         for _, hostPath := range []string{
186                 arvadostest.FooCollection + ".example.com/foo",
187                 arvadostest.FooCollection + "--collections.example.com/foo",
188                 arvadostest.FooCollection + "--collections.example.com/_/foo",
189                 arvadostest.FooCollectionPDH + ".example.com/foo",
190                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
191                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
192         } {
193                 c.Log("doRequests: ", hostPath)
194                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
195         }
196 }
197
198 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
199         for _, tok := range []string{
200                 arvadostest.ActiveToken,
201                 arvadostest.ActiveToken[:15],
202                 arvadostest.SpectatorToken,
203                 "bogus",
204                 "",
205         } {
206                 u := mustParseURL("http://" + hostPath)
207                 req := &http.Request{
208                         Method:     "GET",
209                         Host:       u.Host,
210                         URL:        u,
211                         RequestURI: u.RequestURI(),
212                         Header:     http.Header{},
213                 }
214                 failCode := authz(req, tok)
215                 req, resp := s.doReq(req)
216                 code, body := resp.Code, resp.Body.String()
217
218                 // If the initial request had a (non-empty) token
219                 // showing in the query string, we should have been
220                 // redirected in order to hide it in a cookie.
221                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
222
223                 if tok == arvadostest.ActiveToken {
224                         c.Check(code, check.Equals, http.StatusOK)
225                         c.Check(body, check.Equals, "foo")
226
227                 } else {
228                         c.Check(code >= 400, check.Equals, true)
229                         c.Check(code < 500, check.Equals, true)
230                         if tok == arvadostest.SpectatorToken {
231                                 // Valid token never offers to retry
232                                 // with different credentials.
233                                 c.Check(code, check.Equals, http.StatusNotFound)
234                         } else {
235                                 // Invalid token can ask to retry
236                                 // depending on the authz method.
237                                 c.Check(code, check.Equals, failCode)
238                         }
239                         c.Check(body, check.Equals, "")
240                 }
241         }
242 }
243
244 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
245         resp := httptest.NewRecorder()
246         s.testServer.Handler.ServeHTTP(resp, req)
247         if resp.Code != http.StatusSeeOther {
248                 return req, resp
249         }
250         cookies := (&http.Response{Header: resp.Header()}).Cookies()
251         u, _ := req.URL.Parse(resp.Header().Get("Location"))
252         req = &http.Request{
253                 Method:     "GET",
254                 Host:       u.Host,
255                 URL:        u,
256                 RequestURI: u.RequestURI(),
257                 Header:     http.Header{},
258         }
259         for _, c := range cookies {
260                 req.AddCookie(c)
261         }
262         return s.doReq(req)
263 }
264
265 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
266         s.testVhostRedirectTokenToCookie(c, "GET",
267                 arvadostest.FooCollection+".example.com/foo",
268                 "?api_token="+arvadostest.ActiveToken,
269                 "",
270                 "",
271                 http.StatusOK,
272                 "foo",
273         )
274 }
275
276 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
277         s.testVhostRedirectTokenToCookie(c, "GET",
278                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
279                 "",
280                 "",
281                 "",
282                 http.StatusOK,
283                 "foo",
284         )
285 }
286
287 // Bad token in URL is 404 Not Found because it doesn't make sense to
288 // retry the same URL with different authorization.
289 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
290         s.testVhostRedirectTokenToCookie(c, "GET",
291                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
292                 "",
293                 "",
294                 "",
295                 http.StatusNotFound,
296                 "",
297         )
298 }
299
300 // Bad token in a cookie (even if it got there via our own
301 // query-string-to-cookie redirect) is, in principle, retryable at the
302 // same URL so it's 401 Unauthorized.
303 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
304         s.testVhostRedirectTokenToCookie(c, "GET",
305                 arvadostest.FooCollection+".example.com/foo",
306                 "?api_token=thisisabogustoken",
307                 "",
308                 "",
309                 http.StatusUnauthorized,
310                 "",
311         )
312 }
313
314 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
315         s.testVhostRedirectTokenToCookie(c, "GET",
316                 "example.com/c="+arvadostest.FooCollection+"/foo",
317                 "?api_token="+arvadostest.ActiveToken,
318                 "",
319                 "",
320                 http.StatusBadRequest,
321                 "",
322         )
323 }
324
325 // If client requests an attachment by putting ?disposition=attachment
326 // in the query string, and gets redirected, the redirect target
327 // should respond with an attachment.
328 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
329         resp := s.testVhostRedirectTokenToCookie(c, "GET",
330                 arvadostest.FooCollection+".example.com/foo",
331                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
332                 "",
333                 "",
334                 http.StatusOK,
335                 "foo",
336         )
337         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
338 }
339
340 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
341         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
342         resp := s.testVhostRedirectTokenToCookie(c, "GET",
343                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
344                 "?api_token="+arvadostest.ActiveToken,
345                 "",
346                 "",
347                 http.StatusOK,
348                 "foo",
349         )
350         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
351 }
352
353 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
354         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
355         resp := s.testVhostRedirectTokenToCookie(c, "GET",
356                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
357                 "?api_token="+arvadostest.ActiveToken,
358                 "",
359                 "",
360                 http.StatusOK,
361                 "waz",
362         )
363         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
364         resp = s.testVhostRedirectTokenToCookie(c, "GET",
365                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
366                 "?api_token="+arvadostest.ActiveToken,
367                 "",
368                 "",
369                 http.StatusOK,
370                 "waz",
371         )
372         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
373 }
374
375 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
376         s.testServer.Config.TrustAllContent = true
377         s.testVhostRedirectTokenToCookie(c, "GET",
378                 "example.com/c="+arvadostest.FooCollection+"/foo",
379                 "?api_token="+arvadostest.ActiveToken,
380                 "",
381                 "",
382                 http.StatusOK,
383                 "foo",
384         )
385 }
386
387 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
388         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
389
390         s.testVhostRedirectTokenToCookie(c, "GET",
391                 "example.com/c="+arvadostest.FooCollection+"/foo",
392                 "?api_token="+arvadostest.ActiveToken,
393                 "",
394                 "",
395                 http.StatusBadRequest,
396                 "",
397         )
398
399         resp := s.testVhostRedirectTokenToCookie(c, "GET",
400                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
401                 "?api_token="+arvadostest.ActiveToken,
402                 "",
403                 "",
404                 http.StatusOK,
405                 "foo",
406         )
407         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
408 }
409
410 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
411         s.testVhostRedirectTokenToCookie(c, "POST",
412                 arvadostest.FooCollection+".example.com/foo",
413                 "",
414                 "application/x-www-form-urlencoded",
415                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
416                 http.StatusOK,
417                 "foo",
418         )
419 }
420
421 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
422         s.testVhostRedirectTokenToCookie(c, "POST",
423                 arvadostest.FooCollection+".example.com/foo",
424                 "",
425                 "application/x-www-form-urlencoded",
426                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
427                 http.StatusNotFound,
428                 "",
429         )
430 }
431
432 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
433         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
434         s.testVhostRedirectTokenToCookie(c, "GET",
435                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
436                 "",
437                 "",
438                 "",
439                 http.StatusOK,
440                 "Hello world\n",
441         )
442 }
443
444 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
445         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
446         s.testVhostRedirectTokenToCookie(c, "GET",
447                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
448                 "",
449                 "",
450                 "",
451                 http.StatusNotFound,
452                 "",
453         )
454 }
455
456 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
457         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
458
459         client := s.testServer.Config.Client
460         client.AuthToken = arvadostest.ActiveToken
461         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
462         c.Assert(err, check.IsNil)
463         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
464         c.Assert(err, check.IsNil)
465         f.Close()
466         mtxt, err := fs.MarshalManifest(".")
467         c.Assert(err, check.IsNil)
468         coll := arvados.Collection{ManifestText: mtxt}
469         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
470         c.Assert(err, check.IsNil)
471
472         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
473         req := &http.Request{
474                 Method:     "GET",
475                 Host:       u.Host,
476                 URL:        u,
477                 RequestURI: u.RequestURI(),
478                 Header: http.Header{
479                         "Authorization": {"Bearer " + client.AuthToken},
480                 },
481         }
482         resp := httptest.NewRecorder()
483         s.testServer.Handler.ServeHTTP(resp, req)
484         c.Check(resp.Code, check.Equals, http.StatusOK)
485         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
486 }
487
488 // XHRs can't follow redirect-with-cookie so they rely on method=POST
489 // and disposition=attachment (telling us it's acceptable to respond
490 // with content instead of a redirect) and an Origin header that gets
491 // added automatically by the browser (telling us it's desirable to do
492 // so).
493 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
494         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
495         req := &http.Request{
496                 Method:     "POST",
497                 Host:       u.Host,
498                 URL:        u,
499                 RequestURI: u.RequestURI(),
500                 Header: http.Header{
501                         "Origin":       {"https://origin.example"},
502                         "Content-Type": {"application/x-www-form-urlencoded"},
503                 },
504                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
505                         "api_token":   {arvadostest.ActiveToken},
506                         "disposition": {"attachment"},
507                 }.Encode())),
508         }
509         resp := httptest.NewRecorder()
510         s.testServer.Handler.ServeHTTP(resp, req)
511         c.Check(resp.Code, check.Equals, http.StatusOK)
512         c.Check(resp.Body.String(), check.Equals, "foo")
513         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
514 }
515
516 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
517         u, _ := url.Parse(`http://` + hostPath + queryString)
518         req := &http.Request{
519                 Method:     method,
520                 Host:       u.Host,
521                 URL:        u,
522                 RequestURI: u.RequestURI(),
523                 Header:     http.Header{"Content-Type": {contentType}},
524                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
525         }
526
527         resp := httptest.NewRecorder()
528         defer func() {
529                 c.Check(resp.Code, check.Equals, expectStatus)
530                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
531         }()
532
533         s.testServer.Handler.ServeHTTP(resp, req)
534         if resp.Code != http.StatusSeeOther {
535                 return resp
536         }
537         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
538         cookies := (&http.Response{Header: resp.Header()}).Cookies()
539
540         u, _ = u.Parse(resp.Header().Get("Location"))
541         req = &http.Request{
542                 Method:     "GET",
543                 Host:       u.Host,
544                 URL:        u,
545                 RequestURI: u.RequestURI(),
546                 Header:     http.Header{},
547         }
548         for _, c := range cookies {
549                 req.AddCookie(c)
550         }
551
552         resp = httptest.NewRecorder()
553         s.testServer.Handler.ServeHTTP(resp, req)
554         c.Check(resp.Header().Get("Location"), check.Equals, "")
555         return resp
556 }
557
558 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
559         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
560         authHeader := http.Header{
561                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
562         }
563         for _, trial := range []struct {
564                 uri      string
565                 header   http.Header
566                 expect   []string
567                 redirect string
568                 cutDirs  int
569         }{
570                 {
571                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
572                         header:  authHeader,
573                         expect:  []string{"dir1/foo", "dir1/bar"},
574                         cutDirs: 0,
575                 },
576                 {
577                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
578                         header:  authHeader,
579                         expect:  []string{"foo", "bar"},
580                         cutDirs: 1,
581                 },
582                 {
583                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
584                         header:  authHeader,
585                         expect:  []string{"dir1/foo", "dir1/bar"},
586                         cutDirs: 2,
587                 },
588                 {
589                         uri:     "download.example.com/users/active/foo_file_in_dir/",
590                         header:  authHeader,
591                         expect:  []string{"dir1/"},
592                         cutDirs: 3,
593                 },
594                 {
595                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
596                         header:  authHeader,
597                         expect:  []string{"bar"},
598                         cutDirs: 4,
599                 },
600                 {
601                         uri:     "download.example.com/",
602                         header:  authHeader,
603                         expect:  []string{"users/"},
604                         cutDirs: 0,
605                 },
606                 {
607                         uri:      "download.example.com/users",
608                         header:   authHeader,
609                         redirect: "/users/",
610                         expect:   []string{"active/"},
611                         cutDirs:  1,
612                 },
613                 {
614                         uri:     "download.example.com/users/",
615                         header:  authHeader,
616                         expect:  []string{"active/"},
617                         cutDirs: 1,
618                 },
619                 {
620                         uri:      "download.example.com/users/active",
621                         header:   authHeader,
622                         redirect: "/users/active/",
623                         expect:   []string{"foo_file_in_dir/"},
624                         cutDirs:  2,
625                 },
626                 {
627                         uri:     "download.example.com/users/active/",
628                         header:  authHeader,
629                         expect:  []string{"foo_file_in_dir/"},
630                         cutDirs: 2,
631                 },
632                 {
633                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
634                         header:  nil,
635                         expect:  []string{"dir1/foo", "dir1/bar"},
636                         cutDirs: 4,
637                 },
638                 {
639                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
640                         header:  nil,
641                         expect:  []string{"dir1/foo", "dir1/bar"},
642                         cutDirs: 2,
643                 },
644                 {
645                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
646                         header:  nil,
647                         expect:  []string{"dir1/foo", "dir1/bar"},
648                         cutDirs: 2,
649                 },
650                 {
651                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
652                         header:  authHeader,
653                         expect:  []string{"dir1/foo", "dir1/bar"},
654                         cutDirs: 1,
655                 },
656                 {
657                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
658                         header:   authHeader,
659                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
660                         expect:   []string{"foo", "bar"},
661                         cutDirs:  2,
662                 },
663                 {
664                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
665                         header:  authHeader,
666                         expect:  []string{"foo", "bar"},
667                         cutDirs: 3,
668                 },
669                 {
670                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
671                         header:   authHeader,
672                         redirect: "/dir1/",
673                         expect:   []string{"foo", "bar"},
674                         cutDirs:  1,
675                 },
676                 {
677                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
678                         header: authHeader,
679                         expect: nil,
680                 },
681                 {
682                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
683                         header:  authHeader,
684                         expect:  []string{"waz"},
685                         cutDirs: 1,
686                 },
687                 {
688                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
689                         header:  authHeader,
690                         expect:  []string{"waz"},
691                         cutDirs: 2,
692                 },
693         } {
694                 c.Logf("HTML: %q => %q", trial.uri, trial.expect)
695                 resp := httptest.NewRecorder()
696                 u := mustParseURL("//" + trial.uri)
697                 req := &http.Request{
698                         Method:     "GET",
699                         Host:       u.Host,
700                         URL:        u,
701                         RequestURI: u.RequestURI(),
702                         Header:     copyHeader(trial.header),
703                 }
704                 s.testServer.Handler.ServeHTTP(resp, req)
705                 var cookies []*http.Cookie
706                 for resp.Code == http.StatusSeeOther {
707                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
708                         req = &http.Request{
709                                 Method:     "GET",
710                                 Host:       u.Host,
711                                 URL:        u,
712                                 RequestURI: u.RequestURI(),
713                                 Header:     copyHeader(trial.header),
714                         }
715                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
716                         for _, c := range cookies {
717                                 req.AddCookie(c)
718                         }
719                         resp = httptest.NewRecorder()
720                         s.testServer.Handler.ServeHTTP(resp, req)
721                 }
722                 if trial.redirect != "" {
723                         c.Check(req.URL.Path, check.Equals, trial.redirect)
724                 }
725                 if trial.expect == nil {
726                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
727                 } else {
728                         c.Check(resp.Code, check.Equals, http.StatusOK)
729                         for _, e := range trial.expect {
730                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
731                         }
732                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
733                 }
734
735                 c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
736                 req = &http.Request{
737                         Method:     "OPTIONS",
738                         Host:       u.Host,
739                         URL:        u,
740                         RequestURI: u.RequestURI(),
741                         Header:     copyHeader(trial.header),
742                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
743                 }
744                 resp = httptest.NewRecorder()
745                 s.testServer.Handler.ServeHTTP(resp, req)
746                 if trial.expect == nil {
747                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
748                 } else {
749                         c.Check(resp.Code, check.Equals, http.StatusOK)
750                 }
751
752                 req = &http.Request{
753                         Method:     "PROPFIND",
754                         Host:       u.Host,
755                         URL:        u,
756                         RequestURI: u.RequestURI(),
757                         Header:     copyHeader(trial.header),
758                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
759                 }
760                 resp = httptest.NewRecorder()
761                 s.testServer.Handler.ServeHTTP(resp, req)
762                 if trial.expect == nil {
763                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
764                 } else {
765                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
766                         for _, e := range trial.expect {
767                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
768                         }
769                 }
770         }
771 }
772
773 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
774         arv := arvados.NewClientFromEnv()
775         var newCollection arvados.Collection
776         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", arv.UpdateBody(&arvados.Collection{
777                 OwnerUUID:    arvadostest.ActiveUserUUID,
778                 ManifestText: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
779                 Name:         "keep-web test collection",
780         }), map[string]bool{"ensure_unique_name": true})
781         c.Assert(err, check.IsNil)
782         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
783
784         var updated arvados.Collection
785         for _, fnm := range []string{"foo.txt", "bar.txt"} {
786                 s.testServer.Config.AttachmentOnlyHost = "example.com"
787                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
788                 req := &http.Request{
789                         Method:     "DELETE",
790                         Host:       u.Host,
791                         URL:        u,
792                         RequestURI: u.RequestURI(),
793                         Header: http.Header{
794                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
795                         },
796                 }
797                 resp := httptest.NewRecorder()
798                 s.testServer.Handler.ServeHTTP(resp, req)
799                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
800
801                 updated = arvados.Collection{}
802                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
803                 c.Check(err, check.IsNil)
804                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
805                 c.Logf("updated manifest_text %q", updated.ManifestText)
806         }
807         c.Check(updated.ManifestText, check.Equals, "")
808 }
809
810 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
811         s.testServer.Config.ManagementToken = arvadostest.ManagementToken
812         authHeader := http.Header{
813                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
814         }
815
816         resp := httptest.NewRecorder()
817         u := mustParseURL("http://download.example.com/_health/ping")
818         req := &http.Request{
819                 Method:     "GET",
820                 Host:       u.Host,
821                 URL:        u,
822                 RequestURI: u.RequestURI(),
823                 Header:     authHeader,
824         }
825         s.testServer.Handler.ServeHTTP(resp, req)
826
827         c.Check(resp.Code, check.Equals, http.StatusOK)
828         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
829 }
830
831 func copyHeader(h http.Header) http.Header {
832         hc := http.Header{}
833         for k, v := range h {
834                 hc[k] = append([]string(nil), v...)
835         }
836         return hc
837 }