19889: Serve live logs via webdav.
[arvados.git] / services / keep-web / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package keepweb
6
7 import (
8         "bytes"
9         "context"
10         "fmt"
11         "html"
12         "io"
13         "io/ioutil"
14         "net/http"
15         "net/http/httptest"
16         "net/url"
17         "os"
18         "path/filepath"
19         "regexp"
20         "strings"
21         "time"
22
23         "git.arvados.org/arvados.git/lib/config"
24         "git.arvados.org/arvados.git/sdk/go/arvados"
25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
27         "git.arvados.org/arvados.git/sdk/go/auth"
28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
29         "git.arvados.org/arvados.git/sdk/go/keepclient"
30         "github.com/prometheus/client_golang/prometheus"
31         "github.com/sirupsen/logrus"
32         check "gopkg.in/check.v1"
33 )
34
35 var _ = check.Suite(&UnitSuite{})
36
37 func init() {
38         arvados.DebugLocksPanicMode = true
39 }
40
41 type UnitSuite struct {
42         cluster *arvados.Cluster
43         handler *handler
44 }
45
46 func (s *UnitSuite) SetUpTest(c *check.C) {
47         logger := ctxlog.TestLogger(c)
48         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
49         ldr.Path = "-"
50         cfg, err := ldr.Load()
51         c.Assert(err, check.IsNil)
52         cc, err := cfg.GetCluster("")
53         c.Assert(err, check.IsNil)
54         s.cluster = cc
55         s.handler = &handler{
56                 Cluster: cc,
57                 Cache: cache{
58                         cluster:  cc,
59                         logger:   logger,
60                         registry: prometheus.NewRegistry(),
61                 },
62         }
63 }
64
65 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
66         h := s.handler
67         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
68         req := &http.Request{
69                 Method:     "OPTIONS",
70                 Host:       u.Host,
71                 URL:        u,
72                 RequestURI: u.RequestURI(),
73                 Header: http.Header{
74                         "Origin":                        {"https://workbench.example"},
75                         "Access-Control-Request-Method": {"POST"},
76                 },
77         }
78
79         // Check preflight for an allowed request
80         resp := httptest.NewRecorder()
81         h.ServeHTTP(resp, req)
82         c.Check(resp.Code, check.Equals, http.StatusOK)
83         c.Check(resp.Body.String(), check.Equals, "")
84         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
85         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
86         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
87
88         // Check preflight for a disallowed request
89         resp = httptest.NewRecorder()
90         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
91         h.ServeHTTP(resp, req)
92         c.Check(resp.Body.String(), check.Equals, "")
93         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
94 }
95
96 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
97         for _, trial := range []struct {
98                 dataExists    bool
99                 sendIMSHeader bool
100                 expectStatus  int
101                 logRegexp     string
102         }{
103                 // If we return no content due to a Keep read error,
104                 // we should emit a log message.
105                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
106
107                 // If we return no content because the client sent an
108                 // If-Modified-Since header, our response should be
109                 // 304.  We still expect a "File download" log since it
110                 // counts as a file access for auditing.
111                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
112         } {
113                 c.Logf("trial: %+v", trial)
114                 arvadostest.StartKeep(2, true)
115                 if trial.dataExists {
116                         arv, err := arvadosclient.MakeArvadosClient()
117                         c.Assert(err, check.IsNil)
118                         arv.ApiToken = arvadostest.ActiveToken
119                         kc, err := keepclient.MakeKeepClient(arv)
120                         c.Assert(err, check.IsNil)
121                         _, _, err = kc.PutB([]byte("foo"))
122                         c.Assert(err, check.IsNil)
123                 }
124
125                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
126                 req := &http.Request{
127                         Method:     "GET",
128                         Host:       u.Host,
129                         URL:        u,
130                         RequestURI: u.RequestURI(),
131                         Header: http.Header{
132                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
133                         },
134                 }
135                 if trial.sendIMSHeader {
136                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
137                 }
138
139                 var logbuf bytes.Buffer
140                 logger := logrus.New()
141                 logger.Out = &logbuf
142                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
143
144                 resp := httptest.NewRecorder()
145                 s.handler.ServeHTTP(resp, req)
146                 c.Check(resp.Code, check.Equals, trial.expectStatus)
147                 c.Check(resp.Body.String(), check.Equals, "")
148
149                 c.Log(logbuf.String())
150                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
151         }
152 }
153
154 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
155         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
156         token := arvadostest.ActiveToken
157         for _, trial := range []string{
158                 "http://keep-web/c=" + bogusID + "/foo",
159                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
160                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
161                 "http://keep-web/collections/" + bogusID + "/foo",
162                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
163                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
164         } {
165                 c.Log(trial)
166                 u := mustParseURL(trial)
167                 req := &http.Request{
168                         Method:     "GET",
169                         Host:       u.Host,
170                         URL:        u,
171                         RequestURI: u.RequestURI(),
172                 }
173                 resp := httptest.NewRecorder()
174                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
175                 s.handler.ServeHTTP(resp, req)
176                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
177         }
178 }
179
180 func mustParseURL(s string) *url.URL {
181         r, err := url.Parse(s)
182         if err != nil {
183                 panic("parse URL: " + s)
184         }
185         return r
186 }
187
188 func (s *IntegrationSuite) TestVhost404(c *check.C) {
189         for _, testURL := range []string{
190                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
191                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
192         } {
193                 resp := httptest.NewRecorder()
194                 u := mustParseURL(testURL)
195                 req := &http.Request{
196                         Method:     "GET",
197                         URL:        u,
198                         RequestURI: u.RequestURI(),
199                 }
200                 s.handler.ServeHTTP(resp, req)
201                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
202                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
203         }
204 }
205
206 // An authorizer modifies an HTTP request to make use of the given
207 // token -- by adding it to a header, cookie, query param, or whatever
208 // -- and returns the HTTP status code we should expect from keep-web if
209 // the token is invalid.
210 type authorizer func(*http.Request, string) int
211
212 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
213         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
214 }
215 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
216         r.Header.Add("Authorization", "Bearer "+tok)
217         return http.StatusUnauthorized
218 }
219 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
220         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
221 }
222 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
223         r.Header.Add("Authorization", "Bearer "+tok)
224         return http.StatusUnauthorized
225 }
226
227 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
228         s.doVhostRequests(c, authzViaCookieValue)
229 }
230 func authzViaCookieValue(r *http.Request, tok string) int {
231         r.AddCookie(&http.Cookie{
232                 Name:  "arvados_api_token",
233                 Value: auth.EncodeTokenCookie([]byte(tok)),
234         })
235         return http.StatusUnauthorized
236 }
237
238 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
239         s.doVhostRequests(c, authzViaPath)
240 }
241 func authzViaPath(r *http.Request, tok string) int {
242         r.URL.Path = "/t=" + tok + r.URL.Path
243         return http.StatusNotFound
244 }
245
246 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
247         s.doVhostRequests(c, authzViaQueryString)
248 }
249 func authzViaQueryString(r *http.Request, tok string) int {
250         r.URL.RawQuery = "api_token=" + tok
251         return http.StatusUnauthorized
252 }
253
254 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
255         s.doVhostRequests(c, authzViaPOST)
256 }
257 func authzViaPOST(r *http.Request, tok string) int {
258         r.Method = "POST"
259         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
260         r.Body = ioutil.NopCloser(strings.NewReader(
261                 url.Values{"api_token": {tok}}.Encode()))
262         return http.StatusUnauthorized
263 }
264
265 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
266         s.doVhostRequests(c, authzViaPOST)
267 }
268 func authzViaXHRPOST(r *http.Request, tok string) int {
269         r.Method = "POST"
270         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
271         r.Header.Add("Origin", "https://origin.example")
272         r.Body = ioutil.NopCloser(strings.NewReader(
273                 url.Values{
274                         "api_token":   {tok},
275                         "disposition": {"attachment"},
276                 }.Encode()))
277         return http.StatusUnauthorized
278 }
279
280 // Try some combinations of {url, token} using the given authorization
281 // mechanism, and verify the result is correct.
282 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
283         for _, hostPath := range []string{
284                 arvadostest.FooCollection + ".example.com/foo",
285                 arvadostest.FooCollection + "--collections.example.com/foo",
286                 arvadostest.FooCollection + "--collections.example.com/_/foo",
287                 arvadostest.FooCollectionPDH + ".example.com/foo",
288                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
289                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
290         } {
291                 c.Log("doRequests: ", hostPath)
292                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
293         }
294 }
295
296 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
297         for _, tok := range []string{
298                 arvadostest.ActiveToken,
299                 arvadostest.ActiveToken[:15],
300                 arvadostest.SpectatorToken,
301                 "bogus",
302                 "",
303         } {
304                 u := mustParseURL("http://" + hostPath)
305                 req := &http.Request{
306                         Method:     "GET",
307                         Host:       u.Host,
308                         URL:        u,
309                         RequestURI: u.RequestURI(),
310                         Header:     http.Header{},
311                 }
312                 failCode := authz(req, tok)
313                 req, resp := s.doReq(req)
314                 code, body := resp.Code, resp.Body.String()
315
316                 // If the initial request had a (non-empty) token
317                 // showing in the query string, we should have been
318                 // redirected in order to hide it in a cookie.
319                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
320
321                 if tok == arvadostest.ActiveToken {
322                         c.Check(code, check.Equals, http.StatusOK)
323                         c.Check(body, check.Equals, "foo")
324                 } else {
325                         c.Check(code >= 400, check.Equals, true)
326                         c.Check(code < 500, check.Equals, true)
327                         if tok == arvadostest.SpectatorToken {
328                                 // Valid token never offers to retry
329                                 // with different credentials.
330                                 c.Check(code, check.Equals, http.StatusNotFound)
331                         } else {
332                                 // Invalid token can ask to retry
333                                 // depending on the authz method.
334                                 c.Check(code, check.Equals, failCode)
335                         }
336                         if code == 404 {
337                                 c.Check(body, check.Equals, notFoundMessage+"\n")
338                         } else {
339                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
340                         }
341                 }
342         }
343 }
344
345 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
346         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
347                 for _, port := range []string{"80", "443", "8000"} {
348                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
349                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
350                         req := &http.Request{
351                                 Method:     "GET",
352                                 Host:       u.Host,
353                                 URL:        u,
354                                 RequestURI: u.RequestURI(),
355                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
356                         }
357                         req, resp := s.doReq(req)
358                         code, _ := resp.Code, resp.Body.String()
359
360                         if port == "8000" {
361                                 c.Check(code, check.Equals, 401)
362                         } else {
363                                 c.Check(code, check.Equals, 200)
364                         }
365                 }
366         }
367 }
368
369 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
370         u := mustParseURL(urlstring)
371         if hdr == nil && token != "" {
372                 hdr = http.Header{"Authorization": {"Bearer " + token}}
373         } else if hdr == nil {
374                 hdr = http.Header{}
375         } else if token != "" {
376                 panic("must not pass both token and hdr")
377         }
378         return s.doReq(&http.Request{
379                 Method:     method,
380                 Host:       u.Host,
381                 URL:        u,
382                 RequestURI: u.RequestURI(),
383                 Header:     hdr,
384         })
385 }
386
387 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
388         resp := httptest.NewRecorder()
389         s.handler.ServeHTTP(resp, req)
390         if resp.Code != http.StatusSeeOther {
391                 return req, resp
392         }
393         cookies := (&http.Response{Header: resp.Header()}).Cookies()
394         u, _ := req.URL.Parse(resp.Header().Get("Location"))
395         req = &http.Request{
396                 Method:     "GET",
397                 Host:       u.Host,
398                 URL:        u,
399                 RequestURI: u.RequestURI(),
400                 Header:     http.Header{},
401         }
402         for _, c := range cookies {
403                 req.AddCookie(c)
404         }
405         return s.doReq(req)
406 }
407
408 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
409         s.testVhostRedirectTokenToCookie(c, "GET",
410                 arvadostest.FooCollection+".example.com/foo",
411                 "?api_token="+arvadostest.ActiveToken,
412                 nil,
413                 "",
414                 http.StatusOK,
415                 "foo",
416         )
417 }
418
419 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
420         s.testVhostRedirectTokenToCookie(c, "GET",
421                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
422                 "",
423                 nil,
424                 "",
425                 http.StatusOK,
426                 "foo",
427         )
428 }
429
430 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
431         s.testVhostRedirectTokenToCookie(c, "GET",
432                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
433                 "",
434                 nil,
435                 "",
436                 http.StatusOK,
437                 "foo",
438         )
439         // Same valid sharing token, but requesting a different collection
440         s.testVhostRedirectTokenToCookie(c, "GET",
441                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
442                 "",
443                 nil,
444                 "",
445                 http.StatusNotFound,
446                 regexp.QuoteMeta(notFoundMessage+"\n"),
447         )
448 }
449
450 // Bad token in URL is 404 Not Found because it doesn't make sense to
451 // retry the same URL with different authorization.
452 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
453         s.testVhostRedirectTokenToCookie(c, "GET",
454                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
455                 "",
456                 nil,
457                 "",
458                 http.StatusNotFound,
459                 regexp.QuoteMeta(notFoundMessage+"\n"),
460         )
461 }
462
463 // Bad token in a cookie (even if it got there via our own
464 // query-string-to-cookie redirect) is, in principle, retryable via
465 // wb2-login-and-redirect flow.
466 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
467         // Inline
468         resp := s.testVhostRedirectTokenToCookie(c, "GET",
469                 arvadostest.FooCollection+".example.com/foo",
470                 "?api_token=thisisabogustoken",
471                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
472                 "",
473                 http.StatusSeeOther,
474                 "",
475         )
476         u, err := url.Parse(resp.Header().Get("Location"))
477         c.Assert(err, check.IsNil)
478         c.Logf("redirected to %s", u)
479         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
480         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
481         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
482
483         // Download/attachment indicated by ?disposition=attachment
484         resp = s.testVhostRedirectTokenToCookie(c, "GET",
485                 arvadostest.FooCollection+".example.com/foo",
486                 "?api_token=thisisabogustoken&disposition=attachment",
487                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
488                 "",
489                 http.StatusSeeOther,
490                 "",
491         )
492         u, err = url.Parse(resp.Header().Get("Location"))
493         c.Assert(err, check.IsNil)
494         c.Logf("redirected to %s", u)
495         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
496         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
497         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
498
499         // Download/attachment indicated by vhost
500         resp = s.testVhostRedirectTokenToCookie(c, "GET",
501                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
502                 "?api_token=thisisabogustoken",
503                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
504                 "",
505                 http.StatusSeeOther,
506                 "",
507         )
508         u, err = url.Parse(resp.Header().Get("Location"))
509         c.Assert(err, check.IsNil)
510         c.Logf("redirected to %s", u)
511         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
512         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
513         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
514
515         // Without "Sec-Fetch-Mode: navigate" header, just 401.
516         s.testVhostRedirectTokenToCookie(c, "GET",
517                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
518                 "?api_token=thisisabogustoken",
519                 http.Header{"Sec-Fetch-Mode": {"cors"}},
520                 "",
521                 http.StatusUnauthorized,
522                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
523         )
524         s.testVhostRedirectTokenToCookie(c, "GET",
525                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
526                 "?api_token=thisisabogustoken",
527                 nil,
528                 "",
529                 http.StatusUnauthorized,
530                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
531         )
532 }
533
534 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
535         resp := s.testVhostRedirectTokenToCookie(c, "GET",
536                 arvadostest.FooCollection+".example.com/foo",
537                 "?api_token=thisisabogustoken",
538                 http.Header{
539                         "Sec-Fetch-Mode": {"navigate"},
540                         "Cache-Control":  {"no-cache"},
541                 },
542                 "",
543                 http.StatusSeeOther,
544                 "",
545         )
546         u, err := url.Parse(resp.Header().Get("Location"))
547         c.Assert(err, check.IsNil)
548         c.Logf("redirected to %s", u)
549         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
550         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
551         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
552 }
553
554 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
555         for _, trial := range []struct {
556                 anonToken    bool
557                 cacheControl string
558         }{
559                 {},
560                 {cacheControl: "no-cache"},
561                 {anonToken: true},
562                 {anonToken: true, cacheControl: "no-cache"},
563         } {
564                 c.Logf("trial: %+v", trial)
565
566                 if trial.anonToken {
567                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
568                 } else {
569                         s.handler.Cluster.Users.AnonymousUserToken = ""
570                 }
571                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
572                 c.Assert(err, check.IsNil)
573                 req.Header.Set("Sec-Fetch-Mode", "navigate")
574                 if trial.cacheControl != "" {
575                         req.Header.Set("Cache-Control", trial.cacheControl)
576                 }
577                 resp := httptest.NewRecorder()
578                 s.handler.ServeHTTP(resp, req)
579                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
580                 u, err := url.Parse(resp.Header().Get("Location"))
581                 c.Assert(err, check.IsNil)
582                 c.Logf("redirected to %q", u)
583                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
584                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
585                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
586         }
587 }
588
589 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
590         s.testVhostRedirectTokenToCookie(c, "GET",
591                 "example.com/c="+arvadostest.FooCollection+"/foo",
592                 "?api_token="+arvadostest.ActiveToken,
593                 nil,
594                 "",
595                 http.StatusBadRequest,
596                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
597         )
598 }
599
600 // If client requests an attachment by putting ?disposition=attachment
601 // in the query string, and gets redirected, the redirect target
602 // should respond with an attachment.
603 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
604         resp := s.testVhostRedirectTokenToCookie(c, "GET",
605                 arvadostest.FooCollection+".example.com/foo",
606                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
607                 nil,
608                 "",
609                 http.StatusOK,
610                 "foo",
611         )
612         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
613 }
614
615 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
616         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
617         resp := s.testVhostRedirectTokenToCookie(c, "GET",
618                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
619                 "?api_token="+arvadostest.ActiveToken,
620                 nil,
621                 "",
622                 http.StatusOK,
623                 "foo",
624         )
625         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
626 }
627
628 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
629         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
630         resp := s.testVhostRedirectTokenToCookie(c, "GET",
631                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
632                 "?api_token="+arvadostest.ActiveToken,
633                 nil,
634                 "",
635                 http.StatusOK,
636                 "waz",
637         )
638         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
639         resp = s.testVhostRedirectTokenToCookie(c, "GET",
640                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
641                 "?api_token="+arvadostest.ActiveToken,
642                 nil,
643                 "",
644                 http.StatusOK,
645                 "waz",
646         )
647         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
648 }
649
650 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
651         s.handler.Cluster.Collections.TrustAllContent = true
652         s.testVhostRedirectTokenToCookie(c, "GET",
653                 "example.com/c="+arvadostest.FooCollection+"/foo",
654                 "?api_token="+arvadostest.ActiveToken,
655                 nil,
656                 "",
657                 http.StatusOK,
658                 "foo",
659         )
660 }
661
662 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
663         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
664
665         s.testVhostRedirectTokenToCookie(c, "GET",
666                 "example.com/c="+arvadostest.FooCollection+"/foo",
667                 "?api_token="+arvadostest.ActiveToken,
668                 nil,
669                 "",
670                 http.StatusBadRequest,
671                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
672         )
673
674         resp := s.testVhostRedirectTokenToCookie(c, "GET",
675                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
676                 "?api_token="+arvadostest.ActiveToken,
677                 nil,
678                 "",
679                 http.StatusOK,
680                 "foo",
681         )
682         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
683 }
684
685 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
686         s.testVhostRedirectTokenToCookie(c, "POST",
687                 arvadostest.FooCollection+".example.com/foo",
688                 "",
689                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
690                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
691                 http.StatusOK,
692                 "foo",
693         )
694 }
695
696 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
697         s.testVhostRedirectTokenToCookie(c, "POST",
698                 arvadostest.FooCollection+".example.com/foo",
699                 "",
700                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
701                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
702                 http.StatusNotFound,
703                 regexp.QuoteMeta(notFoundMessage+"\n"),
704         )
705 }
706
707 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
708         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
709         s.testVhostRedirectTokenToCookie(c, "GET",
710                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
711                 "",
712                 nil,
713                 "",
714                 http.StatusOK,
715                 "Hello world\n",
716         )
717 }
718
719 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
720         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
721         s.testVhostRedirectTokenToCookie(c, "GET",
722                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
723                 "",
724                 nil,
725                 "",
726                 http.StatusUnauthorized,
727                 "Authorization tokens are not accepted here: .*\n",
728         )
729 }
730
731 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
732         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
733
734         client := arvados.NewClientFromEnv()
735         client.AuthToken = arvadostest.ActiveToken
736         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
737         c.Assert(err, check.IsNil)
738         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
739         c.Assert(err, check.IsNil)
740         f.Close()
741         mtxt, err := fs.MarshalManifest(".")
742         c.Assert(err, check.IsNil)
743         var coll arvados.Collection
744         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
745                 "collection": map[string]string{
746                         "manifest_text": mtxt,
747                 },
748         })
749         c.Assert(err, check.IsNil)
750
751         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
752         req := &http.Request{
753                 Method:     "GET",
754                 Host:       u.Host,
755                 URL:        u,
756                 RequestURI: u.RequestURI(),
757                 Header: http.Header{
758                         "Authorization": {"Bearer " + client.AuthToken},
759                 },
760         }
761         resp := httptest.NewRecorder()
762         s.handler.ServeHTTP(resp, req)
763         c.Check(resp.Code, check.Equals, http.StatusOK)
764         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
765 }
766
767 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
768         arv := arvados.NewClientFromEnv()
769         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
770         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
771         name := "foo/bar/baz"
772         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
773         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
774
775         client := arvados.NewClientFromEnv()
776         client.AuthToken = arvadostest.ActiveToken
777         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
778         c.Assert(err, check.IsNil)
779         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
780         c.Assert(err, check.IsNil)
781         f.Close()
782         mtxt, err := fs.MarshalManifest(".")
783         c.Assert(err, check.IsNil)
784         var coll arvados.Collection
785         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
786                 "collection": map[string]string{
787                         "manifest_text": mtxt,
788                         "name":          name,
789                         "owner_uuid":    arvadostest.AProjectUUID,
790                 },
791         })
792         c.Assert(err, check.IsNil)
793         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
794
795         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
796         for tryURL, expectRegexp := range map[string]string{
797                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
798                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
799         } {
800                 u, _ := url.Parse(tryURL)
801                 req := &http.Request{
802                         Method:     "GET",
803                         Host:       u.Host,
804                         URL:        u,
805                         RequestURI: u.RequestURI(),
806                         Header: http.Header{
807                                 "Authorization": {"Bearer " + client.AuthToken},
808                         },
809                 }
810                 resp := httptest.NewRecorder()
811                 s.handler.ServeHTTP(resp, req)
812                 c.Check(resp.Code, check.Equals, http.StatusOK)
813                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
814         }
815 }
816
817 // XHRs can't follow redirect-with-cookie so they rely on method=POST
818 // and disposition=attachment (telling us it's acceptable to respond
819 // with content instead of a redirect) and an Origin header that gets
820 // added automatically by the browser (telling us it's desirable to do
821 // so).
822 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
823         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
824         req := &http.Request{
825                 Method:     "POST",
826                 Host:       u.Host,
827                 URL:        u,
828                 RequestURI: u.RequestURI(),
829                 Header: http.Header{
830                         "Origin":       {"https://origin.example"},
831                         "Content-Type": {"application/x-www-form-urlencoded"},
832                 },
833                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
834                         "api_token":   {arvadostest.ActiveToken},
835                         "disposition": {"attachment"},
836                 }.Encode())),
837         }
838         resp := httptest.NewRecorder()
839         s.handler.ServeHTTP(resp, req)
840         c.Check(resp.Code, check.Equals, http.StatusOK)
841         c.Check(resp.Body.String(), check.Equals, "foo")
842         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
843
844         // GET + Origin header is representative of both AJAX GET
845         // requests and inline images via <IMG crossorigin="anonymous"
846         // src="...">.
847         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
848         req = &http.Request{
849                 Method:     "GET",
850                 Host:       u.Host,
851                 URL:        u,
852                 RequestURI: u.RequestURI(),
853                 Header: http.Header{
854                         "Origin": {"https://origin.example"},
855                 },
856         }
857         resp = httptest.NewRecorder()
858         s.handler.ServeHTTP(resp, req)
859         c.Check(resp.Code, check.Equals, http.StatusOK)
860         c.Check(resp.Body.String(), check.Equals, "foo")
861         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
862 }
863
864 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
865         if reqHeader == nil {
866                 reqHeader = http.Header{}
867         }
868         u, _ := url.Parse(`http://` + hostPath + queryString)
869         c.Logf("requesting %s", u)
870         req := &http.Request{
871                 Method:     method,
872                 Host:       u.Host,
873                 URL:        u,
874                 RequestURI: u.RequestURI(),
875                 Header:     reqHeader,
876                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
877         }
878
879         resp := httptest.NewRecorder()
880         defer func() {
881                 c.Check(resp.Code, check.Equals, expectStatus)
882                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
883         }()
884
885         s.handler.ServeHTTP(resp, req)
886         if resp.Code != http.StatusSeeOther {
887                 return resp
888         }
889         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
890         c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
891         cookies := (&http.Response{Header: resp.Header()}).Cookies()
892
893         u, err := u.Parse(resp.Header().Get("Location"))
894         c.Assert(err, check.IsNil)
895         c.Logf("following redirect to %s", u)
896         req = &http.Request{
897                 Method:     "GET",
898                 Host:       u.Host,
899                 URL:        u,
900                 RequestURI: u.RequestURI(),
901                 Header:     reqHeader,
902         }
903         for _, c := range cookies {
904                 req.AddCookie(c)
905         }
906
907         resp = httptest.NewRecorder()
908         s.handler.ServeHTTP(resp, req)
909
910         if resp.Code != http.StatusSeeOther {
911                 c.Check(resp.Header().Get("Location"), check.Equals, "")
912         }
913         return resp
914 }
915
916 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
917         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
918         s.testDirectoryListing(c)
919 }
920
921 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
922         s.handler.Cluster.Users.AnonymousUserToken = ""
923         s.testDirectoryListing(c)
924 }
925
926 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
927         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
928         authHeader := http.Header{
929                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
930         }
931         for _, trial := range []struct {
932                 uri      string
933                 header   http.Header
934                 expect   []string
935                 redirect string
936                 cutDirs  int
937         }{
938                 {
939                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
940                         header:  authHeader,
941                         expect:  []string{"dir1/foo", "dir1/bar"},
942                         cutDirs: 0,
943                 },
944                 {
945                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
946                         header:  authHeader,
947                         expect:  []string{"foo", "bar"},
948                         cutDirs: 1,
949                 },
950                 {
951                         // URLs of this form ignore authHeader, and
952                         // FooAndBarFilesInDirUUID isn't public, so
953                         // this returns 401.
954                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
955                         header: authHeader,
956                         expect: nil,
957                 },
958                 {
959                         uri:     "download.example.com/users/active/foo_file_in_dir/",
960                         header:  authHeader,
961                         expect:  []string{"dir1/"},
962                         cutDirs: 3,
963                 },
964                 {
965                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
966                         header:  authHeader,
967                         expect:  []string{"bar"},
968                         cutDirs: 4,
969                 },
970                 {
971                         uri:     "download.example.com/",
972                         header:  authHeader,
973                         expect:  []string{"users/"},
974                         cutDirs: 0,
975                 },
976                 {
977                         uri:      "download.example.com/users",
978                         header:   authHeader,
979                         redirect: "/users/",
980                         expect:   []string{"active/"},
981                         cutDirs:  1,
982                 },
983                 {
984                         uri:     "download.example.com/users/",
985                         header:  authHeader,
986                         expect:  []string{"active/"},
987                         cutDirs: 1,
988                 },
989                 {
990                         uri:      "download.example.com/users/active",
991                         header:   authHeader,
992                         redirect: "/users/active/",
993                         expect:   []string{"foo_file_in_dir/"},
994                         cutDirs:  2,
995                 },
996                 {
997                         uri:     "download.example.com/users/active/",
998                         header:  authHeader,
999                         expect:  []string{"foo_file_in_dir/"},
1000                         cutDirs: 2,
1001                 },
1002                 {
1003                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1004                         header:  nil,
1005                         expect:  []string{"dir1/foo", "dir1/bar"},
1006                         cutDirs: 4,
1007                 },
1008                 {
1009                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1010                         header:  nil,
1011                         expect:  []string{"dir1/foo", "dir1/bar"},
1012                         cutDirs: 2,
1013                 },
1014                 {
1015                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1016                         header:  nil,
1017                         expect:  []string{"dir1/foo", "dir1/bar"},
1018                         cutDirs: 2,
1019                 },
1020                 {
1021                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1022                         header:  authHeader,
1023                         expect:  []string{"dir1/foo", "dir1/bar"},
1024                         cutDirs: 1,
1025                 },
1026                 {
1027                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1028                         header:   authHeader,
1029                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1030                         expect:   []string{"foo", "bar"},
1031                         cutDirs:  2,
1032                 },
1033                 {
1034                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1035                         header:  authHeader,
1036                         expect:  []string{"foo", "bar"},
1037                         cutDirs: 3,
1038                 },
1039                 {
1040                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1041                         header:   authHeader,
1042                         redirect: "/dir1/",
1043                         expect:   []string{"foo", "bar"},
1044                         cutDirs:  1,
1045                 },
1046                 {
1047                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1048                         header: authHeader,
1049                         expect: nil,
1050                 },
1051                 {
1052                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1053                         header:  authHeader,
1054                         expect:  []string{"waz"},
1055                         cutDirs: 1,
1056                 },
1057                 {
1058                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1059                         header:  authHeader,
1060                         expect:  []string{"waz"},
1061                         cutDirs: 2,
1062                 },
1063         } {
1064                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
1065                 resp := httptest.NewRecorder()
1066                 u := mustParseURL("//" + trial.uri)
1067                 req := &http.Request{
1068                         Method:     "GET",
1069                         Host:       u.Host,
1070                         URL:        u,
1071                         RequestURI: u.RequestURI(),
1072                         Header:     copyHeader(trial.header),
1073                 }
1074                 s.handler.ServeHTTP(resp, req)
1075                 var cookies []*http.Cookie
1076                 for resp.Code == http.StatusSeeOther {
1077                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1078                         req = &http.Request{
1079                                 Method:     "GET",
1080                                 Host:       u.Host,
1081                                 URL:        u,
1082                                 RequestURI: u.RequestURI(),
1083                                 Header:     copyHeader(trial.header),
1084                         }
1085                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1086                         for _, c := range cookies {
1087                                 req.AddCookie(c)
1088                         }
1089                         resp = httptest.NewRecorder()
1090                         s.handler.ServeHTTP(resp, req)
1091                 }
1092                 if trial.redirect != "" {
1093                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1094                 }
1095                 if trial.expect == nil {
1096                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1097                 } else {
1098                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1099                         for _, e := range trial.expect {
1100                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1101                         }
1102                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1103                 }
1104
1105                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1106                 req = &http.Request{
1107                         Method:     "OPTIONS",
1108                         Host:       u.Host,
1109                         URL:        u,
1110                         RequestURI: u.RequestURI(),
1111                         Header:     copyHeader(trial.header),
1112                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1113                 }
1114                 resp = httptest.NewRecorder()
1115                 s.handler.ServeHTTP(resp, req)
1116                 if trial.expect == nil {
1117                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1118                 } else {
1119                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1120                 }
1121
1122                 req = &http.Request{
1123                         Method:     "PROPFIND",
1124                         Host:       u.Host,
1125                         URL:        u,
1126                         RequestURI: u.RequestURI(),
1127                         Header:     copyHeader(trial.header),
1128                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1129                 }
1130                 resp = httptest.NewRecorder()
1131                 s.handler.ServeHTTP(resp, req)
1132                 if trial.expect == nil {
1133                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1134                 } else {
1135                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1136                         for _, e := range trial.expect {
1137                                 if strings.HasSuffix(e, "/") {
1138                                         e = filepath.Join(u.Path, e) + "/"
1139                                 } else {
1140                                         e = filepath.Join(u.Path, e)
1141                                 }
1142                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1143                         }
1144                 }
1145         }
1146 }
1147
1148 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1149         arv := arvados.NewClientFromEnv()
1150         var newCollection arvados.Collection
1151         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1152                 "collection": map[string]string{
1153                         "owner_uuid":    arvadostest.ActiveUserUUID,
1154                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1155                         "name":          "keep-web test collection",
1156                 },
1157                 "ensure_unique_name": true,
1158         })
1159         c.Assert(err, check.IsNil)
1160         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1161
1162         var updated arvados.Collection
1163         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1164                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1165                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1166                 req := &http.Request{
1167                         Method:     "DELETE",
1168                         Host:       u.Host,
1169                         URL:        u,
1170                         RequestURI: u.RequestURI(),
1171                         Header: http.Header{
1172                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1173                         },
1174                 }
1175                 resp := httptest.NewRecorder()
1176                 s.handler.ServeHTTP(resp, req)
1177                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1178
1179                 updated = arvados.Collection{}
1180                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1181                 c.Check(err, check.IsNil)
1182                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1183                 c.Logf("updated manifest_text %q", updated.ManifestText)
1184         }
1185         c.Check(updated.ManifestText, check.Equals, "")
1186 }
1187
1188 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1189         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1190
1191         client := arvados.NewClientFromEnv()
1192         client.AuthToken = arvadostest.ActiveToken
1193         arv, err := arvadosclient.New(client)
1194         c.Assert(err, check.Equals, nil)
1195         kc, err := keepclient.MakeKeepClient(arv)
1196         c.Assert(err, check.Equals, nil)
1197
1198         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1199         c.Assert(err, check.IsNil)
1200
1201         trials := []struct {
1202                 filename    string
1203                 content     string
1204                 contentType string
1205         }{
1206                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1207                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1208                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1209                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1210                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1211         }
1212         for _, trial := range trials {
1213                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1214                 c.Assert(err, check.IsNil)
1215                 _, err = f.Write([]byte(trial.content))
1216                 c.Assert(err, check.IsNil)
1217                 c.Assert(f.Close(), check.IsNil)
1218         }
1219         mtxt, err := fs.MarshalManifest(".")
1220         c.Assert(err, check.IsNil)
1221         var coll arvados.Collection
1222         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1223                 "collection": map[string]string{
1224                         "manifest_text": mtxt,
1225                 },
1226         })
1227         c.Assert(err, check.IsNil)
1228
1229         for _, trial := range trials {
1230                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1231                 req := &http.Request{
1232                         Method:     "GET",
1233                         Host:       u.Host,
1234                         URL:        u,
1235                         RequestURI: u.RequestURI(),
1236                         Header: http.Header{
1237                                 "Authorization": {"Bearer " + client.AuthToken},
1238                         },
1239                 }
1240                 resp := httptest.NewRecorder()
1241                 s.handler.ServeHTTP(resp, req)
1242                 c.Check(resp.Code, check.Equals, http.StatusOK)
1243                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1244                 c.Check(resp.Body.String(), check.Equals, trial.content)
1245         }
1246 }
1247
1248 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1249         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1250         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1251         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1252         req := &http.Request{
1253                 Method:     "GET",
1254                 Host:       u.Host,
1255                 URL:        u,
1256                 RequestURI: u.RequestURI(),
1257         }
1258         resp := httptest.NewRecorder()
1259         s.handler.ServeHTTP(resp, req)
1260         c.Check(resp.Code, check.Equals, http.StatusOK)
1261         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1262 }
1263
1264 // Writing to a collection shouldn't affect its entry in the
1265 // PDH-to-manifest cache.
1266 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1267         arv, err := arvadosclient.MakeArvadosClient()
1268         c.Assert(err, check.Equals, nil)
1269         arv.ApiToken = arvadostest.ActiveToken
1270
1271         u := mustParseURL("http://x.example/testfile")
1272         req := &http.Request{
1273                 Method:     "GET",
1274                 Host:       u.Host,
1275                 URL:        u,
1276                 RequestURI: u.RequestURI(),
1277                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1278         }
1279
1280         checkWithID := func(id string, status int) {
1281                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1282                 req.Host = req.URL.Host
1283                 resp := httptest.NewRecorder()
1284                 s.handler.ServeHTTP(resp, req)
1285                 c.Check(resp.Code, check.Equals, status)
1286         }
1287
1288         var colls [2]arvados.Collection
1289         for i := range colls {
1290                 err := arv.Create("collections",
1291                         map[string]interface{}{
1292                                 "ensure_unique_name": true,
1293                                 "collection": map[string]interface{}{
1294                                         "name": "test collection",
1295                                 },
1296                         }, &colls[i])
1297                 c.Assert(err, check.Equals, nil)
1298         }
1299
1300         // Populate cache with empty collection
1301         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1302
1303         // write a file to colls[0]
1304         reqPut := *req
1305         reqPut.Method = "PUT"
1306         reqPut.URL.Host = colls[0].UUID + ".example"
1307         reqPut.Host = req.URL.Host
1308         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1309         resp := httptest.NewRecorder()
1310         s.handler.ServeHTTP(resp, &reqPut)
1311         c.Check(resp.Code, check.Equals, http.StatusCreated)
1312
1313         // new file should not appear in colls[1]
1314         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1315         checkWithID(colls[1].UUID, http.StatusNotFound)
1316
1317         checkWithID(colls[0].UUID, http.StatusOK)
1318 }
1319
1320 func copyHeader(h http.Header) http.Header {
1321         hc := http.Header{}
1322         for k, v := range h {
1323                 hc[k] = append([]string(nil), v...)
1324         }
1325         return hc
1326 }
1327
1328 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1329         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1330
1331         client := arvados.NewClientFromEnv()
1332         client.AuthToken = arvadostest.AdminToken
1333         var logentries arvados.LogList
1334         limit1 := 1
1335         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1336                 arvados.ResourceListParams{
1337                         Limit: &limit1,
1338                         Order: "created_at desc"})
1339         c.Check(err, check.IsNil)
1340         c.Check(logentries.Items, check.HasLen, 1)
1341         lastLogId := logentries.Items[0].ID
1342         c.Logf("lastLogId: %d", lastLogId)
1343
1344         var logbuf bytes.Buffer
1345         logger := logrus.New()
1346         logger.Out = &logbuf
1347         resp := httptest.NewRecorder()
1348         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1349         s.handler.ServeHTTP(resp, req)
1350
1351         if perm {
1352                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1353                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1354                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1355
1356                 deadline := time.Now().Add(time.Second)
1357                 for {
1358                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1359                         logentries = arvados.LogList{}
1360                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1361                                 arvados.ResourceListParams{
1362                                         Filters: []arvados.Filter{
1363                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1364                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1365                                         },
1366                                         Limit: &limit1,
1367                                         Order: "created_at desc",
1368                                 })
1369                         c.Assert(err, check.IsNil)
1370                         if len(logentries.Items) > 0 &&
1371                                 logentries.Items[0].ID > lastLogId &&
1372                                 logentries.Items[0].ObjectUUID == userUuid &&
1373                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1374                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1375                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1376                                 break
1377                         }
1378                         c.Logf("logentries.Items: %+v", logentries.Items)
1379                         time.Sleep(50 * time.Millisecond)
1380                 }
1381         } else {
1382                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1383                 c.Check(logbuf.String(), check.Equals, "")
1384         }
1385 }
1386
1387 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1388         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1389
1390         s.handler.Cluster.Collections.TrustAllContent = true
1391
1392         for _, adminperm := range []bool{true, false} {
1393                 for _, userperm := range []bool{true, false} {
1394                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1395                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1396
1397                         // Test admin permission
1398                         req := &http.Request{
1399                                 Method:     "GET",
1400                                 Host:       u.Host,
1401                                 URL:        u,
1402                                 RequestURI: u.RequestURI(),
1403                                 Header: http.Header{
1404                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1405                                 },
1406                         }
1407                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1408                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1409
1410                         // Test user permission
1411                         req = &http.Request{
1412                                 Method:     "GET",
1413                                 Host:       u.Host,
1414                                 URL:        u,
1415                                 RequestURI: u.RequestURI(),
1416                                 Header: http.Header{
1417                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1418                                 },
1419                         }
1420                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1421                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1422                 }
1423         }
1424
1425         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1426
1427         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1428                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1429
1430                 u = mustParseURL(tryurl)
1431                 req := &http.Request{
1432                         Method:     "GET",
1433                         Host:       u.Host,
1434                         URL:        u,
1435                         RequestURI: u.RequestURI(),
1436                         Header: http.Header{
1437                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1438                         },
1439                 }
1440                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1441                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1442         }
1443
1444         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1445         req := &http.Request{
1446                 Method:     "GET",
1447                 Host:       u.Host,
1448                 URL:        u,
1449                 RequestURI: u.RequestURI(),
1450                 Header: http.Header{
1451                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1452                 },
1453         }
1454         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1455                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1456 }
1457
1458 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1459         for _, adminperm := range []bool{true, false} {
1460                 for _, userperm := range []bool{true, false} {
1461
1462                         arv := arvados.NewClientFromEnv()
1463                         arv.AuthToken = arvadostest.ActiveToken
1464
1465                         var coll arvados.Collection
1466                         err := arv.RequestAndDecode(&coll,
1467                                 "POST",
1468                                 "/arvados/v1/collections",
1469                                 nil,
1470                                 map[string]interface{}{
1471                                         "ensure_unique_name": true,
1472                                         "collection": map[string]interface{}{
1473                                                 "name": "test collection",
1474                                         },
1475                                 })
1476                         c.Assert(err, check.Equals, nil)
1477
1478                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1479
1480                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1481                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1482
1483                         // Test admin permission
1484                         req := &http.Request{
1485                                 Method:     "PUT",
1486                                 Host:       u.Host,
1487                                 URL:        u,
1488                                 RequestURI: u.RequestURI(),
1489                                 Header: http.Header{
1490                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1491                                 },
1492                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1493                         }
1494                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1495                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1496
1497                         // Test user permission
1498                         req = &http.Request{
1499                                 Method:     "PUT",
1500                                 Host:       u.Host,
1501                                 URL:        u,
1502                                 RequestURI: u.RequestURI(),
1503                                 Header: http.Header{
1504                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1505                                 },
1506                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1507                         }
1508                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1509                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1510                 }
1511         }
1512 }