19015: Merge branch 'main' into 19015-rvm-gemset-passenger-fix
[arvados.git] / services / keep-web / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package keepweb
6
7 import (
8         "bytes"
9         "context"
10         "fmt"
11         "html"
12         "io"
13         "io/ioutil"
14         "net/http"
15         "net/http/httptest"
16         "net/url"
17         "os"
18         "path/filepath"
19         "regexp"
20         "strings"
21         "time"
22
23         "git.arvados.org/arvados.git/lib/config"
24         "git.arvados.org/arvados.git/sdk/go/arvados"
25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
27         "git.arvados.org/arvados.git/sdk/go/auth"
28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
29         "git.arvados.org/arvados.git/sdk/go/keepclient"
30         "github.com/prometheus/client_golang/prometheus"
31         "github.com/sirupsen/logrus"
32         check "gopkg.in/check.v1"
33 )
34
35 var _ = check.Suite(&UnitSuite{})
36
37 func init() {
38         arvados.DebugLocksPanicMode = true
39 }
40
41 type UnitSuite struct {
42         cluster *arvados.Cluster
43         handler *handler
44 }
45
46 func (s *UnitSuite) SetUpTest(c *check.C) {
47         logger := ctxlog.TestLogger(c)
48         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
49         ldr.Path = "-"
50         cfg, err := ldr.Load()
51         c.Assert(err, check.IsNil)
52         cc, err := cfg.GetCluster("")
53         c.Assert(err, check.IsNil)
54         s.cluster = cc
55         s.handler = &handler{
56                 Cluster: cc,
57                 Cache: cache{
58                         cluster:  cc,
59                         logger:   logger,
60                         registry: prometheus.NewRegistry(),
61                 },
62         }
63 }
64
65 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
66         h := s.handler
67         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
68         req := &http.Request{
69                 Method:     "OPTIONS",
70                 Host:       u.Host,
71                 URL:        u,
72                 RequestURI: u.RequestURI(),
73                 Header: http.Header{
74                         "Origin":                        {"https://workbench.example"},
75                         "Access-Control-Request-Method": {"POST"},
76                 },
77         }
78
79         // Check preflight for an allowed request
80         resp := httptest.NewRecorder()
81         h.ServeHTTP(resp, req)
82         c.Check(resp.Code, check.Equals, http.StatusOK)
83         c.Check(resp.Body.String(), check.Equals, "")
84         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
85         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
86         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
87
88         // Check preflight for a disallowed request
89         resp = httptest.NewRecorder()
90         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
91         h.ServeHTTP(resp, req)
92         c.Check(resp.Body.String(), check.Equals, "")
93         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
94 }
95
96 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
97         for _, trial := range []struct {
98                 dataExists    bool
99                 sendIMSHeader bool
100                 expectStatus  int
101                 logRegexp     string
102         }{
103                 // If we return no content due to a Keep read error,
104                 // we should emit a log message.
105                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
106
107                 // If we return no content because the client sent an
108                 // If-Modified-Since header, our response should be
109                 // 304.  We still expect a "File download" log since it
110                 // counts as a file access for auditing.
111                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
112         } {
113                 c.Logf("trial: %+v", trial)
114                 arvadostest.StartKeep(2, true)
115                 if trial.dataExists {
116                         arv, err := arvadosclient.MakeArvadosClient()
117                         c.Assert(err, check.IsNil)
118                         arv.ApiToken = arvadostest.ActiveToken
119                         kc, err := keepclient.MakeKeepClient(arv)
120                         c.Assert(err, check.IsNil)
121                         _, _, err = kc.PutB([]byte("foo"))
122                         c.Assert(err, check.IsNil)
123                 }
124
125                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
126                 req := &http.Request{
127                         Method:     "GET",
128                         Host:       u.Host,
129                         URL:        u,
130                         RequestURI: u.RequestURI(),
131                         Header: http.Header{
132                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
133                         },
134                 }
135                 if trial.sendIMSHeader {
136                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
137                 }
138
139                 var logbuf bytes.Buffer
140                 logger := logrus.New()
141                 logger.Out = &logbuf
142                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
143
144                 resp := httptest.NewRecorder()
145                 s.handler.ServeHTTP(resp, req)
146                 c.Check(resp.Code, check.Equals, trial.expectStatus)
147                 c.Check(resp.Body.String(), check.Equals, "")
148
149                 c.Log(logbuf.String())
150                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
151         }
152 }
153
154 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
155         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
156         token := arvadostest.ActiveToken
157         for _, trial := range []string{
158                 "http://keep-web/c=" + bogusID + "/foo",
159                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
160                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
161                 "http://keep-web/collections/" + bogusID + "/foo",
162                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
163                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
164         } {
165                 c.Log(trial)
166                 u := mustParseURL(trial)
167                 req := &http.Request{
168                         Method:     "GET",
169                         Host:       u.Host,
170                         URL:        u,
171                         RequestURI: u.RequestURI(),
172                 }
173                 resp := httptest.NewRecorder()
174                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
175                 s.handler.ServeHTTP(resp, req)
176                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
177         }
178 }
179
180 func mustParseURL(s string) *url.URL {
181         r, err := url.Parse(s)
182         if err != nil {
183                 panic("parse URL: " + s)
184         }
185         return r
186 }
187
188 func (s *IntegrationSuite) TestVhost404(c *check.C) {
189         for _, testURL := range []string{
190                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
191                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
192         } {
193                 resp := httptest.NewRecorder()
194                 u := mustParseURL(testURL)
195                 req := &http.Request{
196                         Method:     "GET",
197                         URL:        u,
198                         RequestURI: u.RequestURI(),
199                 }
200                 s.handler.ServeHTTP(resp, req)
201                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
202                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
203         }
204 }
205
206 // An authorizer modifies an HTTP request to make use of the given
207 // token -- by adding it to a header, cookie, query param, or whatever
208 // -- and returns the HTTP status code we should expect from keep-web if
209 // the token is invalid.
210 type authorizer func(*http.Request, string) int
211
212 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
213         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
214 }
215 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
216         r.Header.Add("Authorization", "Bearer "+tok)
217         return http.StatusUnauthorized
218 }
219 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
220         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
221 }
222 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
223         r.Header.Add("Authorization", "Bearer "+tok)
224         return http.StatusUnauthorized
225 }
226
227 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
228         s.doVhostRequests(c, authzViaCookieValue)
229 }
230 func authzViaCookieValue(r *http.Request, tok string) int {
231         r.AddCookie(&http.Cookie{
232                 Name:  "arvados_api_token",
233                 Value: auth.EncodeTokenCookie([]byte(tok)),
234         })
235         return http.StatusUnauthorized
236 }
237
238 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
239         s.doVhostRequests(c, authzViaPath)
240 }
241 func authzViaPath(r *http.Request, tok string) int {
242         r.URL.Path = "/t=" + tok + r.URL.Path
243         return http.StatusNotFound
244 }
245
246 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
247         s.doVhostRequests(c, authzViaQueryString)
248 }
249 func authzViaQueryString(r *http.Request, tok string) int {
250         r.URL.RawQuery = "api_token=" + tok
251         return http.StatusUnauthorized
252 }
253
254 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
255         s.doVhostRequests(c, authzViaPOST)
256 }
257 func authzViaPOST(r *http.Request, tok string) int {
258         r.Method = "POST"
259         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
260         r.Body = ioutil.NopCloser(strings.NewReader(
261                 url.Values{"api_token": {tok}}.Encode()))
262         return http.StatusUnauthorized
263 }
264
265 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
266         s.doVhostRequests(c, authzViaPOST)
267 }
268 func authzViaXHRPOST(r *http.Request, tok string) int {
269         r.Method = "POST"
270         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
271         r.Header.Add("Origin", "https://origin.example")
272         r.Body = ioutil.NopCloser(strings.NewReader(
273                 url.Values{
274                         "api_token":   {tok},
275                         "disposition": {"attachment"},
276                 }.Encode()))
277         return http.StatusUnauthorized
278 }
279
280 // Try some combinations of {url, token} using the given authorization
281 // mechanism, and verify the result is correct.
282 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
283         for _, hostPath := range []string{
284                 arvadostest.FooCollection + ".example.com/foo",
285                 arvadostest.FooCollection + "--collections.example.com/foo",
286                 arvadostest.FooCollection + "--collections.example.com/_/foo",
287                 arvadostest.FooCollectionPDH + ".example.com/foo",
288                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
289                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
290         } {
291                 c.Log("doRequests: ", hostPath)
292                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
293         }
294 }
295
296 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
297         for _, tok := range []string{
298                 arvadostest.ActiveToken,
299                 arvadostest.ActiveToken[:15],
300                 arvadostest.SpectatorToken,
301                 "bogus",
302                 "",
303         } {
304                 u := mustParseURL("http://" + hostPath)
305                 req := &http.Request{
306                         Method:     "GET",
307                         Host:       u.Host,
308                         URL:        u,
309                         RequestURI: u.RequestURI(),
310                         Header:     http.Header{},
311                 }
312                 failCode := authz(req, tok)
313                 req, resp := s.doReq(req)
314                 code, body := resp.Code, resp.Body.String()
315
316                 // If the initial request had a (non-empty) token
317                 // showing in the query string, we should have been
318                 // redirected in order to hide it in a cookie.
319                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
320
321                 if tok == arvadostest.ActiveToken {
322                         c.Check(code, check.Equals, http.StatusOK)
323                         c.Check(body, check.Equals, "foo")
324                 } else {
325                         c.Check(code >= 400, check.Equals, true)
326                         c.Check(code < 500, check.Equals, true)
327                         if tok == arvadostest.SpectatorToken {
328                                 // Valid token never offers to retry
329                                 // with different credentials.
330                                 c.Check(code, check.Equals, http.StatusNotFound)
331                         } else {
332                                 // Invalid token can ask to retry
333                                 // depending on the authz method.
334                                 c.Check(code, check.Equals, failCode)
335                         }
336                         if code == 404 {
337                                 c.Check(body, check.Equals, notFoundMessage+"\n")
338                         } else {
339                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
340                         }
341                 }
342         }
343 }
344
345 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
346         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
347                 for _, port := range []string{"80", "443", "8000"} {
348                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
349                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
350                         req := &http.Request{
351                                 Method:     "GET",
352                                 Host:       u.Host,
353                                 URL:        u,
354                                 RequestURI: u.RequestURI(),
355                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
356                         }
357                         req, resp := s.doReq(req)
358                         code, _ := resp.Code, resp.Body.String()
359
360                         if port == "8000" {
361                                 c.Check(code, check.Equals, 401)
362                         } else {
363                                 c.Check(code, check.Equals, 200)
364                         }
365                 }
366         }
367 }
368
369 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
370         resp := httptest.NewRecorder()
371         s.handler.ServeHTTP(resp, req)
372         if resp.Code != http.StatusSeeOther {
373                 return req, resp
374         }
375         cookies := (&http.Response{Header: resp.Header()}).Cookies()
376         u, _ := req.URL.Parse(resp.Header().Get("Location"))
377         req = &http.Request{
378                 Method:     "GET",
379                 Host:       u.Host,
380                 URL:        u,
381                 RequestURI: u.RequestURI(),
382                 Header:     http.Header{},
383         }
384         for _, c := range cookies {
385                 req.AddCookie(c)
386         }
387         return s.doReq(req)
388 }
389
390 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
391         s.testVhostRedirectTokenToCookie(c, "GET",
392                 arvadostest.FooCollection+".example.com/foo",
393                 "?api_token="+arvadostest.ActiveToken,
394                 "",
395                 "",
396                 http.StatusOK,
397                 "foo",
398         )
399 }
400
401 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
402         s.testVhostRedirectTokenToCookie(c, "GET",
403                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
404                 "",
405                 "",
406                 "",
407                 http.StatusOK,
408                 "foo",
409         )
410 }
411
412 // Bad token in URL is 404 Not Found because it doesn't make sense to
413 // retry the same URL with different authorization.
414 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
415         s.testVhostRedirectTokenToCookie(c, "GET",
416                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
417                 "",
418                 "",
419                 "",
420                 http.StatusNotFound,
421                 notFoundMessage+"\n",
422         )
423 }
424
425 // Bad token in a cookie (even if it got there via our own
426 // query-string-to-cookie redirect) is, in principle, retryable at the
427 // same URL so it's 401 Unauthorized.
428 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
429         s.testVhostRedirectTokenToCookie(c, "GET",
430                 arvadostest.FooCollection+".example.com/foo",
431                 "?api_token=thisisabogustoken",
432                 "",
433                 "",
434                 http.StatusUnauthorized,
435                 unauthorizedMessage+"\n",
436         )
437 }
438
439 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
440         s.testVhostRedirectTokenToCookie(c, "GET",
441                 "example.com/c="+arvadostest.FooCollection+"/foo",
442                 "?api_token="+arvadostest.ActiveToken,
443                 "",
444                 "",
445                 http.StatusBadRequest,
446                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
447         )
448 }
449
450 // If client requests an attachment by putting ?disposition=attachment
451 // in the query string, and gets redirected, the redirect target
452 // should respond with an attachment.
453 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
454         resp := s.testVhostRedirectTokenToCookie(c, "GET",
455                 arvadostest.FooCollection+".example.com/foo",
456                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
457                 "",
458                 "",
459                 http.StatusOK,
460                 "foo",
461         )
462         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
463 }
464
465 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
466         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
467         resp := s.testVhostRedirectTokenToCookie(c, "GET",
468                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
469                 "?api_token="+arvadostest.ActiveToken,
470                 "",
471                 "",
472                 http.StatusOK,
473                 "foo",
474         )
475         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
476 }
477
478 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
479         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
480         resp := s.testVhostRedirectTokenToCookie(c, "GET",
481                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
482                 "?api_token="+arvadostest.ActiveToken,
483                 "",
484                 "",
485                 http.StatusOK,
486                 "waz",
487         )
488         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
489         resp = s.testVhostRedirectTokenToCookie(c, "GET",
490                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
491                 "?api_token="+arvadostest.ActiveToken,
492                 "",
493                 "",
494                 http.StatusOK,
495                 "waz",
496         )
497         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
498 }
499
500 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
501         s.handler.Cluster.Collections.TrustAllContent = true
502         s.testVhostRedirectTokenToCookie(c, "GET",
503                 "example.com/c="+arvadostest.FooCollection+"/foo",
504                 "?api_token="+arvadostest.ActiveToken,
505                 "",
506                 "",
507                 http.StatusOK,
508                 "foo",
509         )
510 }
511
512 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
513         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
514
515         s.testVhostRedirectTokenToCookie(c, "GET",
516                 "example.com/c="+arvadostest.FooCollection+"/foo",
517                 "?api_token="+arvadostest.ActiveToken,
518                 "",
519                 "",
520                 http.StatusBadRequest,
521                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
522         )
523
524         resp := s.testVhostRedirectTokenToCookie(c, "GET",
525                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
526                 "?api_token="+arvadostest.ActiveToken,
527                 "",
528                 "",
529                 http.StatusOK,
530                 "foo",
531         )
532         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
533 }
534
535 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
536         s.testVhostRedirectTokenToCookie(c, "POST",
537                 arvadostest.FooCollection+".example.com/foo",
538                 "",
539                 "application/x-www-form-urlencoded",
540                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
541                 http.StatusOK,
542                 "foo",
543         )
544 }
545
546 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
547         s.testVhostRedirectTokenToCookie(c, "POST",
548                 arvadostest.FooCollection+".example.com/foo",
549                 "",
550                 "application/x-www-form-urlencoded",
551                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
552                 http.StatusNotFound,
553                 notFoundMessage+"\n",
554         )
555 }
556
557 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
558         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
559         s.testVhostRedirectTokenToCookie(c, "GET",
560                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
561                 "",
562                 "",
563                 "",
564                 http.StatusOK,
565                 "Hello world\n",
566         )
567 }
568
569 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
570         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
571         s.testVhostRedirectTokenToCookie(c, "GET",
572                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
573                 "",
574                 "",
575                 "",
576                 http.StatusNotFound,
577                 notFoundMessage+"\n",
578         )
579 }
580
581 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
582         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
583
584         client := arvados.NewClientFromEnv()
585         client.AuthToken = arvadostest.ActiveToken
586         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
587         c.Assert(err, check.IsNil)
588         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
589         c.Assert(err, check.IsNil)
590         f.Close()
591         mtxt, err := fs.MarshalManifest(".")
592         c.Assert(err, check.IsNil)
593         var coll arvados.Collection
594         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
595                 "collection": map[string]string{
596                         "manifest_text": mtxt,
597                 },
598         })
599         c.Assert(err, check.IsNil)
600
601         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
602         req := &http.Request{
603                 Method:     "GET",
604                 Host:       u.Host,
605                 URL:        u,
606                 RequestURI: u.RequestURI(),
607                 Header: http.Header{
608                         "Authorization": {"Bearer " + client.AuthToken},
609                 },
610         }
611         resp := httptest.NewRecorder()
612         s.handler.ServeHTTP(resp, req)
613         c.Check(resp.Code, check.Equals, http.StatusOK)
614         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
615 }
616
617 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
618         arv := arvados.NewClientFromEnv()
619         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
620         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
621         name := "foo/bar/baz"
622         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
623         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
624
625         client := arvados.NewClientFromEnv()
626         client.AuthToken = arvadostest.ActiveToken
627         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
628         c.Assert(err, check.IsNil)
629         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
630         c.Assert(err, check.IsNil)
631         f.Close()
632         mtxt, err := fs.MarshalManifest(".")
633         c.Assert(err, check.IsNil)
634         var coll arvados.Collection
635         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
636                 "collection": map[string]string{
637                         "manifest_text": mtxt,
638                         "name":          name,
639                         "owner_uuid":    arvadostest.AProjectUUID,
640                 },
641         })
642         c.Assert(err, check.IsNil)
643         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
644
645         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
646         for tryURL, expectRegexp := range map[string]string{
647                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
648                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
649         } {
650                 u, _ := url.Parse(tryURL)
651                 req := &http.Request{
652                         Method:     "GET",
653                         Host:       u.Host,
654                         URL:        u,
655                         RequestURI: u.RequestURI(),
656                         Header: http.Header{
657                                 "Authorization": {"Bearer " + client.AuthToken},
658                         },
659                 }
660                 resp := httptest.NewRecorder()
661                 s.handler.ServeHTTP(resp, req)
662                 c.Check(resp.Code, check.Equals, http.StatusOK)
663                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
664         }
665 }
666
667 // XHRs can't follow redirect-with-cookie so they rely on method=POST
668 // and disposition=attachment (telling us it's acceptable to respond
669 // with content instead of a redirect) and an Origin header that gets
670 // added automatically by the browser (telling us it's desirable to do
671 // so).
672 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
673         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
674         req := &http.Request{
675                 Method:     "POST",
676                 Host:       u.Host,
677                 URL:        u,
678                 RequestURI: u.RequestURI(),
679                 Header: http.Header{
680                         "Origin":       {"https://origin.example"},
681                         "Content-Type": {"application/x-www-form-urlencoded"},
682                 },
683                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
684                         "api_token":   {arvadostest.ActiveToken},
685                         "disposition": {"attachment"},
686                 }.Encode())),
687         }
688         resp := httptest.NewRecorder()
689         s.handler.ServeHTTP(resp, req)
690         c.Check(resp.Code, check.Equals, http.StatusOK)
691         c.Check(resp.Body.String(), check.Equals, "foo")
692         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
693
694         // GET + Origin header is representative of both AJAX GET
695         // requests and inline images via <IMG crossorigin="anonymous"
696         // src="...">.
697         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
698         req = &http.Request{
699                 Method:     "GET",
700                 Host:       u.Host,
701                 URL:        u,
702                 RequestURI: u.RequestURI(),
703                 Header: http.Header{
704                         "Origin": {"https://origin.example"},
705                 },
706         }
707         resp = httptest.NewRecorder()
708         s.handler.ServeHTTP(resp, req)
709         c.Check(resp.Code, check.Equals, http.StatusOK)
710         c.Check(resp.Body.String(), check.Equals, "foo")
711         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
712 }
713
714 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
715         u, _ := url.Parse(`http://` + hostPath + queryString)
716         req := &http.Request{
717                 Method:     method,
718                 Host:       u.Host,
719                 URL:        u,
720                 RequestURI: u.RequestURI(),
721                 Header:     http.Header{"Content-Type": {contentType}},
722                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
723         }
724
725         resp := httptest.NewRecorder()
726         defer func() {
727                 c.Check(resp.Code, check.Equals, expectStatus)
728                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
729         }()
730
731         s.handler.ServeHTTP(resp, req)
732         if resp.Code != http.StatusSeeOther {
733                 return resp
734         }
735         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
736         cookies := (&http.Response{Header: resp.Header()}).Cookies()
737
738         u, _ = u.Parse(resp.Header().Get("Location"))
739         req = &http.Request{
740                 Method:     "GET",
741                 Host:       u.Host,
742                 URL:        u,
743                 RequestURI: u.RequestURI(),
744                 Header:     http.Header{},
745         }
746         for _, c := range cookies {
747                 req.AddCookie(c)
748         }
749
750         resp = httptest.NewRecorder()
751         s.handler.ServeHTTP(resp, req)
752         c.Check(resp.Header().Get("Location"), check.Equals, "")
753         return resp
754 }
755
756 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
757         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
758         s.testDirectoryListing(c)
759 }
760
761 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
762         s.handler.Cluster.Users.AnonymousUserToken = ""
763         s.testDirectoryListing(c)
764 }
765
766 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
767         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
768         authHeader := http.Header{
769                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
770         }
771         for _, trial := range []struct {
772                 uri      string
773                 header   http.Header
774                 expect   []string
775                 redirect string
776                 cutDirs  int
777         }{
778                 {
779                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
780                         header:  authHeader,
781                         expect:  []string{"dir1/foo", "dir1/bar"},
782                         cutDirs: 0,
783                 },
784                 {
785                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
786                         header:  authHeader,
787                         expect:  []string{"foo", "bar"},
788                         cutDirs: 1,
789                 },
790                 {
791                         // URLs of this form ignore authHeader, and
792                         // FooAndBarFilesInDirUUID isn't public, so
793                         // this returns 401.
794                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
795                         header: authHeader,
796                         expect: nil,
797                 },
798                 {
799                         uri:     "download.example.com/users/active/foo_file_in_dir/",
800                         header:  authHeader,
801                         expect:  []string{"dir1/"},
802                         cutDirs: 3,
803                 },
804                 {
805                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
806                         header:  authHeader,
807                         expect:  []string{"bar"},
808                         cutDirs: 4,
809                 },
810                 {
811                         uri:     "download.example.com/",
812                         header:  authHeader,
813                         expect:  []string{"users/"},
814                         cutDirs: 0,
815                 },
816                 {
817                         uri:      "download.example.com/users",
818                         header:   authHeader,
819                         redirect: "/users/",
820                         expect:   []string{"active/"},
821                         cutDirs:  1,
822                 },
823                 {
824                         uri:     "download.example.com/users/",
825                         header:  authHeader,
826                         expect:  []string{"active/"},
827                         cutDirs: 1,
828                 },
829                 {
830                         uri:      "download.example.com/users/active",
831                         header:   authHeader,
832                         redirect: "/users/active/",
833                         expect:   []string{"foo_file_in_dir/"},
834                         cutDirs:  2,
835                 },
836                 {
837                         uri:     "download.example.com/users/active/",
838                         header:  authHeader,
839                         expect:  []string{"foo_file_in_dir/"},
840                         cutDirs: 2,
841                 },
842                 {
843                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
844                         header:  nil,
845                         expect:  []string{"dir1/foo", "dir1/bar"},
846                         cutDirs: 4,
847                 },
848                 {
849                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
850                         header:  nil,
851                         expect:  []string{"dir1/foo", "dir1/bar"},
852                         cutDirs: 2,
853                 },
854                 {
855                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
856                         header:  nil,
857                         expect:  []string{"dir1/foo", "dir1/bar"},
858                         cutDirs: 2,
859                 },
860                 {
861                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
862                         header:  authHeader,
863                         expect:  []string{"dir1/foo", "dir1/bar"},
864                         cutDirs: 1,
865                 },
866                 {
867                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
868                         header:   authHeader,
869                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
870                         expect:   []string{"foo", "bar"},
871                         cutDirs:  2,
872                 },
873                 {
874                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
875                         header:  authHeader,
876                         expect:  []string{"foo", "bar"},
877                         cutDirs: 3,
878                 },
879                 {
880                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
881                         header:   authHeader,
882                         redirect: "/dir1/",
883                         expect:   []string{"foo", "bar"},
884                         cutDirs:  1,
885                 },
886                 {
887                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
888                         header: authHeader,
889                         expect: nil,
890                 },
891                 {
892                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
893                         header:  authHeader,
894                         expect:  []string{"waz"},
895                         cutDirs: 1,
896                 },
897                 {
898                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
899                         header:  authHeader,
900                         expect:  []string{"waz"},
901                         cutDirs: 2,
902                 },
903         } {
904                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
905                 resp := httptest.NewRecorder()
906                 u := mustParseURL("//" + trial.uri)
907                 req := &http.Request{
908                         Method:     "GET",
909                         Host:       u.Host,
910                         URL:        u,
911                         RequestURI: u.RequestURI(),
912                         Header:     copyHeader(trial.header),
913                 }
914                 s.handler.ServeHTTP(resp, req)
915                 var cookies []*http.Cookie
916                 for resp.Code == http.StatusSeeOther {
917                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
918                         req = &http.Request{
919                                 Method:     "GET",
920                                 Host:       u.Host,
921                                 URL:        u,
922                                 RequestURI: u.RequestURI(),
923                                 Header:     copyHeader(trial.header),
924                         }
925                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
926                         for _, c := range cookies {
927                                 req.AddCookie(c)
928                         }
929                         resp = httptest.NewRecorder()
930                         s.handler.ServeHTTP(resp, req)
931                 }
932                 if trial.redirect != "" {
933                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
934                 }
935                 if trial.expect == nil {
936                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
937                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
938                         } else {
939                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
940                         }
941                 } else {
942                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
943                         for _, e := range trial.expect {
944                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
945                         }
946                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
947                 }
948
949                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
950                 req = &http.Request{
951                         Method:     "OPTIONS",
952                         Host:       u.Host,
953                         URL:        u,
954                         RequestURI: u.RequestURI(),
955                         Header:     copyHeader(trial.header),
956                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
957                 }
958                 resp = httptest.NewRecorder()
959                 s.handler.ServeHTTP(resp, req)
960                 if trial.expect == nil {
961                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
962                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
963                         } else {
964                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
965                         }
966                 } else {
967                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
968                 }
969
970                 req = &http.Request{
971                         Method:     "PROPFIND",
972                         Host:       u.Host,
973                         URL:        u,
974                         RequestURI: u.RequestURI(),
975                         Header:     copyHeader(trial.header),
976                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
977                 }
978                 resp = httptest.NewRecorder()
979                 s.handler.ServeHTTP(resp, req)
980                 if trial.expect == nil {
981                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
982                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
983                         } else {
984                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
985                         }
986                 } else {
987                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
988                         for _, e := range trial.expect {
989                                 if strings.HasSuffix(e, "/") {
990                                         e = filepath.Join(u.Path, e) + "/"
991                                 } else {
992                                         e = filepath.Join(u.Path, e)
993                                 }
994                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
995                         }
996                 }
997         }
998 }
999
1000 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1001         arv := arvados.NewClientFromEnv()
1002         var newCollection arvados.Collection
1003         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1004                 "collection": map[string]string{
1005                         "owner_uuid":    arvadostest.ActiveUserUUID,
1006                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1007                         "name":          "keep-web test collection",
1008                 },
1009                 "ensure_unique_name": true,
1010         })
1011         c.Assert(err, check.IsNil)
1012         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1013
1014         var updated arvados.Collection
1015         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1016                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1017                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1018                 req := &http.Request{
1019                         Method:     "DELETE",
1020                         Host:       u.Host,
1021                         URL:        u,
1022                         RequestURI: u.RequestURI(),
1023                         Header: http.Header{
1024                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1025                         },
1026                 }
1027                 resp := httptest.NewRecorder()
1028                 s.handler.ServeHTTP(resp, req)
1029                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1030
1031                 updated = arvados.Collection{}
1032                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1033                 c.Check(err, check.IsNil)
1034                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1035                 c.Logf("updated manifest_text %q", updated.ManifestText)
1036         }
1037         c.Check(updated.ManifestText, check.Equals, "")
1038 }
1039
1040 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1041         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1042
1043         client := arvados.NewClientFromEnv()
1044         client.AuthToken = arvadostest.ActiveToken
1045         arv, err := arvadosclient.New(client)
1046         c.Assert(err, check.Equals, nil)
1047         kc, err := keepclient.MakeKeepClient(arv)
1048         c.Assert(err, check.Equals, nil)
1049
1050         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1051         c.Assert(err, check.IsNil)
1052
1053         trials := []struct {
1054                 filename    string
1055                 content     string
1056                 contentType string
1057         }{
1058                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1059                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1060                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1061                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1062                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1063         }
1064         for _, trial := range trials {
1065                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1066                 c.Assert(err, check.IsNil)
1067                 _, err = f.Write([]byte(trial.content))
1068                 c.Assert(err, check.IsNil)
1069                 c.Assert(f.Close(), check.IsNil)
1070         }
1071         mtxt, err := fs.MarshalManifest(".")
1072         c.Assert(err, check.IsNil)
1073         var coll arvados.Collection
1074         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1075                 "collection": map[string]string{
1076                         "manifest_text": mtxt,
1077                 },
1078         })
1079         c.Assert(err, check.IsNil)
1080
1081         for _, trial := range trials {
1082                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1083                 req := &http.Request{
1084                         Method:     "GET",
1085                         Host:       u.Host,
1086                         URL:        u,
1087                         RequestURI: u.RequestURI(),
1088                         Header: http.Header{
1089                                 "Authorization": {"Bearer " + client.AuthToken},
1090                         },
1091                 }
1092                 resp := httptest.NewRecorder()
1093                 s.handler.ServeHTTP(resp, req)
1094                 c.Check(resp.Code, check.Equals, http.StatusOK)
1095                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1096                 c.Check(resp.Body.String(), check.Equals, trial.content)
1097         }
1098 }
1099
1100 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1101         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1102         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1103         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1104         req := &http.Request{
1105                 Method:     "GET",
1106                 Host:       u.Host,
1107                 URL:        u,
1108                 RequestURI: u.RequestURI(),
1109         }
1110         resp := httptest.NewRecorder()
1111         s.handler.ServeHTTP(resp, req)
1112         c.Check(resp.Code, check.Equals, http.StatusOK)
1113         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1114 }
1115
1116 // Writing to a collection shouldn't affect its entry in the
1117 // PDH-to-manifest cache.
1118 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1119         arv, err := arvadosclient.MakeArvadosClient()
1120         c.Assert(err, check.Equals, nil)
1121         arv.ApiToken = arvadostest.ActiveToken
1122
1123         u := mustParseURL("http://x.example/testfile")
1124         req := &http.Request{
1125                 Method:     "GET",
1126                 Host:       u.Host,
1127                 URL:        u,
1128                 RequestURI: u.RequestURI(),
1129                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1130         }
1131
1132         checkWithID := func(id string, status int) {
1133                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1134                 req.Host = req.URL.Host
1135                 resp := httptest.NewRecorder()
1136                 s.handler.ServeHTTP(resp, req)
1137                 c.Check(resp.Code, check.Equals, status)
1138         }
1139
1140         var colls [2]arvados.Collection
1141         for i := range colls {
1142                 err := arv.Create("collections",
1143                         map[string]interface{}{
1144                                 "ensure_unique_name": true,
1145                                 "collection": map[string]interface{}{
1146                                         "name": "test collection",
1147                                 },
1148                         }, &colls[i])
1149                 c.Assert(err, check.Equals, nil)
1150         }
1151
1152         // Populate cache with empty collection
1153         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1154
1155         // write a file to colls[0]
1156         reqPut := *req
1157         reqPut.Method = "PUT"
1158         reqPut.URL.Host = colls[0].UUID + ".example"
1159         reqPut.Host = req.URL.Host
1160         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1161         resp := httptest.NewRecorder()
1162         s.handler.ServeHTTP(resp, &reqPut)
1163         c.Check(resp.Code, check.Equals, http.StatusCreated)
1164
1165         // new file should not appear in colls[1]
1166         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1167         checkWithID(colls[1].UUID, http.StatusNotFound)
1168
1169         checkWithID(colls[0].UUID, http.StatusOK)
1170 }
1171
1172 func copyHeader(h http.Header) http.Header {
1173         hc := http.Header{}
1174         for k, v := range h {
1175                 hc[k] = append([]string(nil), v...)
1176         }
1177         return hc
1178 }
1179
1180 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1181         successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
1182
1183         client := arvados.NewClientFromEnv()
1184         client.AuthToken = arvadostest.AdminToken
1185         var logentries arvados.LogList
1186         limit1 := 1
1187         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1188                 arvados.ResourceListParams{
1189                         Limit: &limit1,
1190                         Order: "created_at desc"})
1191         c.Check(err, check.IsNil)
1192         c.Check(logentries.Items, check.HasLen, 1)
1193         lastLogId := logentries.Items[0].ID
1194
1195         var logbuf bytes.Buffer
1196         logger := logrus.New()
1197         logger.Out = &logbuf
1198         resp := httptest.NewRecorder()
1199         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1200         s.handler.ServeHTTP(resp, req)
1201
1202         if perm {
1203                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1204                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1205                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1206
1207                 deadline := time.Now().Add(time.Second)
1208                 for {
1209                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1210                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1211                                 arvados.ResourceListParams{
1212                                         Filters: []arvados.Filter{
1213                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1214                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1215                                         },
1216                                         Limit: &limit1,
1217                                         Order: "created_at desc",
1218                                 })
1219                         c.Assert(err, check.IsNil)
1220                         if len(logentries.Items) > 0 &&
1221                                 logentries.Items[0].ID > lastLogId &&
1222                                 logentries.Items[0].ObjectUUID == userUuid &&
1223                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1224                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1225                                 break
1226                         }
1227                         c.Logf("logentries.Items: %+v", logentries.Items)
1228                         time.Sleep(50 * time.Millisecond)
1229                 }
1230         } else {
1231                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1232                 c.Check(logbuf.String(), check.Equals, "")
1233         }
1234 }
1235
1236 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1237         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1238
1239         s.handler.Cluster.Collections.TrustAllContent = true
1240
1241         for _, adminperm := range []bool{true, false} {
1242                 for _, userperm := range []bool{true, false} {
1243                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1244                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1245
1246                         // Test admin permission
1247                         req := &http.Request{
1248                                 Method:     "GET",
1249                                 Host:       u.Host,
1250                                 URL:        u,
1251                                 RequestURI: u.RequestURI(),
1252                                 Header: http.Header{
1253                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1254                                 },
1255                         }
1256                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1257                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
1258
1259                         // Test user permission
1260                         req = &http.Request{
1261                                 Method:     "GET",
1262                                 Host:       u.Host,
1263                                 URL:        u,
1264                                 RequestURI: u.RequestURI(),
1265                                 Header: http.Header{
1266                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1267                                 },
1268                         }
1269                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1270                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1271                 }
1272         }
1273
1274         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1275
1276         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1277                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1278
1279                 u = mustParseURL(tryurl)
1280                 req := &http.Request{
1281                         Method:     "GET",
1282                         Host:       u.Host,
1283                         URL:        u,
1284                         RequestURI: u.RequestURI(),
1285                         Header: http.Header{
1286                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1287                         },
1288                 }
1289                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1290                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
1291         }
1292
1293         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1294         req := &http.Request{
1295                 Method:     "GET",
1296                 Host:       u.Host,
1297                 URL:        u,
1298                 RequestURI: u.RequestURI(),
1299                 Header: http.Header{
1300                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1301                 },
1302         }
1303         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1304                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1305 }
1306
1307 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1308         for _, adminperm := range []bool{true, false} {
1309                 for _, userperm := range []bool{true, false} {
1310
1311                         arv := arvados.NewClientFromEnv()
1312                         arv.AuthToken = arvadostest.ActiveToken
1313
1314                         var coll arvados.Collection
1315                         err := arv.RequestAndDecode(&coll,
1316                                 "POST",
1317                                 "/arvados/v1/collections",
1318                                 nil,
1319                                 map[string]interface{}{
1320                                         "ensure_unique_name": true,
1321                                         "collection": map[string]interface{}{
1322                                                 "name": "test collection",
1323                                         },
1324                                 })
1325                         c.Assert(err, check.Equals, nil)
1326
1327                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1328
1329                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1330                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1331
1332                         // Test admin permission
1333                         req := &http.Request{
1334                                 Method:     "PUT",
1335                                 Host:       u.Host,
1336                                 URL:        u,
1337                                 RequestURI: u.RequestURI(),
1338                                 Header: http.Header{
1339                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1340                                 },
1341                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1342                         }
1343                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1344                                 arvadostest.AdminUserUUID, coll.UUID, "bar")
1345
1346                         // Test user permission
1347                         req = &http.Request{
1348                                 Method:     "PUT",
1349                                 Host:       u.Host,
1350                                 URL:        u,
1351                                 RequestURI: u.RequestURI(),
1352                                 Header: http.Header{
1353                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1354                                 },
1355                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1356                         }
1357                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1358                                 arvadostest.ActiveUserUUID, coll.UUID, "bar")
1359                 }
1360         }
1361 }