Merge branch '17705-adc-arch-doc'
[arvados.git] / lib / controller / handler_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "context"
9         "crypto/tls"
10         "encoding/json"
11         "io/ioutil"
12         "net/http"
13         "net/http/httptest"
14         "net/url"
15         "os"
16         "strings"
17         "testing"
18         "time"
19
20         "git.arvados.org/arvados.git/sdk/go/arvados"
21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
22         "git.arvados.org/arvados.git/sdk/go/auth"
23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
24         "git.arvados.org/arvados.git/sdk/go/httpserver"
25         "github.com/prometheus/client_golang/prometheus"
26         check "gopkg.in/check.v1"
27 )
28
29 // Gocheck boilerplate
30 func Test(t *testing.T) {
31         check.TestingT(t)
32 }
33
34 var _ = check.Suite(&HandlerSuite{})
35
36 type HandlerSuite struct {
37         cluster *arvados.Cluster
38         handler http.Handler
39         ctx     context.Context
40         cancel  context.CancelFunc
41 }
42
43 func (s *HandlerSuite) SetUpTest(c *check.C) {
44         s.ctx, s.cancel = context.WithCancel(context.Background())
45         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(os.Stderr, "json", "debug"))
46         s.cluster = &arvados.Cluster{
47                 ClusterID:  "zzzzz",
48                 PostgreSQL: integrationTestCluster().PostgreSQL,
49         }
50         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
51         s.cluster.TLS.Insecure = true
52         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
53         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
54         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry())
55 }
56
57 func (s *HandlerSuite) TearDownTest(c *check.C) {
58         s.cancel()
59 }
60
61 func (s *HandlerSuite) TestConfigExport(c *check.C) {
62         s.cluster.ManagementToken = "secret"
63         s.cluster.SystemRootToken = "secret"
64         s.cluster.Collections.BlobSigning = true
65         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
66         for _, method := range []string{"GET", "OPTIONS"} {
67                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
68                 resp := httptest.NewRecorder()
69                 s.handler.ServeHTTP(resp, req)
70                 c.Log(resp.Body.String())
71                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
72                         continue
73                 }
74                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
75                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
76                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
77                 if method == "OPTIONS" {
78                         c.Check(resp.Body.String(), check.HasLen, 0)
79                         continue
80                 }
81                 var cluster arvados.Cluster
82                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
83                 c.Check(err, check.IsNil)
84                 c.Check(cluster.ManagementToken, check.Equals, "")
85                 c.Check(cluster.SystemRootToken, check.Equals, "")
86                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
87                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
88         }
89 }
90
91 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
92         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
93         resp := httptest.NewRecorder()
94         s.handler.ServeHTTP(resp, req)
95         c.Check(resp.Code, check.Equals, http.StatusOK)
96         var dd arvados.DiscoveryDocument
97         err := json.Unmarshal(resp.Body.Bytes(), &dd)
98         c.Check(err, check.IsNil)
99         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
100         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
101         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
102         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
103 }
104
105 func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
106         s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
107         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
108         resp := httptest.NewRecorder()
109         s.handler.ServeHTTP(resp, req)
110         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
111         var jresp httpserver.ErrorResponse
112         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
113         c.Check(err, check.IsNil)
114         c.Assert(len(jresp.Errors), check.Equals, 1)
115         c.Check(jresp.Errors[0], check.Matches, `.*context deadline exceeded.*`)
116 }
117
118 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
119         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
120         resp := httptest.NewRecorder()
121         s.handler.ServeHTTP(resp, req)
122         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
123         jresp := map[string]interface{}{}
124         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
125         c.Check(err, check.IsNil)
126         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
127 }
128
129 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
130         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
131         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
132         resp := httptest.NewRecorder()
133         s.handler.ServeHTTP(resp, req)
134         c.Check(resp.Code, check.Equals, http.StatusOK)
135         var u arvados.User
136         err := json.Unmarshal(resp.Body.Bytes(), &u)
137         c.Check(err, check.IsNil)
138         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
139 }
140
141 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
142         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
143                 "_method":   {"GET"},
144                 "api_token": {arvadostest.ActiveToken},
145         }.Encode()))
146         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
147         resp := httptest.NewRecorder()
148         s.handler.ServeHTTP(resp, req)
149         c.Check(resp.Code, check.Equals, http.StatusOK)
150         var u arvados.User
151         err := json.Unmarshal(resp.Body.Bytes(), &u)
152         c.Check(err, check.IsNil)
153         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
154 }
155
156 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
157         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
158         resp := httptest.NewRecorder()
159         s.handler.ServeHTTP(resp, req)
160         c.Check(resp.Code, check.Equals, http.StatusNotFound)
161         jresp := map[string]interface{}{}
162         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
163         c.Check(err, check.IsNil)
164         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
165 }
166
167 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
168         s.cluster.Login.Google.Enable = true
169         s.cluster.Login.Google.ClientID = "test"
170         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
171         resp := httptest.NewRecorder()
172         s.handler.ServeHTTP(resp, req)
173         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
174                 c.Log(resp.Body.String())
175         }
176         c.Check(resp.Header().Get("Location"), check.Equals, "https://example.com/foo")
177 }
178
179 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
180         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
181         user, ok, err := s.handler.(*Handler).validateAPItoken(req, arvadostest.ActiveToken)
182         c.Assert(err, check.IsNil)
183         c.Check(ok, check.Equals, true)
184         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
185         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
186         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
187         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
188 }
189
190 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
191         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
192         user, ok, err := s.handler.(*Handler).validateAPItoken(req, arvadostest.ActiveTokenV2)
193         c.Assert(err, check.IsNil)
194         c.Check(ok, check.Equals, true)
195         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
196         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
197         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
198         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
199         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
200 }
201
202 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
203         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
204         c.Assert(err, check.IsNil)
205         for _, trial := range []struct {
206                 code  int
207                 token string
208         }{
209                 {http.StatusOK, saltedToken},
210                 {http.StatusUnauthorized, "bogus"},
211         } {
212                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
213                 req.Header.Set("Authorization", "Bearer "+trial.token)
214                 resp := httptest.NewRecorder()
215                 s.handler.ServeHTTP(resp, req)
216                 if !c.Check(resp.Code, check.Equals, trial.code) {
217                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
218                 }
219         }
220 }
221
222 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
223         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
224         auth, err := s.handler.(*Handler).createAPItoken(req, arvadostest.ActiveUserUUID, nil)
225         c.Assert(err, check.IsNil)
226         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
227
228         user, ok, err := s.handler.(*Handler).validateAPItoken(req, auth.TokenV2())
229         c.Assert(err, check.IsNil)
230         c.Check(ok, check.Equals, true)
231         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
232         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
233         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
234         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
235         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
236 }
237
238 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
239         var proxied, direct map[string]interface{}
240         var err error
241
242         // Get collection from controller
243         req := httptest.NewRequest("GET", url, nil)
244         req.Header.Set("Authorization", "Bearer "+token)
245         resp := httptest.NewRecorder()
246         s.handler.ServeHTTP(resp, req)
247         c.Assert(resp.Code, check.Equals, http.StatusOK,
248                 check.Commentf("Wasn't able to get data from the controller at %q", url))
249         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
250         c.Check(err, check.Equals, nil)
251
252         // Get collection directly from RailsAPI
253         client := &http.Client{
254                 Transport: &http.Transport{
255                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
256                 },
257         }
258         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
259         c.Check(err, check.Equals, nil)
260         c.Assert(resp2.StatusCode, check.Equals, http.StatusOK,
261                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url))
262         defer resp2.Body.Close()
263         db, err := ioutil.ReadAll(resp2.Body)
264         c.Check(err, check.Equals, nil)
265         err = json.Unmarshal(db, &direct)
266         c.Check(err, check.Equals, nil)
267
268         // Check that all RailsAPI provided keys exist on the controller response.
269         for k := range direct {
270                 if _, ok := skippedFields[k]; ok {
271                         continue
272                 } else if val, ok := proxied[k]; ok {
273                         if direct["kind"] == "arvados#collection" && k == "manifest_text" {
274                                 // Tokens differ from request to request
275                                 c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
276                         } else {
277                                 c.Check(val, check.DeepEquals, direct[k],
278                                         check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
279                         }
280                 } else {
281                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
282                 }
283         }
284 }
285
286 func (s *HandlerSuite) TestGetObjects(c *check.C) {
287         // Get the 1st keep service's uuid from the running test server.
288         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
289         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
290         resp := httptest.NewRecorder()
291         s.handler.ServeHTTP(resp, req)
292         c.Assert(resp.Code, check.Equals, http.StatusOK)
293         var ksList arvados.KeepServiceList
294         json.Unmarshal(resp.Body.Bytes(), &ksList)
295         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
296         ksUUID := ksList.Items[0].UUID
297
298         testCases := map[string]map[string]bool{
299                 "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID:     nil,
300                 "api_client_authorizations/" + arvadostest.AdminTokenUUID:      nil,
301                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       nil,
302                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     {"href": true},
303                 "containers/" + arvadostest.RunningContainerUUID:               nil,
304                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
305                 "groups/" + arvadostest.AProjectUUID:                           nil,
306                 "keep_services/" + ksUUID:                                      nil,
307                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
308                 "logs/" + arvadostest.CrunchstatForRunningJobLogUUID:           nil,
309                 "nodes/" + arvadostest.IdleNodeUUID:                            nil,
310                 "repositories/" + arvadostest.ArvadosRepoUUID:                  nil,
311                 "users/" + arvadostest.ActiveUserUUID:                          {"href": true},
312                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
313                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
314         }
315         for url, skippedFields := range testCases {
316                 s.CheckObjectType(c, "/arvados/v1/"+url, arvadostest.AdminToken, skippedFields)
317         }
318 }
319
320 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
321         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
322         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
323         resp := httptest.NewRecorder()
324         s.handler.ServeHTTP(resp, req)
325         c.Check(resp.Code, check.Equals, http.StatusNotFound)
326         var jresp struct {
327                 Errors []string
328         }
329         c.Log(resp.Body.String())
330         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
331         c.Assert(jresp.Errors, check.HasLen, 1)
332         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
333         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
334 }