Merge branch '18349-fed-request-id'
[arvados.git] / lib / controller / integration_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "context"
10         "database/sql"
11         "encoding/json"
12         "fmt"
13         "io"
14         "io/ioutil"
15         "math"
16         "net"
17         "net/http"
18         "os"
19         "os/exec"
20         "path/filepath"
21         "strconv"
22         "strings"
23         "sync"
24
25         "git.arvados.org/arvados.git/lib/boot"
26         "git.arvados.org/arvados.git/lib/config"
27         "git.arvados.org/arvados.git/sdk/go/arvados"
28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
30         "git.arvados.org/arvados.git/sdk/go/httpserver"
31         check "gopkg.in/check.v1"
32 )
33
34 var _ = check.Suite(&IntegrationSuite{})
35
36 type IntegrationSuite struct {
37         testClusters map[string]*boot.TestCluster
38         oidcprovider *arvadostest.OIDCProvider
39 }
40
41 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
42         cwd, _ := os.Getwd()
43
44         s.oidcprovider = arvadostest.NewOIDCProvider(c)
45         s.oidcprovider.AuthEmail = "user@example.com"
46         s.oidcprovider.AuthEmailVerified = true
47         s.oidcprovider.AuthName = "Example User"
48         s.oidcprovider.ValidClientID = "clientid"
49         s.oidcprovider.ValidClientSecret = "clientsecret"
50
51         s.testClusters = map[string]*boot.TestCluster{
52                 "z1111": nil,
53                 "z2222": nil,
54                 "z3333": nil,
55         }
56         hostport := map[string]string{}
57         for id := range s.testClusters {
58                 hostport[id] = func() string {
59                         // TODO: Instead of expecting random ports on
60                         // 127.0.0.11, 22, 33 to be race-safe, try
61                         // different 127.x.y.z until finding one that
62                         // isn't in use.
63                         ln, err := net.Listen("tcp", ":0")
64                         c.Assert(err, check.IsNil)
65                         ln.Close()
66                         _, port, err := net.SplitHostPort(ln.Addr().String())
67                         c.Assert(err, check.IsNil)
68                         return "127.0.0." + id[3:] + ":" + port
69                 }()
70         }
71         for id := range s.testClusters {
72                 yaml := `Clusters:
73   ` + id + `:
74     Services:
75       Controller:
76         ExternalURL: https://` + hostport[id] + `
77     TLS:
78       Insecure: true
79     SystemLogs:
80       Format: text
81     RemoteClusters:
82       z1111:
83         Host: ` + hostport["z1111"] + `
84         Scheme: https
85         Insecure: true
86         Proxy: true
87         ActivateUsers: true
88 `
89                 if id != "z2222" {
90                         yaml += `      z2222:
91         Host: ` + hostport["z2222"] + `
92         Scheme: https
93         Insecure: true
94         Proxy: true
95         ActivateUsers: true
96 `
97                 }
98                 if id != "z3333" {
99                         yaml += `      z3333:
100         Host: ` + hostport["z3333"] + `
101         Scheme: https
102         Insecure: true
103         Proxy: true
104         ActivateUsers: true
105 `
106                 }
107                 if id == "z1111" {
108                         yaml += `
109     Login:
110       LoginCluster: z1111
111       OpenIDConnect:
112         Enable: true
113         Issuer: ` + s.oidcprovider.Issuer.URL + `
114         ClientID: ` + s.oidcprovider.ValidClientID + `
115         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
116         EmailClaim: email
117         EmailVerifiedClaim: email_verified
118         AcceptAccessToken: true
119         AcceptAccessTokenScope: ""
120 `
121                 } else {
122                         yaml += `
123     Login:
124       LoginCluster: z1111
125 `
126                 }
127
128                 loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
129                 loader.Path = "-"
130                 loader.SkipLegacy = true
131                 loader.SkipAPICalls = true
132                 cfg, err := loader.Load()
133                 c.Assert(err, check.IsNil)
134                 tc := boot.NewTestCluster(
135                         filepath.Join(cwd, "..", ".."),
136                         id, cfg, "127.0.0."+id[3:], c.Log)
137                 tc.Super.NoWorkbench1 = true
138                 tc.Start()
139                 s.testClusters[id] = tc
140         }
141         for _, tc := range s.testClusters {
142                 ok := tc.WaitReady()
143                 c.Assert(ok, check.Equals, true)
144         }
145 }
146
147 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
148         for _, c := range s.testClusters {
149                 c.Super.Stop()
150         }
151 }
152
153 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
154         conn := s.testClusters["z1111"].Conn()
155         rootctx, _, _ := s.testClusters["z1111"].RootClients()
156         userctx, _, kc, _ := s.testClusters["z1111"].UserClients(rootctx, c, conn, s.oidcprovider.AuthEmail, true)
157         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
158         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
159         c.Assert(err, check.IsNil)
160         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
161 }
162
163 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
164         conn1 := s.testClusters["z1111"].Conn()
165         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
166         conn3 := s.testClusters["z3333"].Conn()
167         userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
168
169         // Create the collection to find its PDH (but don't save it
170         // anywhere yet)
171         var coll1 arvados.Collection
172         fs1, err := coll1.FileSystem(ac1, kc1)
173         c.Assert(err, check.IsNil)
174         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
175         c.Assert(err, check.IsNil)
176         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
177         c.Assert(err, check.IsNil)
178         err = f.Close()
179         c.Assert(err, check.IsNil)
180         mtxt, err := fs1.MarshalManifest(".")
181         c.Assert(err, check.IsNil)
182         pdh := arvados.PortableDataHash(mtxt)
183
184         // Looking up the PDH before saving returns 404 if cycle
185         // detection is working.
186         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
187         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
188
189         // Save the collection on cluster z1111.
190         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
191                 "manifest_text": mtxt,
192         }})
193         c.Assert(err, check.IsNil)
194
195         // Retrieve the collection from cluster z3333.
196         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
197         c.Check(err, check.IsNil)
198         c.Check(coll.PortableDataHash, check.Equals, pdh)
199 }
200
201 // Tests bug #18004
202 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
203         conn1 := s.testClusters["z1111"].Conn()
204         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
205         rootctx2, _, _ := s.testClusters["z2222"].RootClients()
206         conn2 := s.testClusters["z2222"].Conn()
207         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user2@example.com", true)
208
209         var wg1, wg2 sync.WaitGroup
210         creqs := 100
211
212         // Make concurrent requests to z2222 with a local token to make sure more
213         // than one worker is listening.
214         wg1.Add(1)
215         for i := 0; i < creqs; i++ {
216                 wg2.Add(1)
217                 go func() {
218                         defer wg2.Done()
219                         wg1.Wait()
220                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
221                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
222                 }()
223         }
224         wg1.Done()
225         wg2.Wait()
226
227         // Real test pass -- use a new remote token than the one used in the warm-up
228         // phase.
229         wg1.Add(1)
230         for i := 0; i < creqs; i++ {
231                 wg2.Add(1)
232                 go func() {
233                         defer wg2.Done()
234                         wg1.Wait()
235                         // Retrieve the remote collection from cluster z2222.
236                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
237                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
238                 }()
239         }
240         wg1.Done()
241         wg2.Wait()
242 }
243
244 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
245         if _, err := exec.LookPath("s3cmd"); err != nil {
246                 c.Skip("s3cmd not in PATH")
247                 return
248         }
249
250         testText := "IntegrationSuite.TestS3WithFederatedToken"
251
252         conn1 := s.testClusters["z1111"].Conn()
253         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
254         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
255         conn3 := s.testClusters["z3333"].Conn()
256
257         createColl := func(clusterID string) arvados.Collection {
258                 _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken)
259                 var coll arvados.Collection
260                 fs, err := coll.FileSystem(ac, kc)
261                 c.Assert(err, check.IsNil)
262                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
263                 c.Assert(err, check.IsNil)
264                 _, err = io.WriteString(f, testText)
265                 c.Assert(err, check.IsNil)
266                 err = f.Close()
267                 c.Assert(err, check.IsNil)
268                 mtxt, err := fs.MarshalManifest(".")
269                 c.Assert(err, check.IsNil)
270                 coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
271                         "manifest_text": mtxt,
272                 }})
273                 c.Assert(err, check.IsNil)
274                 return coll
275         }
276
277         for _, trial := range []struct {
278                 clusterID string // create the collection on this cluster (then use z3333 to access it)
279                 token     string
280         }{
281                 // Try the hardest test first: z3333 hasn't seen
282                 // z1111's token yet, and we're just passing the
283                 // opaque secret part, so z3333 has to guess that it
284                 // belongs to z1111.
285                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
286                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
287                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
288                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
289         } {
290                 c.Logf("================ %v", trial)
291                 coll := createColl(trial.clusterID)
292
293                 cfgjson, err := conn3.ConfigGet(userctx1)
294                 c.Assert(err, check.IsNil)
295                 var cluster arvados.Cluster
296                 err = json.Unmarshal(cfgjson, &cluster)
297                 c.Assert(err, check.IsNil)
298
299                 c.Logf("TokenV2 is %s", ac1.AuthToken)
300                 host := cluster.Services.WebDAV.ExternalURL.Host
301                 s3args := []string{
302                         "--ssl", "--no-check-certificate",
303                         "--host=" + host, "--host-bucket=" + host,
304                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
305                 }
306                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
307                 c.Check(err, check.IsNil)
308                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
309
310                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
311                 // Command fails because we don't return Etag header.
312                 flen := strconv.Itoa(len(testText))
313                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
314         }
315 }
316
317 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
318         conn1 := s.testClusters["z1111"].Conn()
319         conn3 := s.testClusters["z3333"].Conn()
320         rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients()
321         anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
322
323         // Make sure anonymous token was set
324         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
325
326         // Create the collection to find its PDH (but don't save it
327         // anywhere yet)
328         var coll1 arvados.Collection
329         fs1, err := coll1.FileSystem(rootac1, rootkc1)
330         c.Assert(err, check.IsNil)
331         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
332         c.Assert(err, check.IsNil)
333         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
334         c.Assert(err, check.IsNil)
335         err = f.Close()
336         c.Assert(err, check.IsNil)
337         mtxt, err := fs1.MarshalManifest(".")
338         c.Assert(err, check.IsNil)
339         pdh := arvados.PortableDataHash(mtxt)
340
341         // Save the collection on cluster z1111.
342         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
343                 "manifest_text": mtxt,
344         }})
345         c.Assert(err, check.IsNil)
346
347         // Share it with the anonymous users group.
348         var outLink arvados.Link
349         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
350                 map[string]interface{}{"link": map[string]interface{}{
351                         "link_class": "permission",
352                         "name":       "can_read",
353                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
354                         "head_uuid":  coll1.UUID,
355                 },
356                 })
357         c.Check(err, check.IsNil)
358
359         // Current user should be z3 anonymous user
360         outUser, err := anonac3.CurrentUser()
361         c.Check(err, check.IsNil)
362         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
363
364         // Get the token uuid
365         var outAuth arvados.APIClientAuthorization
366         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
367         c.Check(err, check.IsNil)
368
369         // Make a v2 token of the z3 anonymous user, and use it on z1
370         _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2())
371         outUser2, err := anonac1.CurrentUser()
372         c.Check(err, check.IsNil)
373         // z3 anonymous user will be mapped to the z1 anonymous user
374         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
375
376         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
377         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
378         c.Check(err, check.IsNil)
379         c.Check(coll.PortableDataHash, check.Equals, pdh)
380 }
381
382 // Get a token from the login cluster (z1111), use it to submit a
383 // container request on z2222.
384 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
385         conn1 := s.testClusters["z1111"].Conn()
386         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
387         _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
388
389         // Use ac2 to get the discovery doc with a blank token, so the
390         // SDK doesn't magically pass the z1111 token to z2222 before
391         // we're ready to start our test.
392         _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("")
393         var dd map[string]interface{}
394         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
395         c.Assert(err, check.IsNil)
396
397         var (
398                 body bytes.Buffer
399                 req  *http.Request
400                 resp *http.Response
401                 u    arvados.User
402                 cr   arvados.ContainerRequest
403         )
404         json.NewEncoder(&body).Encode(map[string]interface{}{
405                 "container_request": map[string]interface{}{
406                         "command":         []string{"echo"},
407                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
408                         "cwd":             "/",
409                         "output_path":     "/",
410                 },
411         })
412         ac2.AuthToken = ac1.AuthToken
413
414         c.Log("...post CR with good (but not yet cached) token")
415         cr = arvados.ContainerRequest{}
416         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
417         c.Assert(err, check.IsNil)
418         req.Header.Set("Content-Type", "application/json")
419         err = ac2.DoAndDecode(&cr, req)
420         c.Assert(err, check.IsNil)
421         c.Logf("err == %#v", err)
422
423         c.Log("...get user with good token")
424         u = arvados.User{}
425         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
426         c.Assert(err, check.IsNil)
427         err = ac2.DoAndDecode(&u, req)
428         c.Check(err, check.IsNil)
429         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
430
431         c.Log("...post CR with good cached token")
432         cr = arvados.ContainerRequest{}
433         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
434         c.Assert(err, check.IsNil)
435         req.Header.Set("Content-Type", "application/json")
436         err = ac2.DoAndDecode(&cr, req)
437         c.Check(err, check.IsNil)
438         c.Check(cr.UUID, check.Matches, "z2222-.*")
439
440         c.Log("...post with good cached token ('OAuth2 ...')")
441         cr = arvados.ContainerRequest{}
442         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
443         c.Assert(err, check.IsNil)
444         req.Header.Set("Content-Type", "application/json")
445         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
446         resp, err = arvados.InsecureHTTPClient.Do(req)
447         c.Assert(err, check.IsNil)
448         err = json.NewDecoder(resp.Body).Decode(&cr)
449         c.Check(err, check.IsNil)
450         c.Check(cr.UUID, check.Matches, "z2222-.*")
451 }
452
453 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
454         conn1 := s.testClusters["z1111"].Conn()
455         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
456         _, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
457
458         tests := []struct {
459                 name         string
460                 token        string
461                 expectedCode int
462         }{
463                 {"Good token", ac1.AuthToken, http.StatusOK},
464                 {"Bogus token", "abcdef", http.StatusUnauthorized},
465                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
466                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
467         }
468
469         body, _ := json.Marshal(map[string]interface{}{
470                 "container_request": map[string]interface{}{
471                         "command":         []string{"echo"},
472                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
473                         "cwd":             "/",
474                         "output_path":     "/",
475                 },
476         })
477
478         for _, tt := range tests {
479                 c.Log(c.TestName() + " " + tt.name)
480                 ac1.AuthToken = tt.token
481                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
482                 c.Assert(err, check.IsNil)
483                 req.Header.Set("Content-Type", "application/json")
484                 resp, err := ac1.Do(req)
485                 c.Assert(err, check.IsNil)
486                 c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
487         }
488 }
489
490 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
491         conn1 := s.testClusters["z1111"].Conn()
492         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
493         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
494
495         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
496         c.Check(err, check.IsNil)
497         specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
498         c.Check(err, check.IsNil)
499
500         tests := []struct {
501                 path            string
502                 reqIdProvided   bool
503                 notFoundRequest bool
504         }{
505                 {"/arvados/v1/collections", false, false},
506                 {"/arvados/v1/collections", true, false},
507                 {"/arvados/v1/nonexistant", false, true},
508                 {"/arvados/v1/nonexistant", true, true},
509                 {"/arvados/v1/collections/" + coll.UUID, false, false},
510                 {"/arvados/v1/collections/" + coll.UUID, true, false},
511                 {"/arvados/v1/specimens/" + specimen.UUID, false, false},
512                 {"/arvados/v1/specimens/" + specimen.UUID, true, false},
513                 // new code path (lib/controller/router etc) - single-cluster request
514                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
515                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
516                 // new code path (lib/controller/router etc) - federated request
517                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", false, true},
518                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", true, true},
519                 // old code path (proxyRailsAPI) - single-cluster request
520                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
521                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
522                 // old code path (setupProxyRemoteCluster) - federated request
523                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", false, true},
524                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", true, true},
525         }
526
527         for _, tt := range tests {
528                 c.Log(c.TestName() + " " + tt.path)
529                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
530                 c.Assert(err, check.IsNil)
531                 customReqId := "abcdeG"
532                 if !tt.reqIdProvided {
533                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
534                 } else {
535                         req.Header.Set("X-Request-Id", customReqId)
536                 }
537                 resp, err := ac1.Do(req)
538                 c.Assert(err, check.IsNil)
539                 if tt.notFoundRequest {
540                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
541                 } else {
542                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
543                 }
544                 respHdr := resp.Header.Get("X-Request-Id")
545                 if tt.reqIdProvided {
546                         c.Check(respHdr, check.Equals, customReqId)
547                 } else {
548                         c.Check(respHdr, check.Matches, `req-[0-9a-zA-Z]{20}`)
549                 }
550                 if tt.notFoundRequest {
551                         var jresp httpserver.ErrorResponse
552                         err := json.NewDecoder(resp.Body).Decode(&jresp)
553                         c.Check(err, check.IsNil)
554                         c.Assert(jresp.Errors, check.HasLen, 1)
555                         c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`)
556                 }
557         }
558 }
559
560 // We test the direct access to the database
561 // normally an integration test would not have a database access, but in this case we need
562 // to test tokens that are secret, so there is no API response that will give them back
563 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
564         ctx := context.Background()
565         db, err := sql.Open("postgres", s.testClusters[clusterID].Super.Cluster().PostgreSQL.Connection.String())
566         c.Assert(err, check.IsNil)
567
568         conn, err := db.Conn(ctx)
569         c.Assert(err, check.IsNil)
570
571         rows, err := conn.ExecContext(ctx, `SELECT 1`)
572         c.Assert(err, check.IsNil)
573         n, err := rows.RowsAffected()
574         c.Assert(err, check.IsNil)
575         c.Assert(n, check.Equals, int64(1))
576         return db, conn
577 }
578
579 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
580 // and check the expected results accessing directly to the database if needed.
581 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
582         db, dbconn := s.dbConn(c, "z1111")
583         defer db.Close()
584         defer dbconn.Close()
585         conn1 := s.testClusters["z1111"].Conn()
586         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
587         userctx1, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
588
589         tests := []struct {
590                 name                 string
591                 token                string
592                 expectAToGetAValidCR bool
593                 expectedToken        *string
594         }{
595                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
596                 {"Bogus token", "abcdef", false, nil},
597                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
598                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
599         }
600
601         for _, tt := range tests {
602                 c.Log(c.TestName() + " " + tt.name)
603
604                 rq := map[string]interface{}{
605                         "command":         []string{"echo"},
606                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
607                         "cwd":             "/",
608                         "output_path":     "/",
609                         "runtime_token":   tt.token,
610                 }
611                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
612                 if tt.expectAToGetAValidCR {
613                         c.Check(err, check.IsNil)
614                         c.Check(cr, check.NotNil)
615                         c.Check(cr.UUID, check.Not(check.Equals), "")
616                 }
617
618                 if tt.expectedToken == nil {
619                         continue
620                 }
621
622                 c.Logf("cr.UUID: %s", cr.UUID)
623                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
624                 c.Check(row, check.NotNil)
625                 var token sql.NullString
626                 row.Scan(&token)
627                 if c.Check(token.Valid, check.Equals, true) {
628                         c.Check(token.String, check.Equals, *tt.expectedToken)
629                 }
630         }
631 }
632
633 // TestIntermediateCluster will send a container request to
634 // one cluster with another cluster as the destination
635 // and check the tokens are being handled properly
636 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
637         conn1 := s.testClusters["z1111"].Conn()
638         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
639         uctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
640
641         tests := []struct {
642                 name                 string
643                 token                string
644                 expectedRuntimeToken string
645                 expectedUUIDprefix   string
646         }{
647                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
648         }
649
650         for _, tt := range tests {
651                 c.Log(c.TestName() + " " + tt.name)
652                 rq := map[string]interface{}{
653                         "command":         []string{"echo"},
654                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
655                         "cwd":             "/",
656                         "output_path":     "/",
657                         "runtime_token":   tt.token,
658                 }
659                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
660
661                 c.Check(err, check.IsNil)
662                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
663                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
664         }
665 }
666
667 // Test for #17785
668 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) {
669         rootctx1, rootclnt1, _ := s.testClusters["z1111"].RootClients()
670         conn1 := s.testClusters["z1111"].Conn()
671
672         // Make sure LoginCluster is properly configured
673         for _, cls := range []string{"z1111", "z3333"} {
674                 c.Check(
675                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
676                         check.Equals, "z1111",
677                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
678         }
679         // Get user's UUID & attempt to create a token for it on the remote cluster
680         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1,
681                 "user@example.com", true)
682         _, rootclnt3, _ := s.testClusters["z3333"].ClientsWithToken(rootclnt1.AuthToken)
683         var resp arvados.APIClientAuthorization
684         err := rootclnt3.RequestAndDecode(
685                 &resp, "POST", "arvados/v1/api_client_authorizations", nil,
686                 map[string]interface{}{
687                         "api_client_authorization": map[string]string{
688                                 "owner_uuid": user.UUID,
689                         },
690                 },
691         )
692         c.Assert(err, check.IsNil)
693         newTok := resp.TokenV2()
694         c.Assert(newTok, check.Not(check.Equals), "")
695
696         // Confirm the token is from z1111
697         c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true)
698
699         // Confirm the token works and is from the correct user
700         _, rootclnt3bis, _ := s.testClusters["z3333"].ClientsWithToken(newTok)
701         var curUser arvados.User
702         err = rootclnt3bis.RequestAndDecode(
703                 &curUser, "GET", "arvados/v1/users/current", nil, nil,
704         )
705         c.Assert(err, check.IsNil)
706         c.Assert(curUser.UUID, check.Equals, user.UUID)
707 }
708
709 // Test for bug #18076
710 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
711         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
712         _, rootclnt3, _ := s.testClusters["z3333"].RootClients()
713         conn1 := s.testClusters["z1111"].Conn()
714         conn3 := s.testClusters["z3333"].Conn()
715
716         // Make sure LoginCluster is properly configured
717         for _, cls := range []string{"z1111", "z3333"} {
718                 c.Check(
719                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
720                         check.Equals, "z1111",
721                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
722         }
723
724         for testCaseNr, testCase := range []struct {
725                 name           string
726                 withRepository bool
727         }{
728                 {"User without local repository", false},
729                 {"User with local repository", true},
730         } {
731                 c.Log(c.TestName() + " " + testCase.name)
732                 // Create some users, request them on the federated cluster so they're cached.
733                 var users []arvados.User
734                 for userNr := 0; userNr < 2; userNr++ {
735                         _, _, _, user := s.testClusters["z1111"].UserClients(
736                                 rootctx1,
737                                 c,
738                                 conn1,
739                                 fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
740                                 true)
741                         c.Assert(user.Username, check.Not(check.Equals), "")
742                         users = append(users, user)
743
744                         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
745                         c.Assert(err, check.Equals, nil)
746                         userFound := false
747                         for _, fedUser := range lst.Items {
748                                 if fedUser.UUID == user.UUID {
749                                         c.Assert(fedUser.Username, check.Equals, user.Username)
750                                         userFound = true
751                                         break
752                                 }
753                         }
754                         c.Assert(userFound, check.Equals, true)
755
756                         if testCase.withRepository {
757                                 var repo interface{}
758                                 err = rootclnt3.RequestAndDecode(
759                                         &repo, "POST", "arvados/v1/repositories", nil,
760                                         map[string]interface{}{
761                                                 "repository": map[string]string{
762                                                         "name":       fmt.Sprintf("%s/test", user.Username),
763                                                         "owner_uuid": user.UUID,
764                                                 },
765                                         },
766                                 )
767                                 c.Assert(err, check.IsNil)
768                         }
769                 }
770
771                 // Swap the usernames
772                 _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
773                         UUID: users[0].UUID,
774                         Attrs: map[string]interface{}{
775                                 "username": "",
776                         },
777                 })
778                 c.Assert(err, check.Equals, nil)
779                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
780                         UUID: users[1].UUID,
781                         Attrs: map[string]interface{}{
782                                 "username": users[0].Username,
783                         },
784                 })
785                 c.Assert(err, check.Equals, nil)
786                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
787                         UUID: users[0].UUID,
788                         Attrs: map[string]interface{}{
789                                 "username": users[1].Username,
790                         },
791                 })
792                 c.Assert(err, check.Equals, nil)
793
794                 // Re-request the list on the federated cluster & check for updates
795                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
796                 c.Assert(err, check.Equals, nil)
797                 var user0Found, user1Found bool
798                 for _, user := range lst.Items {
799                         if user.UUID == users[0].UUID {
800                                 user0Found = true
801                                 c.Assert(user.Username, check.Equals, users[1].Username)
802                         } else if user.UUID == users[1].UUID {
803                                 user1Found = true
804                                 c.Assert(user.Username, check.Equals, users[0].Username)
805                         }
806                 }
807                 c.Assert(user0Found, check.Equals, true)
808                 c.Assert(user1Found, check.Equals, true)
809         }
810 }
811
812 // Test for bug #16263
813 func (s *IntegrationSuite) TestListUsers(c *check.C) {
814         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
815         conn1 := s.testClusters["z1111"].Conn()
816         conn3 := s.testClusters["z3333"].Conn()
817         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
818
819         // Make sure LoginCluster is properly configured
820         for cls := range s.testClusters {
821                 c.Check(
822                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
823                         check.Equals, "z1111",
824                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
825         }
826         // Make sure z1111 has users with NULL usernames
827         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
828                 Limit: math.MaxInt64, // check that large limit works (see #16263)
829         })
830         nullUsername := false
831         c.Assert(err, check.IsNil)
832         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
833         for _, user := range lst.Items {
834                 if user.Username == "" {
835                         nullUsername = true
836                         break
837                 }
838         }
839         c.Assert(nullUsername, check.Equals, true)
840
841         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
842         c.Assert(err, check.IsNil)
843         c.Check(user1.IsActive, check.Equals, true)
844
845         // Ask for the user list on z3333 using z1111's system root token
846         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
847         c.Assert(err, check.IsNil)
848         found := false
849         for _, user := range lst.Items {
850                 if user.UUID == user1.UUID {
851                         c.Check(user.IsActive, check.Equals, true)
852                         found = true
853                         break
854                 }
855         }
856         c.Check(found, check.Equals, true)
857
858         // Deactivate user acct on z1111
859         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
860         c.Assert(err, check.IsNil)
861
862         // Get user list from z3333, check the returned z1111 user is
863         // deactivated
864         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
865         c.Assert(err, check.IsNil)
866         found = false
867         for _, user := range lst.Items {
868                 if user.UUID == user1.UUID {
869                         c.Check(user.IsActive, check.Equals, false)
870                         found = true
871                         break
872                 }
873         }
874         c.Check(found, check.Equals, true)
875
876         // Deactivated user no longer has working token
877         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
878         c.Assert(err, check.ErrorMatches, `.*401 Unauthorized.*`)
879 }
880
881 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
882         conn1 := s.testClusters["z1111"].Conn()
883         conn3 := s.testClusters["z3333"].Conn()
884         rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients()
885
886         // Create user on LoginCluster z1111
887         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
888
889         // Make a new root token (because rootClients() uses SystemRootToken)
890         var outAuth arvados.APIClientAuthorization
891         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
892         c.Check(err, check.IsNil)
893
894         // Make a v2 root token to communicate with z3333
895         rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2())
896
897         // Create VM on z3333
898         var outVM arvados.VirtualMachine
899         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
900                 map[string]interface{}{"virtual_machine": map[string]interface{}{
901                         "hostname": "example",
902                 },
903                 })
904         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
905         c.Check(err, check.IsNil)
906
907         // Make sure z3333 user list is up to date
908         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
909         c.Check(err, check.IsNil)
910
911         // Try to set up user on z3333 with the VM
912         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
913         c.Check(err, check.IsNil)
914
915         var outLinks arvados.LinkList
916         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
917                 arvados.ListOptions{
918                         Limit: 1000,
919                         Filters: []arvados.Filter{
920                                 {
921                                         Attr:     "tail_uuid",
922                                         Operator: "=",
923                                         Operand:  user.UUID,
924                                 },
925                                 {
926                                         Attr:     "head_uuid",
927                                         Operator: "=",
928                                         Operand:  outVM.UUID,
929                                 },
930                                 {
931                                         Attr:     "name",
932                                         Operator: "=",
933                                         Operand:  "can_login",
934                                 },
935                                 {
936                                         Attr:     "link_class",
937                                         Operator: "=",
938                                         Operand:  "permission",
939                                 }}})
940         c.Check(err, check.IsNil)
941
942         c.Check(len(outLinks.Items), check.Equals, 1)
943 }
944
945 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
946         conn1 := s.testClusters["z1111"].Conn()
947         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
948         s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
949
950         accesstoken := s.oidcprovider.ValidAccessToken()
951
952         for _, clusterID := range []string{"z1111", "z2222"} {
953
954                 var coll arvados.Collection
955
956                 // Write some file data and create a collection
957                 {
958                         c.Logf("save collection to %s", clusterID)
959
960                         conn := s.testClusters[clusterID].Conn()
961                         ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
962
963                         fs, err := coll.FileSystem(ac, kc)
964                         c.Assert(err, check.IsNil)
965                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
966                         c.Assert(err, check.IsNil)
967                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
968                         c.Assert(err, check.IsNil)
969                         err = f.Close()
970                         c.Assert(err, check.IsNil)
971                         mtxt, err := fs.MarshalManifest(".")
972                         c.Assert(err, check.IsNil)
973                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
974                                 "manifest_text": mtxt,
975                         }})
976                         c.Assert(err, check.IsNil)
977                 }
978
979                 // Read the collection & file data -- both from the
980                 // cluster where it was created, and from the other
981                 // cluster.
982                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
983                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
984
985                         conn := s.testClusters[readClusterID].Conn()
986                         ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken)
987
988                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
989                         c.Assert(err, check.IsNil)
990                         c.Check(user.FullName, check.Equals, "Example User")
991                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
992                         c.Assert(err, check.IsNil)
993                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
994                         fs, err := readcoll.FileSystem(ac, kc)
995                         c.Assert(err, check.IsNil)
996                         f, err := fs.Open("test.txt")
997                         c.Assert(err, check.IsNil)
998                         buf, err := ioutil.ReadAll(f)
999                         c.Assert(err, check.IsNil)
1000                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
1001                 }
1002         }
1003 }
1004
1005 // z3333 should not forward a locally-issued container runtime token,
1006 // associated with a z1111 user, to its login cluster z1111. z1111
1007 // would only call back to z3333 and then reject the response because
1008 // the user ID does not match the token prefix. See
1009 // dev.arvados.org/issues/18346
1010 func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) {
1011         db3, db3conn := s.dbConn(c, "z3333")
1012         defer db3.Close()
1013         defer db3conn.Close()
1014         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
1015         rootctx3, _, _ := s.testClusters["z3333"].RootClients()
1016         conn1 := s.testClusters["z1111"].Conn()
1017         conn3 := s.testClusters["z3333"].Conn()
1018         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
1019
1020         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
1021         c.Assert(err, check.IsNil)
1022         c.Logf("user1 %+v", user1)
1023
1024         imageColl, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1025                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n",
1026         }})
1027         c.Assert(err, check.IsNil)
1028         c.Logf("imageColl %+v", imageColl)
1029
1030         cr, err := conn3.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1031                 "state":           "Committed",
1032                 "command":         []string{"echo"},
1033                 "container_image": imageColl.PortableDataHash,
1034                 "cwd":             "/",
1035                 "output_path":     "/",
1036                 "priority":        1,
1037                 "runtime_constraints": arvados.RuntimeConstraints{
1038                         VCPUs: 1,
1039                         RAM:   1000000000,
1040                 },
1041         }})
1042         c.Assert(err, check.IsNil)
1043         c.Logf("container request %+v", cr)
1044         ctr, err := conn3.ContainerLock(rootctx3, arvados.GetOptions{UUID: cr.ContainerUUID})
1045         c.Assert(err, check.IsNil)
1046         c.Logf("container %+v", ctr)
1047
1048         // We could use conn3.ContainerAuth() here, but that API
1049         // hasn't been added to sdk/go/arvados/api.go yet.
1050         row := db3conn.QueryRowContext(context.Background(), `SELECT api_token from api_client_authorizations where uuid=$1`, ctr.AuthUUID)
1051         c.Check(row, check.NotNil)
1052         var val sql.NullString
1053         row.Scan(&val)
1054         c.Assert(val.Valid, check.Equals, true)
1055         runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String
1056         ctrctx, _, _ := s.testClusters["z3333"].ClientsWithToken(runtimeToken)
1057         c.Logf("container runtime token %+v", runtimeToken)
1058
1059         _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID})
1060         c.Assert(err, check.NotNil)
1061         c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`)
1062         c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`)
1063 }