Merge branch '18376-nfs-readdirent'
[arvados.git] / lib / controller / integration_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "context"
10         "database/sql"
11         "encoding/json"
12         "fmt"
13         "io"
14         "io/ioutil"
15         "math"
16         "net"
17         "net/http"
18         "os"
19         "os/exec"
20         "path/filepath"
21         "strconv"
22         "strings"
23         "sync"
24
25         "git.arvados.org/arvados.git/lib/boot"
26         "git.arvados.org/arvados.git/lib/config"
27         "git.arvados.org/arvados.git/sdk/go/arvados"
28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
30         "git.arvados.org/arvados.git/sdk/go/httpserver"
31         check "gopkg.in/check.v1"
32 )
33
34 var _ = check.Suite(&IntegrationSuite{})
35
36 type IntegrationSuite struct {
37         testClusters map[string]*boot.TestCluster
38         oidcprovider *arvadostest.OIDCProvider
39 }
40
41 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
42         cwd, _ := os.Getwd()
43
44         s.oidcprovider = arvadostest.NewOIDCProvider(c)
45         s.oidcprovider.AuthEmail = "user@example.com"
46         s.oidcprovider.AuthEmailVerified = true
47         s.oidcprovider.AuthName = "Example User"
48         s.oidcprovider.ValidClientID = "clientid"
49         s.oidcprovider.ValidClientSecret = "clientsecret"
50
51         s.testClusters = map[string]*boot.TestCluster{
52                 "z1111": nil,
53                 "z2222": nil,
54                 "z3333": nil,
55         }
56         hostport := map[string]string{}
57         for id := range s.testClusters {
58                 hostport[id] = func() string {
59                         // TODO: Instead of expecting random ports on
60                         // 127.0.0.11, 22, 33 to be race-safe, try
61                         // different 127.x.y.z until finding one that
62                         // isn't in use.
63                         ln, err := net.Listen("tcp", ":0")
64                         c.Assert(err, check.IsNil)
65                         ln.Close()
66                         _, port, err := net.SplitHostPort(ln.Addr().String())
67                         c.Assert(err, check.IsNil)
68                         return "127.0.0." + id[3:] + ":" + port
69                 }()
70         }
71         for id := range s.testClusters {
72                 yaml := `Clusters:
73   ` + id + `:
74     Services:
75       Controller:
76         ExternalURL: https://` + hostport[id] + `
77     TLS:
78       Insecure: true
79     SystemLogs:
80       Format: text
81     RemoteClusters:
82       z1111:
83         Host: ` + hostport["z1111"] + `
84         Scheme: https
85         Insecure: true
86         Proxy: true
87         ActivateUsers: true
88 `
89                 if id != "z2222" {
90                         yaml += `      z2222:
91         Host: ` + hostport["z2222"] + `
92         Scheme: https
93         Insecure: true
94         Proxy: true
95         ActivateUsers: true
96 `
97                 }
98                 if id != "z3333" {
99                         yaml += `      z3333:
100         Host: ` + hostport["z3333"] + `
101         Scheme: https
102         Insecure: true
103         Proxy: true
104         ActivateUsers: true
105 `
106                 }
107                 if id == "z1111" {
108                         yaml += `
109     Login:
110       LoginCluster: z1111
111       OpenIDConnect:
112         Enable: true
113         Issuer: ` + s.oidcprovider.Issuer.URL + `
114         ClientID: ` + s.oidcprovider.ValidClientID + `
115         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
116         EmailClaim: email
117         EmailVerifiedClaim: email_verified
118         AcceptAccessToken: true
119         AcceptAccessTokenScope: ""
120 `
121                 } else {
122                         yaml += `
123     Login:
124       LoginCluster: z1111
125 `
126                 }
127
128                 loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
129                 loader.Path = "-"
130                 loader.SkipLegacy = true
131                 loader.SkipAPICalls = true
132                 cfg, err := loader.Load()
133                 c.Assert(err, check.IsNil)
134                 tc := boot.NewTestCluster(
135                         filepath.Join(cwd, "..", ".."),
136                         id, cfg, "127.0.0."+id[3:], c.Log)
137                 tc.Super.NoWorkbench1 = true
138                 tc.Start()
139                 s.testClusters[id] = tc
140         }
141         for _, tc := range s.testClusters {
142                 ok := tc.WaitReady()
143                 c.Assert(ok, check.Equals, true)
144         }
145 }
146
147 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
148         for _, c := range s.testClusters {
149                 c.Super.Stop()
150         }
151 }
152
153 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
154         conn := s.testClusters["z1111"].Conn()
155         rootctx, _, _ := s.testClusters["z1111"].RootClients()
156         userctx, _, kc, _ := s.testClusters["z1111"].UserClients(rootctx, c, conn, s.oidcprovider.AuthEmail, true)
157         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
158         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
159         c.Assert(err, check.IsNil)
160         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
161 }
162
163 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
164         conn1 := s.testClusters["z1111"].Conn()
165         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
166         conn3 := s.testClusters["z3333"].Conn()
167         userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
168
169         // Create the collection to find its PDH (but don't save it
170         // anywhere yet)
171         var coll1 arvados.Collection
172         fs1, err := coll1.FileSystem(ac1, kc1)
173         c.Assert(err, check.IsNil)
174         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
175         c.Assert(err, check.IsNil)
176         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
177         c.Assert(err, check.IsNil)
178         err = f.Close()
179         c.Assert(err, check.IsNil)
180         mtxt, err := fs1.MarshalManifest(".")
181         c.Assert(err, check.IsNil)
182         pdh := arvados.PortableDataHash(mtxt)
183
184         // Looking up the PDH before saving returns 404 if cycle
185         // detection is working.
186         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
187         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
188
189         // Save the collection on cluster z1111.
190         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
191                 "manifest_text": mtxt,
192         }})
193         c.Assert(err, check.IsNil)
194
195         // Retrieve the collection from cluster z3333.
196         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
197         c.Check(err, check.IsNil)
198         c.Check(coll.PortableDataHash, check.Equals, pdh)
199 }
200
201 // Tests bug #18004
202 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
203         conn1 := s.testClusters["z1111"].Conn()
204         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
205         rootctx2, _, _ := s.testClusters["z2222"].RootClients()
206         conn2 := s.testClusters["z2222"].Conn()
207         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user2@example.com", true)
208
209         var wg1, wg2 sync.WaitGroup
210         creqs := 100
211
212         // Make concurrent requests to z2222 with a local token to make sure more
213         // than one worker is listening.
214         wg1.Add(1)
215         for i := 0; i < creqs; i++ {
216                 wg2.Add(1)
217                 go func() {
218                         defer wg2.Done()
219                         wg1.Wait()
220                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
221                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
222                 }()
223         }
224         wg1.Done()
225         wg2.Wait()
226
227         // Real test pass -- use a new remote token than the one used in the warm-up
228         // phase.
229         wg1.Add(1)
230         for i := 0; i < creqs; i++ {
231                 wg2.Add(1)
232                 go func() {
233                         defer wg2.Done()
234                         wg1.Wait()
235                         // Retrieve the remote collection from cluster z2222.
236                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
237                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
238                 }()
239         }
240         wg1.Done()
241         wg2.Wait()
242 }
243
244 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
245         if _, err := exec.LookPath("s3cmd"); err != nil {
246                 c.Skip("s3cmd not in PATH")
247                 return
248         }
249
250         testText := "IntegrationSuite.TestS3WithFederatedToken"
251
252         conn1 := s.testClusters["z1111"].Conn()
253         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
254         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
255         conn3 := s.testClusters["z3333"].Conn()
256
257         createColl := func(clusterID string) arvados.Collection {
258                 _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken)
259                 var coll arvados.Collection
260                 fs, err := coll.FileSystem(ac, kc)
261                 c.Assert(err, check.IsNil)
262                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
263                 c.Assert(err, check.IsNil)
264                 _, err = io.WriteString(f, testText)
265                 c.Assert(err, check.IsNil)
266                 err = f.Close()
267                 c.Assert(err, check.IsNil)
268                 mtxt, err := fs.MarshalManifest(".")
269                 c.Assert(err, check.IsNil)
270                 coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
271                         "manifest_text": mtxt,
272                 }})
273                 c.Assert(err, check.IsNil)
274                 return coll
275         }
276
277         for _, trial := range []struct {
278                 clusterID string // create the collection on this cluster (then use z3333 to access it)
279                 token     string
280         }{
281                 // Try the hardest test first: z3333 hasn't seen
282                 // z1111's token yet, and we're just passing the
283                 // opaque secret part, so z3333 has to guess that it
284                 // belongs to z1111.
285                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
286                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
287                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
288                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
289         } {
290                 c.Logf("================ %v", trial)
291                 coll := createColl(trial.clusterID)
292
293                 cfgjson, err := conn3.ConfigGet(userctx1)
294                 c.Assert(err, check.IsNil)
295                 var cluster arvados.Cluster
296                 err = json.Unmarshal(cfgjson, &cluster)
297                 c.Assert(err, check.IsNil)
298
299                 c.Logf("TokenV2 is %s", ac1.AuthToken)
300                 host := cluster.Services.WebDAV.ExternalURL.Host
301                 s3args := []string{
302                         "--ssl", "--no-check-certificate",
303                         "--host=" + host, "--host-bucket=" + host,
304                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
305                 }
306                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
307                 c.Check(err, check.IsNil)
308                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
309
310                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
311                 // Command fails because we don't return Etag header.
312                 flen := strconv.Itoa(len(testText))
313                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
314         }
315 }
316
317 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
318         conn1 := s.testClusters["z1111"].Conn()
319         conn3 := s.testClusters["z3333"].Conn()
320         rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients()
321         anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
322
323         // Make sure anonymous token was set
324         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
325
326         // Create the collection to find its PDH (but don't save it
327         // anywhere yet)
328         var coll1 arvados.Collection
329         fs1, err := coll1.FileSystem(rootac1, rootkc1)
330         c.Assert(err, check.IsNil)
331         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
332         c.Assert(err, check.IsNil)
333         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
334         c.Assert(err, check.IsNil)
335         err = f.Close()
336         c.Assert(err, check.IsNil)
337         mtxt, err := fs1.MarshalManifest(".")
338         c.Assert(err, check.IsNil)
339         pdh := arvados.PortableDataHash(mtxt)
340
341         // Save the collection on cluster z1111.
342         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
343                 "manifest_text": mtxt,
344         }})
345         c.Assert(err, check.IsNil)
346
347         // Share it with the anonymous users group.
348         var outLink arvados.Link
349         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
350                 map[string]interface{}{"link": map[string]interface{}{
351                         "link_class": "permission",
352                         "name":       "can_read",
353                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
354                         "head_uuid":  coll1.UUID,
355                 },
356                 })
357         c.Check(err, check.IsNil)
358
359         // Current user should be z3 anonymous user
360         outUser, err := anonac3.CurrentUser()
361         c.Check(err, check.IsNil)
362         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
363
364         // Get the token uuid
365         var outAuth arvados.APIClientAuthorization
366         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
367         c.Check(err, check.IsNil)
368
369         // Make a v2 token of the z3 anonymous user, and use it on z1
370         _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2())
371         outUser2, err := anonac1.CurrentUser()
372         c.Check(err, check.IsNil)
373         // z3 anonymous user will be mapped to the z1 anonymous user
374         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
375
376         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
377         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
378         c.Check(err, check.IsNil)
379         c.Check(coll.PortableDataHash, check.Equals, pdh)
380 }
381
382 // Get a token from the login cluster (z1111), use it to submit a
383 // container request on z2222.
384 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
385         conn1 := s.testClusters["z1111"].Conn()
386         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
387         _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
388
389         // Use ac2 to get the discovery doc with a blank token, so the
390         // SDK doesn't magically pass the z1111 token to z2222 before
391         // we're ready to start our test.
392         _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("")
393         var dd map[string]interface{}
394         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
395         c.Assert(err, check.IsNil)
396
397         var (
398                 body bytes.Buffer
399                 req  *http.Request
400                 resp *http.Response
401                 u    arvados.User
402                 cr   arvados.ContainerRequest
403         )
404         json.NewEncoder(&body).Encode(map[string]interface{}{
405                 "container_request": map[string]interface{}{
406                         "command":         []string{"echo"},
407                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
408                         "cwd":             "/",
409                         "output_path":     "/",
410                 },
411         })
412         ac2.AuthToken = ac1.AuthToken
413
414         c.Log("...post CR with good (but not yet cached) token")
415         cr = arvados.ContainerRequest{}
416         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
417         c.Assert(err, check.IsNil)
418         req.Header.Set("Content-Type", "application/json")
419         err = ac2.DoAndDecode(&cr, req)
420         c.Assert(err, check.IsNil)
421         c.Logf("err == %#v", err)
422
423         c.Log("...get user with good token")
424         u = arvados.User{}
425         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
426         c.Assert(err, check.IsNil)
427         err = ac2.DoAndDecode(&u, req)
428         c.Check(err, check.IsNil)
429         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
430
431         c.Log("...post CR with good cached token")
432         cr = arvados.ContainerRequest{}
433         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
434         c.Assert(err, check.IsNil)
435         req.Header.Set("Content-Type", "application/json")
436         err = ac2.DoAndDecode(&cr, req)
437         c.Check(err, check.IsNil)
438         c.Check(cr.UUID, check.Matches, "z2222-.*")
439
440         c.Log("...post with good cached token ('OAuth2 ...')")
441         cr = arvados.ContainerRequest{}
442         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
443         c.Assert(err, check.IsNil)
444         req.Header.Set("Content-Type", "application/json")
445         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
446         resp, err = arvados.InsecureHTTPClient.Do(req)
447         c.Assert(err, check.IsNil)
448         err = json.NewDecoder(resp.Body).Decode(&cr)
449         c.Check(err, check.IsNil)
450         c.Check(cr.UUID, check.Matches, "z2222-.*")
451 }
452
453 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
454         conn1 := s.testClusters["z1111"].Conn()
455         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
456         _, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
457
458         tests := []struct {
459                 name         string
460                 token        string
461                 expectedCode int
462         }{
463                 {"Good token", ac1.AuthToken, http.StatusOK},
464                 {"Bogus token", "abcdef", http.StatusUnauthorized},
465                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
466                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
467         }
468
469         body, _ := json.Marshal(map[string]interface{}{
470                 "container_request": map[string]interface{}{
471                         "command":         []string{"echo"},
472                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
473                         "cwd":             "/",
474                         "output_path":     "/",
475                 },
476         })
477
478         for _, tt := range tests {
479                 c.Log(c.TestName() + " " + tt.name)
480                 ac1.AuthToken = tt.token
481                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
482                 c.Assert(err, check.IsNil)
483                 req.Header.Set("Content-Type", "application/json")
484                 resp, err := ac1.Do(req)
485                 c.Assert(err, check.IsNil)
486                 c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
487         }
488 }
489
490 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
491         conn1 := s.testClusters["z1111"].Conn()
492         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
493         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
494
495         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
496         c.Check(err, check.IsNil)
497         specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
498         c.Check(err, check.IsNil)
499
500         tests := []struct {
501                 path            string
502                 reqIdProvided   bool
503                 notFoundRequest bool
504         }{
505                 {"/arvados/v1/collections", false, false},
506                 {"/arvados/v1/collections", true, false},
507                 {"/arvados/v1/nonexistant", false, true},
508                 {"/arvados/v1/nonexistant", true, true},
509                 {"/arvados/v1/collections/" + coll.UUID, false, false},
510                 {"/arvados/v1/collections/" + coll.UUID, true, false},
511                 {"/arvados/v1/specimens/" + specimen.UUID, false, false},
512                 {"/arvados/v1/specimens/" + specimen.UUID, true, false},
513                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
514                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
515                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
516                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
517         }
518
519         for _, tt := range tests {
520                 c.Log(c.TestName() + " " + tt.path)
521                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
522                 c.Assert(err, check.IsNil)
523                 customReqId := "abcdeG"
524                 if !tt.reqIdProvided {
525                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
526                 } else {
527                         req.Header.Set("X-Request-Id", customReqId)
528                 }
529                 resp, err := ac1.Do(req)
530                 c.Assert(err, check.IsNil)
531                 if tt.notFoundRequest {
532                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
533                 } else {
534                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
535                 }
536                 if !tt.reqIdProvided {
537                         c.Check(resp.Header.Get("X-Request-Id"), check.Matches, "^req-[0-9a-zA-Z]{20}$")
538                         if tt.notFoundRequest {
539                                 var jresp httpserver.ErrorResponse
540                                 err := json.NewDecoder(resp.Body).Decode(&jresp)
541                                 c.Check(err, check.IsNil)
542                                 c.Assert(jresp.Errors, check.HasLen, 1)
543                                 c.Check(jresp.Errors[0], check.Matches, "^.*(req-[0-9a-zA-Z]{20}).*$")
544                         }
545                 } else {
546                         c.Check(resp.Header.Get("X-Request-Id"), check.Equals, customReqId)
547                         if tt.notFoundRequest {
548                                 var jresp httpserver.ErrorResponse
549                                 err := json.NewDecoder(resp.Body).Decode(&jresp)
550                                 c.Check(err, check.IsNil)
551                                 c.Assert(jresp.Errors, check.HasLen, 1)
552                                 c.Check(jresp.Errors[0], check.Matches, "^.*("+customReqId+").*$")
553                         }
554                 }
555         }
556 }
557
558 // We test the direct access to the database
559 // normally an integration test would not have a database access, but in this case we need
560 // to test tokens that are secret, so there is no API response that will give them back
561 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
562         ctx := context.Background()
563         db, err := sql.Open("postgres", s.testClusters[clusterID].Super.Cluster().PostgreSQL.Connection.String())
564         c.Assert(err, check.IsNil)
565
566         conn, err := db.Conn(ctx)
567         c.Assert(err, check.IsNil)
568
569         rows, err := conn.ExecContext(ctx, `SELECT 1`)
570         c.Assert(err, check.IsNil)
571         n, err := rows.RowsAffected()
572         c.Assert(err, check.IsNil)
573         c.Assert(n, check.Equals, int64(1))
574         return db, conn
575 }
576
577 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
578 // and check the expected results accessing directly to the database if needed.
579 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
580         db, dbconn := s.dbConn(c, "z1111")
581         defer db.Close()
582         defer dbconn.Close()
583         conn1 := s.testClusters["z1111"].Conn()
584         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
585         userctx1, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
586
587         tests := []struct {
588                 name                 string
589                 token                string
590                 expectAToGetAValidCR bool
591                 expectedToken        *string
592         }{
593                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
594                 {"Bogus token", "abcdef", false, nil},
595                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
596                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
597         }
598
599         for _, tt := range tests {
600                 c.Log(c.TestName() + " " + tt.name)
601
602                 rq := map[string]interface{}{
603                         "command":         []string{"echo"},
604                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
605                         "cwd":             "/",
606                         "output_path":     "/",
607                         "runtime_token":   tt.token,
608                 }
609                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
610                 if tt.expectAToGetAValidCR {
611                         c.Check(err, check.IsNil)
612                         c.Check(cr, check.NotNil)
613                         c.Check(cr.UUID, check.Not(check.Equals), "")
614                 }
615
616                 if tt.expectedToken == nil {
617                         continue
618                 }
619
620                 c.Logf("cr.UUID: %s", cr.UUID)
621                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
622                 c.Check(row, check.NotNil)
623                 var token sql.NullString
624                 row.Scan(&token)
625                 if c.Check(token.Valid, check.Equals, true) {
626                         c.Check(token.String, check.Equals, *tt.expectedToken)
627                 }
628         }
629 }
630
631 // TestIntermediateCluster will send a container request to
632 // one cluster with another cluster as the destination
633 // and check the tokens are being handled properly
634 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
635         conn1 := s.testClusters["z1111"].Conn()
636         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
637         uctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
638
639         tests := []struct {
640                 name                 string
641                 token                string
642                 expectedRuntimeToken string
643                 expectedUUIDprefix   string
644         }{
645                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
646         }
647
648         for _, tt := range tests {
649                 c.Log(c.TestName() + " " + tt.name)
650                 rq := map[string]interface{}{
651                         "command":         []string{"echo"},
652                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
653                         "cwd":             "/",
654                         "output_path":     "/",
655                         "runtime_token":   tt.token,
656                 }
657                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
658
659                 c.Check(err, check.IsNil)
660                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
661                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
662         }
663 }
664
665 // Test for #17785
666 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) {
667         rootctx1, rootclnt1, _ := s.testClusters["z1111"].RootClients()
668         conn1 := s.testClusters["z1111"].Conn()
669
670         // Make sure LoginCluster is properly configured
671         for _, cls := range []string{"z1111", "z3333"} {
672                 c.Check(
673                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
674                         check.Equals, "z1111",
675                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
676         }
677         // Get user's UUID & attempt to create a token for it on the remote cluster
678         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1,
679                 "user@example.com", true)
680         _, rootclnt3, _ := s.testClusters["z3333"].ClientsWithToken(rootclnt1.AuthToken)
681         var resp arvados.APIClientAuthorization
682         err := rootclnt3.RequestAndDecode(
683                 &resp, "POST", "arvados/v1/api_client_authorizations", nil,
684                 map[string]interface{}{
685                         "api_client_authorization": map[string]string{
686                                 "owner_uuid": user.UUID,
687                         },
688                 },
689         )
690         c.Assert(err, check.IsNil)
691         newTok := resp.TokenV2()
692         c.Assert(newTok, check.Not(check.Equals), "")
693
694         // Confirm the token is from z1111
695         c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true)
696
697         // Confirm the token works and is from the correct user
698         _, rootclnt3bis, _ := s.testClusters["z3333"].ClientsWithToken(newTok)
699         var curUser arvados.User
700         err = rootclnt3bis.RequestAndDecode(
701                 &curUser, "GET", "arvados/v1/users/current", nil, nil,
702         )
703         c.Assert(err, check.IsNil)
704         c.Assert(curUser.UUID, check.Equals, user.UUID)
705 }
706
707 // Test for bug #18076
708 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
709         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
710         _, rootclnt3, _ := s.testClusters["z3333"].RootClients()
711         conn1 := s.testClusters["z1111"].Conn()
712         conn3 := s.testClusters["z3333"].Conn()
713
714         // Make sure LoginCluster is properly configured
715         for _, cls := range []string{"z1111", "z3333"} {
716                 c.Check(
717                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
718                         check.Equals, "z1111",
719                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
720         }
721
722         for testCaseNr, testCase := range []struct {
723                 name           string
724                 withRepository bool
725         }{
726                 {"User without local repository", false},
727                 {"User with local repository", true},
728         } {
729                 c.Log(c.TestName() + " " + testCase.name)
730                 // Create some users, request them on the federated cluster so they're cached.
731                 var users []arvados.User
732                 for userNr := 0; userNr < 2; userNr++ {
733                         _, _, _, user := s.testClusters["z1111"].UserClients(
734                                 rootctx1,
735                                 c,
736                                 conn1,
737                                 fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
738                                 true)
739                         c.Assert(user.Username, check.Not(check.Equals), "")
740                         users = append(users, user)
741
742                         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
743                         c.Assert(err, check.Equals, nil)
744                         userFound := false
745                         for _, fedUser := range lst.Items {
746                                 if fedUser.UUID == user.UUID {
747                                         c.Assert(fedUser.Username, check.Equals, user.Username)
748                                         userFound = true
749                                         break
750                                 }
751                         }
752                         c.Assert(userFound, check.Equals, true)
753
754                         if testCase.withRepository {
755                                 var repo interface{}
756                                 err = rootclnt3.RequestAndDecode(
757                                         &repo, "POST", "arvados/v1/repositories", nil,
758                                         map[string]interface{}{
759                                                 "repository": map[string]string{
760                                                         "name":       fmt.Sprintf("%s/test", user.Username),
761                                                         "owner_uuid": user.UUID,
762                                                 },
763                                         },
764                                 )
765                                 c.Assert(err, check.IsNil)
766                         }
767                 }
768
769                 // Swap the usernames
770                 _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
771                         UUID: users[0].UUID,
772                         Attrs: map[string]interface{}{
773                                 "username": "",
774                         },
775                 })
776                 c.Assert(err, check.Equals, nil)
777                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
778                         UUID: users[1].UUID,
779                         Attrs: map[string]interface{}{
780                                 "username": users[0].Username,
781                         },
782                 })
783                 c.Assert(err, check.Equals, nil)
784                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
785                         UUID: users[0].UUID,
786                         Attrs: map[string]interface{}{
787                                 "username": users[1].Username,
788                         },
789                 })
790                 c.Assert(err, check.Equals, nil)
791
792                 // Re-request the list on the federated cluster & check for updates
793                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
794                 c.Assert(err, check.Equals, nil)
795                 var user0Found, user1Found bool
796                 for _, user := range lst.Items {
797                         if user.UUID == users[0].UUID {
798                                 user0Found = true
799                                 c.Assert(user.Username, check.Equals, users[1].Username)
800                         } else if user.UUID == users[1].UUID {
801                                 user1Found = true
802                                 c.Assert(user.Username, check.Equals, users[0].Username)
803                         }
804                 }
805                 c.Assert(user0Found, check.Equals, true)
806                 c.Assert(user1Found, check.Equals, true)
807         }
808 }
809
810 // Test for bug #16263
811 func (s *IntegrationSuite) TestListUsers(c *check.C) {
812         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
813         conn1 := s.testClusters["z1111"].Conn()
814         conn3 := s.testClusters["z3333"].Conn()
815         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
816
817         // Make sure LoginCluster is properly configured
818         for cls := range s.testClusters {
819                 c.Check(
820                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
821                         check.Equals, "z1111",
822                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
823         }
824         // Make sure z1111 has users with NULL usernames
825         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
826                 Limit: math.MaxInt64, // check that large limit works (see #16263)
827         })
828         nullUsername := false
829         c.Assert(err, check.IsNil)
830         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
831         for _, user := range lst.Items {
832                 if user.Username == "" {
833                         nullUsername = true
834                         break
835                 }
836         }
837         c.Assert(nullUsername, check.Equals, true)
838
839         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
840         c.Assert(err, check.IsNil)
841         c.Check(user1.IsActive, check.Equals, true)
842
843         // Ask for the user list on z3333 using z1111's system root token
844         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
845         c.Assert(err, check.IsNil)
846         found := false
847         for _, user := range lst.Items {
848                 if user.UUID == user1.UUID {
849                         c.Check(user.IsActive, check.Equals, true)
850                         found = true
851                         break
852                 }
853         }
854         c.Check(found, check.Equals, true)
855
856         // Deactivate user acct on z1111
857         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
858         c.Assert(err, check.IsNil)
859
860         // Get user list from z3333, check the returned z1111 user is
861         // deactivated
862         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
863         c.Assert(err, check.IsNil)
864         found = false
865         for _, user := range lst.Items {
866                 if user.UUID == user1.UUID {
867                         c.Check(user.IsActive, check.Equals, false)
868                         found = true
869                         break
870                 }
871         }
872         c.Check(found, check.Equals, true)
873
874         // Deactivated user no longer has working token
875         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
876         c.Assert(err, check.ErrorMatches, `.*401 Unauthorized.*`)
877 }
878
879 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
880         conn1 := s.testClusters["z1111"].Conn()
881         conn3 := s.testClusters["z3333"].Conn()
882         rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients()
883
884         // Create user on LoginCluster z1111
885         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
886
887         // Make a new root token (because rootClients() uses SystemRootToken)
888         var outAuth arvados.APIClientAuthorization
889         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
890         c.Check(err, check.IsNil)
891
892         // Make a v2 root token to communicate with z3333
893         rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2())
894
895         // Create VM on z3333
896         var outVM arvados.VirtualMachine
897         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
898                 map[string]interface{}{"virtual_machine": map[string]interface{}{
899                         "hostname": "example",
900                 },
901                 })
902         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
903         c.Check(err, check.IsNil)
904
905         // Make sure z3333 user list is up to date
906         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
907         c.Check(err, check.IsNil)
908
909         // Try to set up user on z3333 with the VM
910         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
911         c.Check(err, check.IsNil)
912
913         var outLinks arvados.LinkList
914         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
915                 arvados.ListOptions{
916                         Limit: 1000,
917                         Filters: []arvados.Filter{
918                                 {
919                                         Attr:     "tail_uuid",
920                                         Operator: "=",
921                                         Operand:  user.UUID,
922                                 },
923                                 {
924                                         Attr:     "head_uuid",
925                                         Operator: "=",
926                                         Operand:  outVM.UUID,
927                                 },
928                                 {
929                                         Attr:     "name",
930                                         Operator: "=",
931                                         Operand:  "can_login",
932                                 },
933                                 {
934                                         Attr:     "link_class",
935                                         Operator: "=",
936                                         Operand:  "permission",
937                                 }}})
938         c.Check(err, check.IsNil)
939
940         c.Check(len(outLinks.Items), check.Equals, 1)
941 }
942
943 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
944         conn1 := s.testClusters["z1111"].Conn()
945         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
946         s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
947
948         accesstoken := s.oidcprovider.ValidAccessToken()
949
950         for _, clusterID := range []string{"z1111", "z2222"} {
951
952                 var coll arvados.Collection
953
954                 // Write some file data and create a collection
955                 {
956                         c.Logf("save collection to %s", clusterID)
957
958                         conn := s.testClusters[clusterID].Conn()
959                         ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
960
961                         fs, err := coll.FileSystem(ac, kc)
962                         c.Assert(err, check.IsNil)
963                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
964                         c.Assert(err, check.IsNil)
965                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
966                         c.Assert(err, check.IsNil)
967                         err = f.Close()
968                         c.Assert(err, check.IsNil)
969                         mtxt, err := fs.MarshalManifest(".")
970                         c.Assert(err, check.IsNil)
971                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
972                                 "manifest_text": mtxt,
973                         }})
974                         c.Assert(err, check.IsNil)
975                 }
976
977                 // Read the collection & file data -- both from the
978                 // cluster where it was created, and from the other
979                 // cluster.
980                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
981                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
982
983                         conn := s.testClusters[readClusterID].Conn()
984                         ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken)
985
986                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
987                         c.Assert(err, check.IsNil)
988                         c.Check(user.FullName, check.Equals, "Example User")
989                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
990                         c.Assert(err, check.IsNil)
991                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
992                         fs, err := readcoll.FileSystem(ac, kc)
993                         c.Assert(err, check.IsNil)
994                         f, err := fs.Open("test.txt")
995                         c.Assert(err, check.IsNil)
996                         buf, err := ioutil.ReadAll(f)
997                         c.Assert(err, check.IsNil)
998                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
999                 }
1000         }
1001 }
1002
1003 // z3333 should not forward a locally-issued container runtime token,
1004 // associated with a z1111 user, to its login cluster z1111. z1111
1005 // would only call back to z3333 and then reject the response because
1006 // the user ID does not match the token prefix. See
1007 // dev.arvados.org/issues/18346
1008 func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) {
1009         db3, db3conn := s.dbConn(c, "z3333")
1010         defer db3.Close()
1011         defer db3conn.Close()
1012         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
1013         rootctx3, _, _ := s.testClusters["z3333"].RootClients()
1014         conn1 := s.testClusters["z1111"].Conn()
1015         conn3 := s.testClusters["z3333"].Conn()
1016         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
1017
1018         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
1019         c.Assert(err, check.IsNil)
1020         c.Logf("user1 %+v", user1)
1021
1022         imageColl, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1023                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n",
1024         }})
1025         c.Assert(err, check.IsNil)
1026         c.Logf("imageColl %+v", imageColl)
1027
1028         cr, err := conn3.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1029                 "state":           "Committed",
1030                 "command":         []string{"echo"},
1031                 "container_image": imageColl.PortableDataHash,
1032                 "cwd":             "/",
1033                 "output_path":     "/",
1034                 "priority":        1,
1035                 "runtime_constraints": arvados.RuntimeConstraints{
1036                         VCPUs: 1,
1037                         RAM:   1000000000,
1038                 },
1039         }})
1040         c.Assert(err, check.IsNil)
1041         c.Logf("container request %+v", cr)
1042         ctr, err := conn3.ContainerLock(rootctx3, arvados.GetOptions{UUID: cr.ContainerUUID})
1043         c.Assert(err, check.IsNil)
1044         c.Logf("container %+v", ctr)
1045
1046         // We could use conn3.ContainerAuth() here, but that API
1047         // hasn't been added to sdk/go/arvados/api.go yet.
1048         row := db3conn.QueryRowContext(context.Background(), `SELECT api_token from api_client_authorizations where uuid=$1`, ctr.AuthUUID)
1049         c.Check(row, check.NotNil)
1050         var val sql.NullString
1051         row.Scan(&val)
1052         c.Assert(val.Valid, check.Equals, true)
1053         runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String
1054         ctrctx, _, _ := s.testClusters["z3333"].ClientsWithToken(runtimeToken)
1055         c.Logf("container runtime token %+v", runtimeToken)
1056
1057         _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID})
1058         c.Assert(err, check.NotNil)
1059         c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`)
1060         c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`)
1061 }