Merge branch '14018-acr-set-container-properties' into main
[arvados.git] / lib / controller / integration_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "context"
10         "database/sql"
11         "encoding/json"
12         "fmt"
13         "io"
14         "io/ioutil"
15         "math"
16         "net"
17         "net/http"
18         "os"
19         "os/exec"
20         "path/filepath"
21         "strconv"
22         "strings"
23         "sync"
24
25         "git.arvados.org/arvados.git/lib/boot"
26         "git.arvados.org/arvados.git/lib/config"
27         "git.arvados.org/arvados.git/sdk/go/arvados"
28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
30         "git.arvados.org/arvados.git/sdk/go/httpserver"
31         check "gopkg.in/check.v1"
32 )
33
34 var _ = check.Suite(&IntegrationSuite{})
35
36 type IntegrationSuite struct {
37         testClusters map[string]*boot.TestCluster
38         oidcprovider *arvadostest.OIDCProvider
39 }
40
41 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
42         cwd, _ := os.Getwd()
43
44         s.oidcprovider = arvadostest.NewOIDCProvider(c)
45         s.oidcprovider.AuthEmail = "user@example.com"
46         s.oidcprovider.AuthEmailVerified = true
47         s.oidcprovider.AuthName = "Example User"
48         s.oidcprovider.ValidClientID = "clientid"
49         s.oidcprovider.ValidClientSecret = "clientsecret"
50
51         s.testClusters = map[string]*boot.TestCluster{
52                 "z1111": nil,
53                 "z2222": nil,
54                 "z3333": nil,
55         }
56         hostport := map[string]string{}
57         for id := range s.testClusters {
58                 hostport[id] = func() string {
59                         // TODO: Instead of expecting random ports on
60                         // 127.0.0.11, 22, 33 to be race-safe, try
61                         // different 127.x.y.z until finding one that
62                         // isn't in use.
63                         ln, err := net.Listen("tcp", ":0")
64                         c.Assert(err, check.IsNil)
65                         ln.Close()
66                         _, port, err := net.SplitHostPort(ln.Addr().String())
67                         c.Assert(err, check.IsNil)
68                         return "127.0.0." + id[3:] + ":" + port
69                 }()
70         }
71         for id := range s.testClusters {
72                 yaml := `Clusters:
73   ` + id + `:
74     Services:
75       Controller:
76         ExternalURL: https://` + hostport[id] + `
77     TLS:
78       Insecure: true
79     SystemLogs:
80       Format: text
81     RemoteClusters:
82       z1111:
83         Host: ` + hostport["z1111"] + `
84         Scheme: https
85         Insecure: true
86         Proxy: true
87         ActivateUsers: true
88 `
89                 if id != "z2222" {
90                         yaml += `      z2222:
91         Host: ` + hostport["z2222"] + `
92         Scheme: https
93         Insecure: true
94         Proxy: true
95         ActivateUsers: true
96 `
97                 }
98                 if id != "z3333" {
99                         yaml += `      z3333:
100         Host: ` + hostport["z3333"] + `
101         Scheme: https
102         Insecure: true
103         Proxy: true
104         ActivateUsers: true
105 `
106                 }
107                 if id == "z1111" {
108                         yaml += `
109     Login:
110       LoginCluster: z1111
111       OpenIDConnect:
112         Enable: true
113         Issuer: ` + s.oidcprovider.Issuer.URL + `
114         ClientID: ` + s.oidcprovider.ValidClientID + `
115         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
116         EmailClaim: email
117         EmailVerifiedClaim: email_verified
118         AcceptAccessToken: true
119         AcceptAccessTokenScope: ""
120 `
121                 } else {
122                         yaml += `
123     Login:
124       LoginCluster: z1111
125 `
126                 }
127
128                 loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
129                 loader.Path = "-"
130                 loader.SkipLegacy = true
131                 loader.SkipAPICalls = true
132                 cfg, err := loader.Load()
133                 c.Assert(err, check.IsNil)
134                 tc := boot.NewTestCluster(
135                         filepath.Join(cwd, "..", ".."),
136                         id, cfg, "127.0.0."+id[3:], c.Log)
137                 tc.Super.NoWorkbench1 = true
138                 tc.Start()
139                 s.testClusters[id] = tc
140         }
141         for _, tc := range s.testClusters {
142                 ok := tc.WaitReady()
143                 c.Assert(ok, check.Equals, true)
144         }
145 }
146
147 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
148         for _, c := range s.testClusters {
149                 c.Super.Stop()
150         }
151 }
152
153 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
154         conn := s.testClusters["z1111"].Conn()
155         rootctx, _, _ := s.testClusters["z1111"].RootClients()
156         userctx, _, kc, _ := s.testClusters["z1111"].UserClients(rootctx, c, conn, s.oidcprovider.AuthEmail, true)
157         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
158         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
159         c.Assert(err, check.IsNil)
160         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
161 }
162
163 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
164         conn1 := s.testClusters["z1111"].Conn()
165         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
166         conn3 := s.testClusters["z3333"].Conn()
167         userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
168
169         // Create the collection to find its PDH (but don't save it
170         // anywhere yet)
171         var coll1 arvados.Collection
172         fs1, err := coll1.FileSystem(ac1, kc1)
173         c.Assert(err, check.IsNil)
174         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
175         c.Assert(err, check.IsNil)
176         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
177         c.Assert(err, check.IsNil)
178         err = f.Close()
179         c.Assert(err, check.IsNil)
180         mtxt, err := fs1.MarshalManifest(".")
181         c.Assert(err, check.IsNil)
182         pdh := arvados.PortableDataHash(mtxt)
183
184         // Looking up the PDH before saving returns 404 if cycle
185         // detection is working.
186         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
187         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
188
189         // Save the collection on cluster z1111.
190         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
191                 "manifest_text": mtxt,
192         }})
193         c.Assert(err, check.IsNil)
194
195         // Retrieve the collection from cluster z3333.
196         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
197         c.Check(err, check.IsNil)
198         c.Check(coll.PortableDataHash, check.Equals, pdh)
199 }
200
201 // Tests bug #18004
202 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
203         conn1 := s.testClusters["z1111"].Conn()
204         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
205         rootctx2, _, _ := s.testClusters["z2222"].RootClients()
206         conn2 := s.testClusters["z2222"].Conn()
207         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user2@example.com", true)
208
209         var wg1, wg2 sync.WaitGroup
210         creqs := 100
211
212         // Make concurrent requests to z2222 with a local token to make sure more
213         // than one worker is listening.
214         wg1.Add(1)
215         for i := 0; i < creqs; i++ {
216                 wg2.Add(1)
217                 go func() {
218                         defer wg2.Done()
219                         wg1.Wait()
220                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
221                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
222                 }()
223         }
224         wg1.Done()
225         wg2.Wait()
226
227         // Real test pass -- use a new remote token than the one used in the warm-up
228         // phase.
229         wg1.Add(1)
230         for i := 0; i < creqs; i++ {
231                 wg2.Add(1)
232                 go func() {
233                         defer wg2.Done()
234                         wg1.Wait()
235                         // Retrieve the remote collection from cluster z2222.
236                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
237                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
238                 }()
239         }
240         wg1.Done()
241         wg2.Wait()
242 }
243
244 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
245         if _, err := exec.LookPath("s3cmd"); err != nil {
246                 c.Skip("s3cmd not in PATH")
247                 return
248         }
249
250         testText := "IntegrationSuite.TestS3WithFederatedToken"
251
252         conn1 := s.testClusters["z1111"].Conn()
253         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
254         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
255         conn3 := s.testClusters["z3333"].Conn()
256
257         createColl := func(clusterID string) arvados.Collection {
258                 _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken)
259                 var coll arvados.Collection
260                 fs, err := coll.FileSystem(ac, kc)
261                 c.Assert(err, check.IsNil)
262                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
263                 c.Assert(err, check.IsNil)
264                 _, err = io.WriteString(f, testText)
265                 c.Assert(err, check.IsNil)
266                 err = f.Close()
267                 c.Assert(err, check.IsNil)
268                 mtxt, err := fs.MarshalManifest(".")
269                 c.Assert(err, check.IsNil)
270                 coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
271                         "manifest_text": mtxt,
272                 }})
273                 c.Assert(err, check.IsNil)
274                 return coll
275         }
276
277         for _, trial := range []struct {
278                 clusterID string // create the collection on this cluster (then use z3333 to access it)
279                 token     string
280         }{
281                 // Try the hardest test first: z3333 hasn't seen
282                 // z1111's token yet, and we're just passing the
283                 // opaque secret part, so z3333 has to guess that it
284                 // belongs to z1111.
285                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
286                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
287                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
288                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
289         } {
290                 c.Logf("================ %v", trial)
291                 coll := createColl(trial.clusterID)
292
293                 cfgjson, err := conn3.ConfigGet(userctx1)
294                 c.Assert(err, check.IsNil)
295                 var cluster arvados.Cluster
296                 err = json.Unmarshal(cfgjson, &cluster)
297                 c.Assert(err, check.IsNil)
298
299                 c.Logf("TokenV2 is %s", ac1.AuthToken)
300                 host := cluster.Services.WebDAV.ExternalURL.Host
301                 s3args := []string{
302                         "--ssl", "--no-check-certificate",
303                         "--host=" + host, "--host-bucket=" + host,
304                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
305                 }
306                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
307                 c.Check(err, check.IsNil)
308                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
309
310                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
311                 // Command fails because we don't return Etag header.
312                 flen := strconv.Itoa(len(testText))
313                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
314         }
315 }
316
317 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
318         conn1 := s.testClusters["z1111"].Conn()
319         conn3 := s.testClusters["z3333"].Conn()
320         rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients()
321         anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
322
323         // Make sure anonymous token was set
324         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
325
326         // Create the collection to find its PDH (but don't save it
327         // anywhere yet)
328         var coll1 arvados.Collection
329         fs1, err := coll1.FileSystem(rootac1, rootkc1)
330         c.Assert(err, check.IsNil)
331         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
332         c.Assert(err, check.IsNil)
333         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
334         c.Assert(err, check.IsNil)
335         err = f.Close()
336         c.Assert(err, check.IsNil)
337         mtxt, err := fs1.MarshalManifest(".")
338         c.Assert(err, check.IsNil)
339         pdh := arvados.PortableDataHash(mtxt)
340
341         // Save the collection on cluster z1111.
342         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
343                 "manifest_text": mtxt,
344         }})
345         c.Assert(err, check.IsNil)
346
347         // Share it with the anonymous users group.
348         var outLink arvados.Link
349         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
350                 map[string]interface{}{"link": map[string]interface{}{
351                         "link_class": "permission",
352                         "name":       "can_read",
353                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
354                         "head_uuid":  coll1.UUID,
355                 },
356                 })
357         c.Check(err, check.IsNil)
358
359         // Current user should be z3 anonymous user
360         outUser, err := anonac3.CurrentUser()
361         c.Check(err, check.IsNil)
362         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
363
364         // Get the token uuid
365         var outAuth arvados.APIClientAuthorization
366         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
367         c.Check(err, check.IsNil)
368
369         // Make a v2 token of the z3 anonymous user, and use it on z1
370         _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2())
371         outUser2, err := anonac1.CurrentUser()
372         c.Check(err, check.IsNil)
373         // z3 anonymous user will be mapped to the z1 anonymous user
374         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
375
376         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
377         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
378         c.Check(err, check.IsNil)
379         c.Check(coll.PortableDataHash, check.Equals, pdh)
380 }
381
382 // Get a token from the login cluster (z1111), use it to submit a
383 // container request on z2222.
384 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
385         conn1 := s.testClusters["z1111"].Conn()
386         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
387         _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
388
389         // Use ac2 to get the discovery doc with a blank token, so the
390         // SDK doesn't magically pass the z1111 token to z2222 before
391         // we're ready to start our test.
392         _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("")
393         var dd map[string]interface{}
394         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
395         c.Assert(err, check.IsNil)
396
397         var (
398                 body bytes.Buffer
399                 req  *http.Request
400                 resp *http.Response
401                 u    arvados.User
402                 cr   arvados.ContainerRequest
403         )
404         json.NewEncoder(&body).Encode(map[string]interface{}{
405                 "container_request": map[string]interface{}{
406                         "command":         []string{"echo"},
407                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
408                         "cwd":             "/",
409                         "output_path":     "/",
410                 },
411         })
412         ac2.AuthToken = ac1.AuthToken
413
414         c.Log("...post CR with good (but not yet cached) token")
415         cr = arvados.ContainerRequest{}
416         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
417         c.Assert(err, check.IsNil)
418         req.Header.Set("Content-Type", "application/json")
419         err = ac2.DoAndDecode(&cr, req)
420         c.Assert(err, check.IsNil)
421         c.Logf("err == %#v", err)
422
423         c.Log("...get user with good token")
424         u = arvados.User{}
425         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
426         c.Assert(err, check.IsNil)
427         err = ac2.DoAndDecode(&u, req)
428         c.Check(err, check.IsNil)
429         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
430
431         c.Log("...post CR with good cached token")
432         cr = arvados.ContainerRequest{}
433         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
434         c.Assert(err, check.IsNil)
435         req.Header.Set("Content-Type", "application/json")
436         err = ac2.DoAndDecode(&cr, req)
437         c.Check(err, check.IsNil)
438         c.Check(cr.UUID, check.Matches, "z2222-.*")
439
440         c.Log("...post with good cached token ('OAuth2 ...')")
441         cr = arvados.ContainerRequest{}
442         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
443         c.Assert(err, check.IsNil)
444         req.Header.Set("Content-Type", "application/json")
445         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
446         resp, err = arvados.InsecureHTTPClient.Do(req)
447         c.Assert(err, check.IsNil)
448         err = json.NewDecoder(resp.Body).Decode(&cr)
449         c.Check(err, check.IsNil)
450         c.Check(cr.UUID, check.Matches, "z2222-.*")
451 }
452
453 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
454         conn1 := s.testClusters["z1111"].Conn()
455         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
456         _, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
457
458         tests := []struct {
459                 name         string
460                 token        string
461                 expectedCode int
462         }{
463                 {"Good token", ac1.AuthToken, http.StatusOK},
464                 {"Bogus token", "abcdef", http.StatusUnauthorized},
465                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
466                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
467         }
468
469         body, _ := json.Marshal(map[string]interface{}{
470                 "container_request": map[string]interface{}{
471                         "command":         []string{"echo"},
472                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
473                         "cwd":             "/",
474                         "output_path":     "/",
475                 },
476         })
477
478         for _, tt := range tests {
479                 c.Log(c.TestName() + " " + tt.name)
480                 ac1.AuthToken = tt.token
481                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
482                 c.Assert(err, check.IsNil)
483                 req.Header.Set("Content-Type", "application/json")
484                 resp, err := ac1.Do(req)
485                 c.Assert(err, check.IsNil)
486                 c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
487         }
488 }
489
490 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
491         conn1 := s.testClusters["z1111"].Conn()
492         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
493         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
494
495         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
496         c.Check(err, check.IsNil)
497         specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
498         c.Check(err, check.IsNil)
499
500         tests := []struct {
501                 path            string
502                 reqIdProvided   bool
503                 notFoundRequest bool
504         }{
505                 {"/arvados/v1/collections", false, false},
506                 {"/arvados/v1/collections", true, false},
507                 {"/arvados/v1/nonexistant", false, true},
508                 {"/arvados/v1/nonexistant", true, true},
509                 {"/arvados/v1/collections/" + coll.UUID, false, false},
510                 {"/arvados/v1/collections/" + coll.UUID, true, false},
511                 {"/arvados/v1/specimens/" + specimen.UUID, false, false},
512                 {"/arvados/v1/specimens/" + specimen.UUID, true, false},
513                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
514                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
515                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
516                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
517         }
518
519         for _, tt := range tests {
520                 c.Log(c.TestName() + " " + tt.path)
521                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
522                 c.Assert(err, check.IsNil)
523                 customReqId := "abcdeG"
524                 if !tt.reqIdProvided {
525                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
526                 } else {
527                         req.Header.Set("X-Request-Id", customReqId)
528                 }
529                 resp, err := ac1.Do(req)
530                 c.Assert(err, check.IsNil)
531                 if tt.notFoundRequest {
532                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
533                 } else {
534                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
535                 }
536                 if !tt.reqIdProvided {
537                         c.Check(resp.Header.Get("X-Request-Id"), check.Matches, "^req-[0-9a-zA-Z]{20}$")
538                         if tt.notFoundRequest {
539                                 var jresp httpserver.ErrorResponse
540                                 err := json.NewDecoder(resp.Body).Decode(&jresp)
541                                 c.Check(err, check.IsNil)
542                                 c.Assert(jresp.Errors, check.HasLen, 1)
543                                 c.Check(jresp.Errors[0], check.Matches, "^.*(req-[0-9a-zA-Z]{20}).*$")
544                         }
545                 } else {
546                         c.Check(resp.Header.Get("X-Request-Id"), check.Equals, customReqId)
547                         if tt.notFoundRequest {
548                                 var jresp httpserver.ErrorResponse
549                                 err := json.NewDecoder(resp.Body).Decode(&jresp)
550                                 c.Check(err, check.IsNil)
551                                 c.Assert(jresp.Errors, check.HasLen, 1)
552                                 c.Check(jresp.Errors[0], check.Matches, "^.*("+customReqId+").*$")
553                         }
554                 }
555         }
556 }
557
558 // We test the direct access to the database
559 // normally an integration test would not have a database access, but in this case we need
560 // to test tokens that are secret, so there is no API response that will give them back
561 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
562         ctx := context.Background()
563         db, err := sql.Open("postgres", s.testClusters[clusterID].Super.Cluster().PostgreSQL.Connection.String())
564         c.Assert(err, check.IsNil)
565
566         conn, err := db.Conn(ctx)
567         c.Assert(err, check.IsNil)
568
569         rows, err := conn.ExecContext(ctx, `SELECT 1`)
570         c.Assert(err, check.IsNil)
571         n, err := rows.RowsAffected()
572         c.Assert(err, check.IsNil)
573         c.Assert(n, check.Equals, int64(1))
574         return db, conn
575 }
576
577 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
578 // and check the expected results accessing directly to the database if needed.
579 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
580         db, dbconn := s.dbConn(c, "z1111")
581         defer db.Close()
582         defer dbconn.Close()
583         conn1 := s.testClusters["z1111"].Conn()
584         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
585         userctx1, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
586
587         tests := []struct {
588                 name                 string
589                 token                string
590                 expectAToGetAValidCR bool
591                 expectedToken        *string
592         }{
593                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
594                 {"Bogus token", "abcdef", false, nil},
595                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
596                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
597         }
598
599         for _, tt := range tests {
600                 c.Log(c.TestName() + " " + tt.name)
601
602                 rq := map[string]interface{}{
603                         "command":         []string{"echo"},
604                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
605                         "cwd":             "/",
606                         "output_path":     "/",
607                         "runtime_token":   tt.token,
608                 }
609                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
610                 if tt.expectAToGetAValidCR {
611                         c.Check(err, check.IsNil)
612                         c.Check(cr, check.NotNil)
613                         c.Check(cr.UUID, check.Not(check.Equals), "")
614                 }
615
616                 if tt.expectedToken == nil {
617                         continue
618                 }
619
620                 c.Logf("cr.UUID: %s", cr.UUID)
621                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
622                 c.Check(row, check.NotNil)
623                 var token sql.NullString
624                 row.Scan(&token)
625                 if c.Check(token.Valid, check.Equals, true) {
626                         c.Check(token.String, check.Equals, *tt.expectedToken)
627                 }
628         }
629 }
630
631 // TestIntermediateCluster will send a container request to
632 // one cluster with another cluster as the destination
633 // and check the tokens are being handled properly
634 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
635         conn1 := s.testClusters["z1111"].Conn()
636         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
637         uctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
638
639         tests := []struct {
640                 name                 string
641                 token                string
642                 expectedRuntimeToken string
643                 expectedUUIDprefix   string
644         }{
645                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
646         }
647
648         for _, tt := range tests {
649                 c.Log(c.TestName() + " " + tt.name)
650                 rq := map[string]interface{}{
651                         "command":         []string{"echo"},
652                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
653                         "cwd":             "/",
654                         "output_path":     "/",
655                         "runtime_token":   tt.token,
656                 }
657                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
658
659                 c.Check(err, check.IsNil)
660                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
661                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
662         }
663 }
664
665 // Test for bug #18076
666 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
667         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
668         _, rootclnt3, _ := s.testClusters["z3333"].RootClients()
669         conn1 := s.testClusters["z1111"].Conn()
670         conn3 := s.testClusters["z3333"].Conn()
671
672         // Make sure LoginCluster is properly configured
673         for cls := range s.testClusters {
674                 if cls == "z1111" || cls == "z3333" {
675                         c.Check(
676                                 s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
677                                 check.Equals, "z1111",
678                                 check.Commentf("incorrect LoginCluster config on cluster %q", cls))
679                 }
680         }
681
682         for testCaseNr, testCase := range []struct {
683                 name           string
684                 withRepository bool
685         }{
686                 {"User without local repository", false},
687                 {"User with local repository", true},
688         } {
689                 c.Log(c.TestName() + " " + testCase.name)
690                 // Create some users, request them on the federated cluster so they're cached.
691                 var users []arvados.User
692                 for userNr := 0; userNr < 2; userNr++ {
693                         _, _, _, user := s.testClusters["z1111"].UserClients(
694                                 rootctx1,
695                                 c,
696                                 conn1,
697                                 fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
698                                 true)
699                         c.Assert(user.Username, check.Not(check.Equals), "")
700                         users = append(users, user)
701
702                         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
703                         c.Assert(err, check.Equals, nil)
704                         userFound := false
705                         for _, fedUser := range lst.Items {
706                                 if fedUser.UUID == user.UUID {
707                                         c.Assert(fedUser.Username, check.Equals, user.Username)
708                                         userFound = true
709                                         break
710                                 }
711                         }
712                         c.Assert(userFound, check.Equals, true)
713
714                         if testCase.withRepository {
715                                 var repo interface{}
716                                 err = rootclnt3.RequestAndDecode(
717                                         &repo, "POST", "arvados/v1/repositories", nil,
718                                         map[string]interface{}{
719                                                 "repository": map[string]string{
720                                                         "name":       fmt.Sprintf("%s/test", user.Username),
721                                                         "owner_uuid": user.UUID,
722                                                 },
723                                         },
724                                 )
725                                 c.Assert(err, check.IsNil)
726                         }
727                 }
728
729                 // Swap the usernames
730                 _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
731                         UUID: users[0].UUID,
732                         Attrs: map[string]interface{}{
733                                 "username": "",
734                         },
735                 })
736                 c.Assert(err, check.Equals, nil)
737                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
738                         UUID: users[1].UUID,
739                         Attrs: map[string]interface{}{
740                                 "username": users[0].Username,
741                         },
742                 })
743                 c.Assert(err, check.Equals, nil)
744                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
745                         UUID: users[0].UUID,
746                         Attrs: map[string]interface{}{
747                                 "username": users[1].Username,
748                         },
749                 })
750                 c.Assert(err, check.Equals, nil)
751
752                 // Re-request the list on the federated cluster & check for updates
753                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
754                 c.Assert(err, check.Equals, nil)
755                 var user0Found, user1Found bool
756                 for _, user := range lst.Items {
757                         if user.UUID == users[0].UUID {
758                                 user0Found = true
759                                 c.Assert(user.Username, check.Equals, users[1].Username)
760                         } else if user.UUID == users[1].UUID {
761                                 user1Found = true
762                                 c.Assert(user.Username, check.Equals, users[0].Username)
763                         }
764                 }
765                 c.Assert(user0Found, check.Equals, true)
766                 c.Assert(user1Found, check.Equals, true)
767         }
768 }
769
770 // Test for bug #16263
771 func (s *IntegrationSuite) TestListUsers(c *check.C) {
772         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
773         conn1 := s.testClusters["z1111"].Conn()
774         conn3 := s.testClusters["z3333"].Conn()
775         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
776
777         // Make sure LoginCluster is properly configured
778         for cls := range s.testClusters {
779                 c.Check(
780                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
781                         check.Equals, "z1111",
782                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
783         }
784         // Make sure z1111 has users with NULL usernames
785         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
786                 Limit: math.MaxInt64, // check that large limit works (see #16263)
787         })
788         nullUsername := false
789         c.Assert(err, check.IsNil)
790         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
791         for _, user := range lst.Items {
792                 if user.Username == "" {
793                         nullUsername = true
794                         break
795                 }
796         }
797         c.Assert(nullUsername, check.Equals, true)
798
799         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
800         c.Assert(err, check.IsNil)
801         c.Check(user1.IsActive, check.Equals, true)
802
803         // Ask for the user list on z3333 using z1111's system root token
804         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
805         c.Assert(err, check.IsNil)
806         found := false
807         for _, user := range lst.Items {
808                 if user.UUID == user1.UUID {
809                         c.Check(user.IsActive, check.Equals, true)
810                         found = true
811                         break
812                 }
813         }
814         c.Check(found, check.Equals, true)
815
816         // Deactivate user acct on z1111
817         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
818         c.Assert(err, check.IsNil)
819
820         // Get user list from z3333, check the returned z1111 user is
821         // deactivated
822         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
823         c.Assert(err, check.IsNil)
824         found = false
825         for _, user := range lst.Items {
826                 if user.UUID == user1.UUID {
827                         c.Check(user.IsActive, check.Equals, false)
828                         found = true
829                         break
830                 }
831         }
832         c.Check(found, check.Equals, true)
833
834         // Deactivated user can see is_active==false via "get current
835         // user" API
836         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
837         c.Assert(err, check.IsNil)
838         c.Check(user1.IsActive, check.Equals, false)
839 }
840
841 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
842         conn1 := s.testClusters["z1111"].Conn()
843         conn3 := s.testClusters["z3333"].Conn()
844         rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients()
845
846         // Create user on LoginCluster z1111
847         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
848
849         // Make a new root token (because rootClients() uses SystemRootToken)
850         var outAuth arvados.APIClientAuthorization
851         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
852         c.Check(err, check.IsNil)
853
854         // Make a v2 root token to communicate with z3333
855         rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2())
856
857         // Create VM on z3333
858         var outVM arvados.VirtualMachine
859         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
860                 map[string]interface{}{"virtual_machine": map[string]interface{}{
861                         "hostname": "example",
862                 },
863                 })
864         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
865         c.Check(err, check.IsNil)
866
867         // Make sure z3333 user list is up to date
868         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
869         c.Check(err, check.IsNil)
870
871         // Try to set up user on z3333 with the VM
872         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
873         c.Check(err, check.IsNil)
874
875         var outLinks arvados.LinkList
876         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
877                 arvados.ListOptions{
878                         Limit: 1000,
879                         Filters: []arvados.Filter{
880                                 {
881                                         Attr:     "tail_uuid",
882                                         Operator: "=",
883                                         Operand:  user.UUID,
884                                 },
885                                 {
886                                         Attr:     "head_uuid",
887                                         Operator: "=",
888                                         Operand:  outVM.UUID,
889                                 },
890                                 {
891                                         Attr:     "name",
892                                         Operator: "=",
893                                         Operand:  "can_login",
894                                 },
895                                 {
896                                         Attr:     "link_class",
897                                         Operator: "=",
898                                         Operand:  "permission",
899                                 }}})
900         c.Check(err, check.IsNil)
901
902         c.Check(len(outLinks.Items), check.Equals, 1)
903 }
904
905 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
906         conn1 := s.testClusters["z1111"].Conn()
907         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
908         s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
909
910         accesstoken := s.oidcprovider.ValidAccessToken()
911
912         for _, clusterID := range []string{"z1111", "z2222"} {
913
914                 var coll arvados.Collection
915
916                 // Write some file data and create a collection
917                 {
918                         c.Logf("save collection to %s", clusterID)
919
920                         conn := s.testClusters[clusterID].Conn()
921                         ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
922
923                         fs, err := coll.FileSystem(ac, kc)
924                         c.Assert(err, check.IsNil)
925                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
926                         c.Assert(err, check.IsNil)
927                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
928                         c.Assert(err, check.IsNil)
929                         err = f.Close()
930                         c.Assert(err, check.IsNil)
931                         mtxt, err := fs.MarshalManifest(".")
932                         c.Assert(err, check.IsNil)
933                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
934                                 "manifest_text": mtxt,
935                         }})
936                         c.Assert(err, check.IsNil)
937                 }
938
939                 // Read the collection & file data -- both from the
940                 // cluster where it was created, and from the other
941                 // cluster.
942                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
943                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
944
945                         conn := s.testClusters[readClusterID].Conn()
946                         ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken)
947
948                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
949                         c.Assert(err, check.IsNil)
950                         c.Check(user.FullName, check.Equals, "Example User")
951                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
952                         c.Assert(err, check.IsNil)
953                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
954                         fs, err := readcoll.FileSystem(ac, kc)
955                         c.Assert(err, check.IsNil)
956                         f, err := fs.Open("test.txt")
957                         c.Assert(err, check.IsNil)
958                         buf, err := ioutil.ReadAll(f)
959                         c.Assert(err, check.IsNil)
960                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
961                 }
962         }
963 }