16981: Adds federation test on sync-groups.
[arvados.git] / lib / controller / integration_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "encoding/json"
10         "fmt"
11         "io"
12         "io/ioutil"
13         "math"
14         "net"
15         "net/http"
16         "os"
17         "os/exec"
18         "path/filepath"
19         "strconv"
20         "strings"
21
22         "git.arvados.org/arvados.git/sdk/go/arvados"
23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
24         check "gopkg.in/check.v1"
25 )
26
27 var _ = check.Suite(&IntegrationSuite{})
28
29 type IntegrationSuite struct {
30         testClusters map[string]*arvadostest.TestCluster
31         oidcprovider *arvadostest.OIDCProvider
32 }
33
34 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
35         if forceLegacyAPI14 {
36                 c.Skip("heavy integration tests don't run with forceLegacyAPI14")
37                 return
38         }
39
40         cwd, _ := os.Getwd()
41
42         s.oidcprovider = arvadostest.NewOIDCProvider(c)
43         s.oidcprovider.AuthEmail = "user@example.com"
44         s.oidcprovider.AuthEmailVerified = true
45         s.oidcprovider.AuthName = "Example User"
46         s.oidcprovider.ValidClientID = "clientid"
47         s.oidcprovider.ValidClientSecret = "clientsecret"
48
49         s.testClusters = map[string]*arvadostest.TestCluster{
50                 "z1111": nil,
51                 "z2222": nil,
52                 "z3333": nil,
53         }
54         hostport := map[string]string{}
55         for id := range s.testClusters {
56                 hostport[id] = func() string {
57                         // TODO: Instead of expecting random ports on
58                         // 127.0.0.11, 22, 33 to be race-safe, try
59                         // different 127.x.y.z until finding one that
60                         // isn't in use.
61                         ln, err := net.Listen("tcp", ":0")
62                         c.Assert(err, check.IsNil)
63                         ln.Close()
64                         _, port, err := net.SplitHostPort(ln.Addr().String())
65                         c.Assert(err, check.IsNil)
66                         return "127.0.0." + id[3:] + ":" + port
67                 }()
68         }
69         for id := range s.testClusters {
70                 yaml := `Clusters:
71   ` + id + `:
72     Services:
73       Controller:
74         ExternalURL: https://` + hostport[id] + `
75     TLS:
76       Insecure: true
77     SystemLogs:
78       Format: text
79     RemoteClusters:
80       z1111:
81         Host: ` + hostport["z1111"] + `
82         Scheme: https
83         Insecure: true
84         Proxy: true
85         ActivateUsers: true
86 `
87                 if id != "z2222" {
88                         yaml += `      z2222:
89         Host: ` + hostport["z2222"] + `
90         Scheme: https
91         Insecure: true
92         Proxy: true
93         ActivateUsers: true
94 `
95                 }
96                 if id != "z3333" {
97                         yaml += `      z3333:
98         Host: ` + hostport["z3333"] + `
99         Scheme: https
100         Insecure: true
101         Proxy: true
102         ActivateUsers: true
103 `
104                 }
105                 if id == "z1111" {
106                         yaml += `
107     Login:
108       LoginCluster: z1111
109       OpenIDConnect:
110         Enable: true
111         Issuer: ` + s.oidcprovider.Issuer.URL + `
112         ClientID: ` + s.oidcprovider.ValidClientID + `
113         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
114         EmailClaim: email
115         EmailVerifiedClaim: email_verified
116 `
117                 } else {
118                         yaml += `
119     Login:
120       LoginCluster: z1111
121 `
122                 }
123
124                 tc, err := arvadostest.NewTestCluster(
125                         filepath.Join(cwd, "..", ".."),
126                         id, yaml, "127.0.0."+id[3:], c.Log)
127                 c.Assert(err, check.IsNil)
128                 s.testClusters[id] = tc
129                 s.testClusters[id].Start()
130         }
131         for _, tc := range s.testClusters {
132                 ok := tc.WaitReady()
133                 c.Assert(ok, check.Equals, true)
134         }
135 }
136
137 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
138         for _, c := range s.testClusters {
139                 c.Super.Stop()
140         }
141 }
142
143 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
144         conn1 := s.testClusters["z1111"].Conn()
145         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
146         conn3 := s.testClusters["z3333"].Conn()
147         userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
148
149         // Create the collection to find its PDH (but don't save it
150         // anywhere yet)
151         var coll1 arvados.Collection
152         fs1, err := coll1.FileSystem(ac1, kc1)
153         c.Assert(err, check.IsNil)
154         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
155         c.Assert(err, check.IsNil)
156         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
157         c.Assert(err, check.IsNil)
158         err = f.Close()
159         c.Assert(err, check.IsNil)
160         mtxt, err := fs1.MarshalManifest(".")
161         c.Assert(err, check.IsNil)
162         pdh := arvados.PortableDataHash(mtxt)
163
164         // Looking up the PDH before saving returns 404 if cycle
165         // detection is working.
166         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
167         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
168
169         // Save the collection on cluster z1111.
170         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
171                 "manifest_text": mtxt,
172         }})
173         c.Assert(err, check.IsNil)
174
175         // Retrieve the collection from cluster z3333.
176         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
177         c.Check(err, check.IsNil)
178         c.Check(coll.PortableDataHash, check.Equals, pdh)
179 }
180
181 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
182         if _, err := exec.LookPath("s3cmd"); err != nil {
183                 c.Skip("s3cmd not in PATH")
184                 return
185         }
186
187         testText := "IntegrationSuite.TestS3WithFederatedToken"
188
189         conn1 := s.testClusters["z1111"].Conn()
190         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
191         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
192         conn3 := s.testClusters["z3333"].Conn()
193
194         createColl := func(clusterID string) arvados.Collection {
195                 _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken)
196                 var coll arvados.Collection
197                 fs, err := coll.FileSystem(ac, kc)
198                 c.Assert(err, check.IsNil)
199                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
200                 c.Assert(err, check.IsNil)
201                 _, err = io.WriteString(f, testText)
202                 c.Assert(err, check.IsNil)
203                 err = f.Close()
204                 c.Assert(err, check.IsNil)
205                 mtxt, err := fs.MarshalManifest(".")
206                 c.Assert(err, check.IsNil)
207                 coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
208                         "manifest_text": mtxt,
209                 }})
210                 c.Assert(err, check.IsNil)
211                 return coll
212         }
213
214         for _, trial := range []struct {
215                 clusterID string // create the collection on this cluster (then use z3333 to access it)
216                 token     string
217         }{
218                 // Try the hardest test first: z3333 hasn't seen
219                 // z1111's token yet, and we're just passing the
220                 // opaque secret part, so z3333 has to guess that it
221                 // belongs to z1111.
222                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
223                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
224                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
225                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
226         } {
227                 c.Logf("================ %v", trial)
228                 coll := createColl(trial.clusterID)
229
230                 cfgjson, err := conn3.ConfigGet(userctx1)
231                 c.Assert(err, check.IsNil)
232                 var cluster arvados.Cluster
233                 err = json.Unmarshal(cfgjson, &cluster)
234                 c.Assert(err, check.IsNil)
235
236                 c.Logf("TokenV2 is %s", ac1.AuthToken)
237                 host := cluster.Services.WebDAV.ExternalURL.Host
238                 s3args := []string{
239                         "--ssl", "--no-check-certificate",
240                         "--host=" + host, "--host-bucket=" + host,
241                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
242                 }
243                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
244                 c.Check(err, check.IsNil)
245                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
246
247                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
248                 // Command fails because we don't return Etag header.
249                 flen := strconv.Itoa(len(testText))
250                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
251         }
252 }
253
254 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
255         conn1 := s.testClusters["z1111"].Conn()
256         conn3 := s.testClusters["z3333"].Conn()
257         rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients()
258         anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
259
260         // Make sure anonymous token was set
261         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
262
263         // Create the collection to find its PDH (but don't save it
264         // anywhere yet)
265         var coll1 arvados.Collection
266         fs1, err := coll1.FileSystem(rootac1, rootkc1)
267         c.Assert(err, check.IsNil)
268         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
269         c.Assert(err, check.IsNil)
270         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
271         c.Assert(err, check.IsNil)
272         err = f.Close()
273         c.Assert(err, check.IsNil)
274         mtxt, err := fs1.MarshalManifest(".")
275         c.Assert(err, check.IsNil)
276         pdh := arvados.PortableDataHash(mtxt)
277
278         // Save the collection on cluster z1111.
279         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
280                 "manifest_text": mtxt,
281         }})
282         c.Assert(err, check.IsNil)
283
284         // Share it with the anonymous users group.
285         var outLink arvados.Link
286         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
287                 map[string]interface{}{"link": map[string]interface{}{
288                         "link_class": "permission",
289                         "name":       "can_read",
290                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
291                         "head_uuid":  coll1.UUID,
292                 },
293                 })
294         c.Check(err, check.IsNil)
295
296         // Current user should be z3 anonymous user
297         outUser, err := anonac3.CurrentUser()
298         c.Check(err, check.IsNil)
299         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
300
301         // Get the token uuid
302         var outAuth arvados.APIClientAuthorization
303         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
304         c.Check(err, check.IsNil)
305
306         // Make a v2 token of the z3 anonymous user, and use it on z1
307         _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2())
308         outUser2, err := anonac1.CurrentUser()
309         c.Check(err, check.IsNil)
310         // z3 anonymous user will be mapped to the z1 anonymous user
311         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
312
313         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
314         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
315         c.Check(err, check.IsNil)
316         c.Check(coll.PortableDataHash, check.Equals, pdh)
317 }
318
319 // Get a token from the login cluster (z1111), use it to submit a
320 // container request on z2222.
321 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
322         conn1 := s.testClusters["z1111"].Conn()
323         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
324         _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
325
326         // Use ac2 to get the discovery doc with a blank token, so the
327         // SDK doesn't magically pass the z1111 token to z2222 before
328         // we're ready to start our test.
329         _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("")
330         var dd map[string]interface{}
331         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
332         c.Assert(err, check.IsNil)
333
334         var (
335                 body bytes.Buffer
336                 req  *http.Request
337                 resp *http.Response
338                 u    arvados.User
339                 cr   arvados.ContainerRequest
340         )
341         json.NewEncoder(&body).Encode(map[string]interface{}{
342                 "container_request": map[string]interface{}{
343                         "command":         []string{"echo"},
344                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
345                         "cwd":             "/",
346                         "output_path":     "/",
347                 },
348         })
349         ac2.AuthToken = ac1.AuthToken
350
351         c.Log("...post CR with good (but not yet cached) token")
352         cr = arvados.ContainerRequest{}
353         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
354         c.Assert(err, check.IsNil)
355         req.Header.Set("Content-Type", "application/json")
356         err = ac2.DoAndDecode(&cr, req)
357         c.Logf("err == %#v", err)
358
359         c.Log("...get user with good token")
360         u = arvados.User{}
361         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
362         c.Assert(err, check.IsNil)
363         err = ac2.DoAndDecode(&u, req)
364         c.Check(err, check.IsNil)
365         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
366
367         c.Log("...post CR with good cached token")
368         cr = arvados.ContainerRequest{}
369         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
370         c.Assert(err, check.IsNil)
371         req.Header.Set("Content-Type", "application/json")
372         err = ac2.DoAndDecode(&cr, req)
373         c.Check(err, check.IsNil)
374         c.Check(cr.UUID, check.Matches, "z2222-.*")
375
376         c.Log("...post with good cached token ('OAuth2 ...')")
377         cr = arvados.ContainerRequest{}
378         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
379         c.Assert(err, check.IsNil)
380         req.Header.Set("Content-Type", "application/json")
381         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
382         resp, err = arvados.InsecureHTTPClient.Do(req)
383         if c.Check(err, check.IsNil) {
384                 err = json.NewDecoder(resp.Body).Decode(&cr)
385                 c.Check(err, check.IsNil)
386                 c.Check(cr.UUID, check.Matches, "z2222-.*")
387         }
388 }
389
390 // Test for bug #16263
391 func (s *IntegrationSuite) TestListUsers(c *check.C) {
392         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
393         conn1 := s.testClusters["z1111"].Conn()
394         conn3 := s.testClusters["z3333"].Conn()
395         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
396
397         // Make sure LoginCluster is properly configured
398         for cls := range s.testClusters {
399                 c.Check(
400                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
401                         check.Equals, "z1111",
402                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
403         }
404         // Make sure z1111 has users with NULL usernames
405         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
406                 Limit: math.MaxInt64, // check that large limit works (see #16263)
407         })
408         nullUsername := false
409         c.Assert(err, check.IsNil)
410         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
411         for _, user := range lst.Items {
412                 if user.Username == "" {
413                         nullUsername = true
414                 }
415         }
416         c.Assert(nullUsername, check.Equals, true)
417
418         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
419         c.Assert(err, check.IsNil)
420         c.Check(user1.IsActive, check.Equals, true)
421
422         // Ask for the user list on z3333 using z1111's system root token
423         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
424         c.Assert(err, check.IsNil)
425         found := false
426         for _, user := range lst.Items {
427                 if user.UUID == user1.UUID {
428                         c.Check(user.IsActive, check.Equals, true)
429                         found = true
430                         break
431                 }
432         }
433         c.Check(found, check.Equals, true)
434
435         // Deactivate user acct on z1111
436         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
437         c.Assert(err, check.IsNil)
438
439         // Get user list from z3333, check the returned z1111 user is
440         // deactivated
441         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
442         c.Assert(err, check.IsNil)
443         found = false
444         for _, user := range lst.Items {
445                 if user.UUID == user1.UUID {
446                         c.Check(user.IsActive, check.Equals, false)
447                         found = true
448                         break
449                 }
450         }
451         c.Check(found, check.Equals, true)
452
453         // Deactivated user can see is_active==false via "get current
454         // user" API
455         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
456         c.Assert(err, check.IsNil)
457         c.Check(user1.IsActive, check.Equals, false)
458 }
459
460 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
461         conn1 := s.testClusters["z1111"].Conn()
462         conn3 := s.testClusters["z3333"].Conn()
463         rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients()
464
465         // Create user on LoginCluster z1111
466         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
467
468         // Make a new root token (because rootClients() uses SystemRootToken)
469         var outAuth arvados.APIClientAuthorization
470         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
471         c.Check(err, check.IsNil)
472
473         // Make a v2 root token to communicate with z3333
474         rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2())
475
476         // Create VM on z3333
477         var outVM arvados.VirtualMachine
478         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
479                 map[string]interface{}{"virtual_machine": map[string]interface{}{
480                         "hostname": "example",
481                 },
482                 })
483         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
484         c.Check(err, check.IsNil)
485
486         // Make sure z3333 user list is up to date
487         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
488         c.Check(err, check.IsNil)
489
490         // Try to set up user on z3333 with the VM
491         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
492         c.Check(err, check.IsNil)
493
494         var outLinks arvados.LinkList
495         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
496                 arvados.ListOptions{
497                         Limit: 1000,
498                         Filters: []arvados.Filter{
499                                 {
500                                         Attr:     "tail_uuid",
501                                         Operator: "=",
502                                         Operand:  user.UUID,
503                                 },
504                                 {
505                                         Attr:     "head_uuid",
506                                         Operator: "=",
507                                         Operand:  outVM.UUID,
508                                 },
509                                 {
510                                         Attr:     "name",
511                                         Operator: "=",
512                                         Operand:  "can_login",
513                                 },
514                                 {
515                                         Attr:     "link_class",
516                                         Operator: "=",
517                                         Operand:  "permission",
518                                 }}})
519         c.Check(err, check.IsNil)
520
521         c.Check(len(outLinks.Items), check.Equals, 1)
522 }
523
524 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
525         conn1 := s.testClusters["z1111"].Conn()
526         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
527         s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
528
529         accesstoken := s.oidcprovider.ValidAccessToken()
530
531         for _, clusterID := range []string{"z1111", "z2222"} {
532                 c.Logf("trying clusterid %s", clusterID)
533
534                 conn := s.testClusters[clusterID].Conn()
535                 ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
536
537                 var coll arvados.Collection
538
539                 // Write some file data and create a collection
540                 {
541                         fs, err := coll.FileSystem(ac, kc)
542                         c.Assert(err, check.IsNil)
543                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
544                         c.Assert(err, check.IsNil)
545                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
546                         c.Assert(err, check.IsNil)
547                         err = f.Close()
548                         c.Assert(err, check.IsNil)
549                         mtxt, err := fs.MarshalManifest(".")
550                         c.Assert(err, check.IsNil)
551                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
552                                 "manifest_text": mtxt,
553                         }})
554                         c.Assert(err, check.IsNil)
555                 }
556
557                 // Read the collection & file data
558                 {
559                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
560                         c.Assert(err, check.IsNil)
561                         c.Check(user.FullName, check.Equals, "Example User")
562                         coll, err = conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
563                         c.Assert(err, check.IsNil)
564                         c.Check(coll.ManifestText, check.Not(check.Equals), "")
565                         fs, err := coll.FileSystem(ac, kc)
566                         c.Assert(err, check.IsNil)
567                         f, err := fs.Open("test.txt")
568                         c.Assert(err, check.IsNil)
569                         buf, err := ioutil.ReadAll(f)
570                         c.Assert(err, check.IsNil)
571                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
572                 }
573         }
574 }