lib/controller/integration_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "database/sql"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "math"
  16         "net"
  17         "net/http"
  18         "os"
  19         "os/exec"
  20         "path/filepath"
  21         "strconv"
  22         "strings"
  23         "sync"
  24
  25         "git.arvados.org/arvados.git/lib/boot"
  26         "git.arvados.org/arvados.git/lib/config"
  27         "git.arvados.org/arvados.git/sdk/go/arvados"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/httpserver"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 var _ = check.Suite(&IntegrationSuite{})
  35
  36 type IntegrationSuite struct {
  37         testClusters map[string]*boot.TestCluster
  38         oidcprovider *arvadostest.OIDCProvider
  39 }
  40
  41 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
  42         cwd, _ := os.Getwd()
  43
  44         s.oidcprovider = arvadostest.NewOIDCProvider(c)
  45         s.oidcprovider.AuthEmail = "user@example.com"
  46         s.oidcprovider.AuthEmailVerified = true
  47         s.oidcprovider.AuthName = "Example User"
  48         s.oidcprovider.ValidClientID = "clientid"
  49         s.oidcprovider.ValidClientSecret = "clientsecret"
  50
  51         s.testClusters = map[string]*boot.TestCluster{
  52                 "z1111": nil,
  53                 "z2222": nil,
  54                 "z3333": nil,
  55         }
  56         hostport := map[string]string{}
  57         for id := range s.testClusters {
  58                 hostport[id] = func() string {
  59                         // TODO: Instead of expecting random ports on
  60                         // 127.0.0.11, 22, 33 to be race-safe, try
  61                         // different 127.x.y.z until finding one that
  62                         // isn't in use.
  63                         ln, err := net.Listen("tcp", ":0")
  64                         c.Assert(err, check.IsNil)
  65                         ln.Close()
  66                         _, port, err := net.SplitHostPort(ln.Addr().String())
  67                         c.Assert(err, check.IsNil)
  68                         return "127.0.0." + id[3:] + ":" + port
  69                 }()
  70         }
  71         for id := range s.testClusters {
  72                 yaml := `Clusters:
  73   ` + id + `:
  74     Services:
  75       Controller:
  76         ExternalURL: https://` + hostport[id] + `
  77     TLS:
  78       Insecure: true
  79     SystemLogs:
  80       Format: text
  81     RemoteClusters:
  82       z1111:
  83         Host: ` + hostport["z1111"] + `
  84         Scheme: https
  85         Insecure: true
  86         Proxy: true
  87         ActivateUsers: true
  88 `
  89                 if id != "z2222" {
  90                         yaml += `      z2222:
  91         Host: ` + hostport["z2222"] + `
  92         Scheme: https
  93         Insecure: true
  94         Proxy: true
  95         ActivateUsers: true
  96 `
  97                 }
  98                 if id != "z3333" {
  99                         yaml += `      z3333:
 100         Host: ` + hostport["z3333"] + `
 101         Scheme: https
 102         Insecure: true
 103         Proxy: true
 104         ActivateUsers: true
 105 `
 106                 }
 107                 if id == "z1111" {
 108                         yaml += `
 109     Login:
 110       LoginCluster: z1111
 111       OpenIDConnect:
 112         Enable: true
 113         Issuer: ` + s.oidcprovider.Issuer.URL + `
 114         ClientID: ` + s.oidcprovider.ValidClientID + `
 115         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
 116         EmailClaim: email
 117         EmailVerifiedClaim: email_verified
 118         AcceptAccessToken: true
 119         AcceptAccessTokenScope: ""
 120 `
 121                 } else {
 122                         yaml += `
 123     Login:
 124       LoginCluster: z1111
 125 `
 126                 }
 127
 128                 loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
 129                 loader.Path = "-"
 130                 loader.SkipLegacy = true
 131                 loader.SkipAPICalls = true
 132                 cfg, err := loader.Load()
 133                 c.Assert(err, check.IsNil)
 134                 tc := boot.NewTestCluster(
 135                         filepath.Join(cwd, "..", ".."),
 136                         id, cfg, "127.0.0."+id[3:], c.Log)
 137                 tc.Super.NoWorkbench1 = true
 138                 tc.Super.NoWorkbench2 = true
 139                 tc.Start()
 140                 s.testClusters[id] = tc
 141         }
 142         for _, tc := range s.testClusters {
 143                 ok := tc.WaitReady()
 144                 c.Assert(ok, check.Equals, true)
 145         }
 146 }
 147
 148 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 149         for _, c := range s.testClusters {
 150                 c.Super.Stop()
 151         }
 152 }
 153
 154 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
 155         conn := s.testClusters["z1111"].Conn()
 156         rootctx, _, _ := s.testClusters["z1111"].RootClients()
 157         userctx, _, kc, _ := s.testClusters["z1111"].UserClients(rootctx, c, conn, s.oidcprovider.AuthEmail, true)
 158         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
 159         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
 160         c.Assert(err, check.IsNil)
 161         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
 162 }
 163
 164 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
 165         conn1 := s.testClusters["z1111"].Conn()
 166         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 167         conn3 := s.testClusters["z3333"].Conn()
 168         userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 169
 170         // Create the collection to find its PDH (but don't save it
 171         // anywhere yet)
 172         var coll1 arvados.Collection
 173         fs1, err := coll1.FileSystem(ac1, kc1)
 174         c.Assert(err, check.IsNil)
 175         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 176         c.Assert(err, check.IsNil)
 177         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
 178         c.Assert(err, check.IsNil)
 179         err = f.Close()
 180         c.Assert(err, check.IsNil)
 181         mtxt, err := fs1.MarshalManifest(".")
 182         c.Assert(err, check.IsNil)
 183         pdh := arvados.PortableDataHash(mtxt)
 184
 185         // Looking up the PDH before saving returns 404 if cycle
 186         // detection is working.
 187         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
 188         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
 189
 190         // Save the collection on cluster z1111.
 191         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 192                 "manifest_text": mtxt,
 193         }})
 194         c.Assert(err, check.IsNil)
 195
 196         // Retrieve the collection from cluster z3333.
 197         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
 198         c.Check(err, check.IsNil)
 199         c.Check(coll.PortableDataHash, check.Equals, pdh)
 200 }
 201
 202 // Tests bug #18004
 203 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
 204         conn1 := s.testClusters["z1111"].Conn()
 205         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 206         rootctx2, _, _ := s.testClusters["z2222"].RootClients()
 207         conn2 := s.testClusters["z2222"].Conn()
 208         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user2@example.com", true)
 209
 210         var wg1, wg2 sync.WaitGroup
 211         creqs := 100
 212
 213         // Make concurrent requests to z2222 with a local token to make sure more
 214         // than one worker is listening.
 215         wg1.Add(1)
 216         for i := 0; i < creqs; i++ {
 217                 wg2.Add(1)
 218                 go func() {
 219                         defer wg2.Done()
 220                         wg1.Wait()
 221                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
 222                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
 223                 }()
 224         }
 225         wg1.Done()
 226         wg2.Wait()
 227
 228         // Real test pass -- use a new remote token than the one used in the warm-up
 229         // phase.
 230         wg1.Add(1)
 231         for i := 0; i < creqs; i++ {
 232                 wg2.Add(1)
 233                 go func() {
 234                         defer wg2.Done()
 235                         wg1.Wait()
 236                         // Retrieve the remote collection from cluster z2222.
 237                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
 238                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
 239                 }()
 240         }
 241         wg1.Done()
 242         wg2.Wait()
 243 }
 244
 245 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
 246         if _, err := exec.LookPath("s3cmd"); err != nil {
 247                 c.Skip("s3cmd not in PATH")
 248                 return
 249         }
 250
 251         testText := "IntegrationSuite.TestS3WithFederatedToken"
 252
 253         conn1 := s.testClusters["z1111"].Conn()
 254         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 255         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 256         conn3 := s.testClusters["z3333"].Conn()
 257
 258         createColl := func(clusterID string) arvados.Collection {
 259                 _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken)
 260                 var coll arvados.Collection
 261                 fs, err := coll.FileSystem(ac, kc)
 262                 c.Assert(err, check.IsNil)
 263                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 264                 c.Assert(err, check.IsNil)
 265                 _, err = io.WriteString(f, testText)
 266                 c.Assert(err, check.IsNil)
 267                 err = f.Close()
 268                 c.Assert(err, check.IsNil)
 269                 mtxt, err := fs.MarshalManifest(".")
 270                 c.Assert(err, check.IsNil)
 271                 coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 272                         "manifest_text": mtxt,
 273                 }})
 274                 c.Assert(err, check.IsNil)
 275                 return coll
 276         }
 277
 278         for _, trial := range []struct {
 279                 clusterID string // create the collection on this cluster (then use z3333 to access it)
 280                 token     string
 281         }{
 282                 // Try the hardest test first: z3333 hasn't seen
 283                 // z1111's token yet, and we're just passing the
 284                 // opaque secret part, so z3333 has to guess that it
 285                 // belongs to z1111.
 286                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
 287                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
 288                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
 289                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
 290         } {
 291                 c.Logf("================ %v", trial)
 292                 coll := createColl(trial.clusterID)
 293
 294                 cfgjson, err := conn3.ConfigGet(userctx1)
 295                 c.Assert(err, check.IsNil)
 296                 var cluster arvados.Cluster
 297                 err = json.Unmarshal(cfgjson, &cluster)
 298                 c.Assert(err, check.IsNil)
 299
 300                 c.Logf("TokenV2 is %s", ac1.AuthToken)
 301                 host := cluster.Services.WebDAV.ExternalURL.Host
 302                 s3args := []string{
 303                         "--ssl", "--no-check-certificate",
 304                         "--host=" + host, "--host-bucket=" + host,
 305                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
 306                 }
 307                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
 308                 c.Check(err, check.IsNil)
 309                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
 310
 311                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
 312                 // Command fails because we don't return Etag header.
 313                 flen := strconv.Itoa(len(testText))
 314                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
 315         }
 316 }
 317
 318 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
 319         conn1 := s.testClusters["z1111"].Conn()
 320         conn3 := s.testClusters["z3333"].Conn()
 321         rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients()
 322         anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
 323
 324         // Make sure anonymous token was set
 325         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
 326
 327         // Create the collection to find its PDH (but don't save it
 328         // anywhere yet)
 329         var coll1 arvados.Collection
 330         fs1, err := coll1.FileSystem(rootac1, rootkc1)
 331         c.Assert(err, check.IsNil)
 332         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 333         c.Assert(err, check.IsNil)
 334         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
 335         c.Assert(err, check.IsNil)
 336         err = f.Close()
 337         c.Assert(err, check.IsNil)
 338         mtxt, err := fs1.MarshalManifest(".")
 339         c.Assert(err, check.IsNil)
 340         pdh := arvados.PortableDataHash(mtxt)
 341
 342         // Save the collection on cluster z1111.
 343         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 344                 "manifest_text": mtxt,
 345         }})
 346         c.Assert(err, check.IsNil)
 347
 348         // Share it with the anonymous users group.
 349         var outLink arvados.Link
 350         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
 351                 map[string]interface{}{"link": map[string]interface{}{
 352                         "link_class": "permission",
 353                         "name":       "can_read",
 354                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
 355                         "head_uuid":  coll1.UUID,
 356                 },
 357                 })
 358         c.Check(err, check.IsNil)
 359
 360         // Current user should be z3 anonymous user
 361         outUser, err := anonac3.CurrentUser()
 362         c.Check(err, check.IsNil)
 363         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
 364
 365         // Get the token uuid
 366         var outAuth arvados.APIClientAuthorization
 367         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
 368         c.Check(err, check.IsNil)
 369
 370         // Make a v2 token of the z3 anonymous user, and use it on z1
 371         _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2())
 372         outUser2, err := anonac1.CurrentUser()
 373         c.Check(err, check.IsNil)
 374         // z3 anonymous user will be mapped to the z1 anonymous user
 375         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
 376
 377         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
 378         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
 379         c.Check(err, check.IsNil)
 380         c.Check(coll.PortableDataHash, check.Equals, pdh)
 381 }
 382
 383 // z3333 should forward the locally-issued anonymous user token to its login
 384 // cluster z1111. That is no problem because the login cluster controller will
 385 // map any anonymous user token to its local anonymous user.
 386 //
 387 // This needs to work because wb1 has a tendency to slap the local anonymous
 388 // user token on every request as a reader_token, which gets folded into the
 389 // request token list controller.
 390 //
 391 // Use a z1111 user token and the anonymous token from z3333 passed in as a
 392 // reader_token to do a request on z3333, asking for the z1111 anonymous user
 393 // object. The request will be forwarded to the z1111 cluster. The presence of
 394 // the z3333 anonymous user token should not prohibit the request from being
 395 // forwarded.
 396 func (s *IntegrationSuite) TestForwardAnonymousTokenToLoginCluster(c *check.C) {
 397         conn1 := s.testClusters["z1111"].Conn()
 398         s.testClusters["z3333"].Conn()
 399
 400         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 401         _, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
 402
 403         // Make a user connection to z3333 (using a z1111 user, because that's the login cluster)
 404         _, userac1, _, _ := s.testClusters["z3333"].UserClients(rootctx1, c, conn1, "user@example.com", true)
 405
 406         // Get the anonymous user token for z3333
 407         var anon3Auth arvados.APIClientAuthorization
 408         err := anonac3.RequestAndDecode(&anon3Auth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
 409         c.Check(err, check.IsNil)
 410
 411         var userList arvados.UserList
 412         where := make(map[string]string)
 413         where["uuid"] = "z1111-tpzed-anonymouspublic"
 414         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
 415                 map[string]interface{}{
 416                         "reader_tokens": []string{anon3Auth.TokenV2()},
 417                         "where":         where,
 418                 },
 419         )
 420         // The local z3333 anonymous token must be allowed to be forwarded to the login cluster
 421         c.Check(err, check.IsNil)
 422
 423         userac1.AuthToken = "v2/z1111-gj3su-asdfasdfasdfasd/this-token-does-not-validate-so-anonymous-token-will-be-used-instead"
 424         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
 425                 map[string]interface{}{
 426                         "reader_tokens": []string{anon3Auth.TokenV2()},
 427                         "where":         where,
 428                 },
 429         )
 430         c.Check(err, check.IsNil)
 431 }
 432
 433 // Get a token from the login cluster (z1111), use it to submit a
 434 // container request on z2222.
 435 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
 436         conn1 := s.testClusters["z1111"].Conn()
 437         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 438         _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 439
 440         // Use ac2 to get the discovery doc with a blank token, so the
 441         // SDK doesn't magically pass the z1111 token to z2222 before
 442         // we're ready to start our test.
 443         _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("")
 444         var dd map[string]interface{}
 445         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
 446         c.Assert(err, check.IsNil)
 447
 448         var (
 449                 body bytes.Buffer
 450                 req  *http.Request
 451                 resp *http.Response
 452                 u    arvados.User
 453                 cr   arvados.ContainerRequest
 454         )
 455         json.NewEncoder(&body).Encode(map[string]interface{}{
 456                 "container_request": map[string]interface{}{
 457                         "command":         []string{"echo"},
 458                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 459                         "cwd":             "/",
 460                         "output_path":     "/",
 461                 },
 462         })
 463         ac2.AuthToken = ac1.AuthToken
 464
 465         c.Log("...post CR with good (but not yet cached) token")
 466         cr = arvados.ContainerRequest{}
 467         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 468         c.Assert(err, check.IsNil)
 469         req.Header.Set("Content-Type", "application/json")
 470         err = ac2.DoAndDecode(&cr, req)
 471         c.Assert(err, check.IsNil)
 472         c.Logf("err == %#v", err)
 473
 474         c.Log("...get user with good token")
 475         u = arvados.User{}
 476         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
 477         c.Assert(err, check.IsNil)
 478         err = ac2.DoAndDecode(&u, req)
 479         c.Check(err, check.IsNil)
 480         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
 481
 482         c.Log("...post CR with good cached token")
 483         cr = arvados.ContainerRequest{}
 484         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 485         c.Assert(err, check.IsNil)
 486         req.Header.Set("Content-Type", "application/json")
 487         err = ac2.DoAndDecode(&cr, req)
 488         c.Check(err, check.IsNil)
 489         c.Check(cr.UUID, check.Matches, "z2222-.*")
 490
 491         c.Log("...post with good cached token ('OAuth2 ...')")
 492         cr = arvados.ContainerRequest{}
 493         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 494         c.Assert(err, check.IsNil)
 495         req.Header.Set("Content-Type", "application/json")
 496         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
 497         resp, err = arvados.InsecureHTTPClient.Do(req)
 498         c.Assert(err, check.IsNil)
 499         err = json.NewDecoder(resp.Body).Decode(&cr)
 500         c.Check(err, check.IsNil)
 501         c.Check(cr.UUID, check.Matches, "z2222-.*")
 502 }
 503
 504 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
 505         conn1 := s.testClusters["z1111"].Conn()
 506         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 507         _, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
 508
 509         tests := []struct {
 510                 name         string
 511                 token        string
 512                 expectedCode int
 513         }{
 514                 {"Good token", ac1.AuthToken, http.StatusOK},
 515                 {"Bogus token", "abcdef", http.StatusUnauthorized},
 516                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
 517                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
 518         }
 519
 520         body, _ := json.Marshal(map[string]interface{}{
 521                 "container_request": map[string]interface{}{
 522                         "command":         []string{"echo"},
 523                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 524                         "cwd":             "/",
 525                         "output_path":     "/",
 526                 },
 527         })
 528
 529         for _, tt := range tests {
 530                 c.Log(c.TestName() + " " + tt.name)
 531                 ac1.AuthToken = tt.token
 532                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
 533                 c.Assert(err, check.IsNil)
 534                 req.Header.Set("Content-Type", "application/json")
 535                 resp, err := ac1.Do(req)
 536                 c.Assert(err, check.IsNil)
 537                 c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
 538         }
 539 }
 540
 541 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
 542         conn1 := s.testClusters["z1111"].Conn()
 543         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 544         userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
 545
 546         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
 547         c.Check(err, check.IsNil)
 548         specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
 549         c.Check(err, check.IsNil)
 550
 551         tests := []struct {
 552                 path            string
 553                 reqIdProvided   bool
 554                 notFoundRequest bool
 555         }{
 556                 {"/arvados/v1/collections", false, false},
 557                 {"/arvados/v1/collections", true, false},
 558                 {"/arvados/v1/nonexistant", false, true},
 559                 {"/arvados/v1/nonexistant", true, true},
 560                 {"/arvados/v1/collections/" + coll.UUID, false, false},
 561                 {"/arvados/v1/collections/" + coll.UUID, true, false},
 562                 {"/arvados/v1/specimens/" + specimen.UUID, false, false},
 563                 {"/arvados/v1/specimens/" + specimen.UUID, true, false},
 564                 // new code path (lib/controller/router etc) - single-cluster request
 565                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
 566                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
 567                 // new code path (lib/controller/router etc) - federated request
 568                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", false, true},
 569                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", true, true},
 570                 // old code path (proxyRailsAPI) - single-cluster request
 571                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
 572                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
 573                 // old code path (setupProxyRemoteCluster) - federated request
 574                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", false, true},
 575                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", true, true},
 576         }
 577
 578         for _, tt := range tests {
 579                 c.Log(c.TestName() + " " + tt.path)
 580                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
 581                 c.Assert(err, check.IsNil)
 582                 customReqId := "abcdeG"
 583                 if !tt.reqIdProvided {
 584                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
 585                 } else {
 586                         req.Header.Set("X-Request-Id", customReqId)
 587                 }
 588                 resp, err := ac1.Do(req)
 589                 c.Assert(err, check.IsNil)
 590                 if tt.notFoundRequest {
 591                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 592                 } else {
 593                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 594                 }
 595                 respHdr := resp.Header.Get("X-Request-Id")
 596                 if tt.reqIdProvided {
 597                         c.Check(respHdr, check.Equals, customReqId)
 598                 } else {
 599                         c.Check(respHdr, check.Matches, `req-[0-9a-zA-Z]{20}`)
 600                 }
 601                 if tt.notFoundRequest {
 602                         var jresp httpserver.ErrorResponse
 603                         err := json.NewDecoder(resp.Body).Decode(&jresp)
 604                         c.Check(err, check.IsNil)
 605                         c.Assert(jresp.Errors, check.HasLen, 1)
 606                         c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`)
 607                 }
 608         }
 609 }
 610
 611 // We test the direct access to the database
 612 // normally an integration test would not have a database access, but in this case we need
 613 // to test tokens that are secret, so there is no API response that will give them back
 614 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
 615         ctx := context.Background()
 616         db, err := sql.Open("postgres", s.testClusters[clusterID].Super.Cluster().PostgreSQL.Connection.String())
 617         c.Assert(err, check.IsNil)
 618
 619         conn, err := db.Conn(ctx)
 620         c.Assert(err, check.IsNil)
 621
 622         rows, err := conn.ExecContext(ctx, `SELECT 1`)
 623         c.Assert(err, check.IsNil)
 624         n, err := rows.RowsAffected()
 625         c.Assert(err, check.IsNil)
 626         c.Assert(n, check.Equals, int64(1))
 627         return db, conn
 628 }
 629
 630 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
 631 // and check the expected results accessing directly to the database if needed.
 632 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
 633         db, dbconn := s.dbConn(c, "z1111")
 634         defer db.Close()
 635         defer dbconn.Close()
 636         conn1 := s.testClusters["z1111"].Conn()
 637         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 638         userctx1, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
 639
 640         tests := []struct {
 641                 name                 string
 642                 token                string
 643                 expectAToGetAValidCR bool
 644                 expectedToken        *string
 645         }{
 646                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
 647                 {"Bogus token", "abcdef", false, nil},
 648                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
 649                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
 650         }
 651
 652         for _, tt := range tests {
 653                 c.Log(c.TestName() + " " + tt.name)
 654
 655                 rq := map[string]interface{}{
 656                         "command":         []string{"echo"},
 657                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 658                         "cwd":             "/",
 659                         "output_path":     "/",
 660                         "runtime_token":   tt.token,
 661                 }
 662                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
 663                 if tt.expectAToGetAValidCR {
 664                         c.Check(err, check.IsNil)
 665                         c.Check(cr, check.NotNil)
 666                         c.Check(cr.UUID, check.Not(check.Equals), "")
 667                 }
 668
 669                 if tt.expectedToken == nil {
 670                         continue
 671                 }
 672
 673                 c.Logf("cr.UUID: %s", cr.UUID)
 674                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
 675                 c.Check(row, check.NotNil)
 676                 var token sql.NullString
 677                 row.Scan(&token)
 678                 if c.Check(token.Valid, check.Equals, true) {
 679                         c.Check(token.String, check.Equals, *tt.expectedToken)
 680                 }
 681         }
 682 }
 683
 684 // TestIntermediateCluster will send a container request to
 685 // one cluster with another cluster as the destination
 686 // and check the tokens are being handled properly
 687 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
 688         conn1 := s.testClusters["z1111"].Conn()
 689         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 690         uctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
 691
 692         tests := []struct {
 693                 name                 string
 694                 token                string
 695                 expectedRuntimeToken string
 696                 expectedUUIDprefix   string
 697         }{
 698                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
 699         }
 700
 701         for _, tt := range tests {
 702                 c.Log(c.TestName() + " " + tt.name)
 703                 rq := map[string]interface{}{
 704                         "command":         []string{"echo"},
 705                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 706                         "cwd":             "/",
 707                         "output_path":     "/",
 708                         "runtime_token":   tt.token,
 709                 }
 710                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
 711
 712                 c.Check(err, check.IsNil)
 713                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
 714                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
 715         }
 716 }
 717
 718 // Test for #17785
 719 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) {
 720         rootctx1, rootclnt1, _ := s.testClusters["z1111"].RootClients()
 721         conn1 := s.testClusters["z1111"].Conn()
 722
 723         // Make sure LoginCluster is properly configured
 724         for _, cls := range []string{"z1111", "z3333"} {
 725                 c.Check(
 726                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
 727                         check.Equals, "z1111",
 728                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 729         }
 730         // Get user's UUID & attempt to create a token for it on the remote cluster
 731         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1,
 732                 "user@example.com", true)
 733         _, rootclnt3, _ := s.testClusters["z3333"].ClientsWithToken(rootclnt1.AuthToken)
 734         var resp arvados.APIClientAuthorization
 735         err := rootclnt3.RequestAndDecode(
 736                 &resp, "POST", "arvados/v1/api_client_authorizations", nil,
 737                 map[string]interface{}{
 738                         "api_client_authorization": map[string]string{
 739                                 "owner_uuid": user.UUID,
 740                         },
 741                 },
 742         )
 743         c.Assert(err, check.IsNil)
 744         c.Assert(resp.APIClientID, check.Not(check.Equals), 0)
 745         newTok := resp.TokenV2()
 746         c.Assert(newTok, check.Not(check.Equals), "")
 747
 748         // Confirm the token is from z1111
 749         c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true)
 750
 751         // Confirm the token works and is from the correct user
 752         _, rootclnt3bis, _ := s.testClusters["z3333"].ClientsWithToken(newTok)
 753         var curUser arvados.User
 754         err = rootclnt3bis.RequestAndDecode(
 755                 &curUser, "GET", "arvados/v1/users/current", nil, nil,
 756         )
 757         c.Assert(err, check.IsNil)
 758         c.Assert(curUser.UUID, check.Equals, user.UUID)
 759
 760         // Request the ApiClientAuthorization list using the new token
 761         _, userClient, _ := s.testClusters["z3333"].ClientsWithToken(newTok)
 762         var acaLst arvados.APIClientAuthorizationList
 763         err = userClient.RequestAndDecode(
 764                 &acaLst, "GET", "arvados/v1/api_client_authorizations", nil, nil,
 765         )
 766         c.Assert(err, check.IsNil)
 767 }
 768
 769 // Test for bug #18076
 770 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
 771         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 772         _, rootclnt3, _ := s.testClusters["z3333"].RootClients()
 773         conn1 := s.testClusters["z1111"].Conn()
 774         conn3 := s.testClusters["z3333"].Conn()
 775
 776         // Make sure LoginCluster is properly configured
 777         for _, cls := range []string{"z1111", "z3333"} {
 778                 c.Check(
 779                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
 780                         check.Equals, "z1111",
 781                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 782         }
 783
 784         for testCaseNr, testCase := range []struct {
 785                 name           string
 786                 withRepository bool
 787         }{
 788                 {"User without local repository", false},
 789                 {"User with local repository", true},
 790         } {
 791                 c.Log(c.TestName() + " " + testCase.name)
 792                 // Create some users, request them on the federated cluster so they're cached.
 793                 var users []arvados.User
 794                 for userNr := 0; userNr < 2; userNr++ {
 795                         _, _, _, user := s.testClusters["z1111"].UserClients(
 796                                 rootctx1,
 797                                 c,
 798                                 conn1,
 799                                 fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
 800                                 true)
 801                         c.Assert(user.Username, check.Not(check.Equals), "")
 802                         users = append(users, user)
 803
 804                         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 805                         c.Assert(err, check.Equals, nil)
 806                         userFound := false
 807                         for _, fedUser := range lst.Items {
 808                                 if fedUser.UUID == user.UUID {
 809                                         c.Assert(fedUser.Username, check.Equals, user.Username)
 810                                         userFound = true
 811                                         break
 812                                 }
 813                         }
 814                         c.Assert(userFound, check.Equals, true)
 815
 816                         if testCase.withRepository {
 817                                 var repo interface{}
 818                                 err = rootclnt3.RequestAndDecode(
 819                                         &repo, "POST", "arvados/v1/repositories", nil,
 820                                         map[string]interface{}{
 821                                                 "repository": map[string]string{
 822                                                         "name":       fmt.Sprintf("%s/test", user.Username),
 823                                                         "owner_uuid": user.UUID,
 824                                                 },
 825                                         },
 826                                 )
 827                                 c.Assert(err, check.IsNil)
 828                         }
 829                 }
 830
 831                 // Swap the usernames
 832                 _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 833                         UUID: users[0].UUID,
 834                         Attrs: map[string]interface{}{
 835                                 "username": "",
 836                         },
 837                 })
 838                 c.Assert(err, check.Equals, nil)
 839                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 840                         UUID: users[1].UUID,
 841                         Attrs: map[string]interface{}{
 842                                 "username": users[0].Username,
 843                         },
 844                 })
 845                 c.Assert(err, check.Equals, nil)
 846                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 847                         UUID: users[0].UUID,
 848                         Attrs: map[string]interface{}{
 849                                 "username": users[1].Username,
 850                         },
 851                 })
 852                 c.Assert(err, check.Equals, nil)
 853
 854                 // Re-request the list on the federated cluster & check for updates
 855                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 856                 c.Assert(err, check.Equals, nil)
 857                 var user0Found, user1Found bool
 858                 for _, user := range lst.Items {
 859                         if user.UUID == users[0].UUID {
 860                                 user0Found = true
 861                                 c.Assert(user.Username, check.Equals, users[1].Username)
 862                         } else if user.UUID == users[1].UUID {
 863                                 user1Found = true
 864                                 c.Assert(user.Username, check.Equals, users[0].Username)
 865                         }
 866                 }
 867                 c.Assert(user0Found, check.Equals, true)
 868                 c.Assert(user1Found, check.Equals, true)
 869         }
 870 }
 871
 872 // Test for bug #16263
 873 func (s *IntegrationSuite) TestListUsers(c *check.C) {
 874         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
 875         conn1 := s.testClusters["z1111"].Conn()
 876         conn3 := s.testClusters["z3333"].Conn()
 877         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 878
 879         // Make sure LoginCluster is properly configured
 880         for cls := range s.testClusters {
 881                 c.Check(
 882                         s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster,
 883                         check.Equals, "z1111",
 884                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 885         }
 886         // Make sure z1111 has users with NULL usernames
 887         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
 888                 Limit: math.MaxInt64, // check that large limit works (see #16263)
 889         })
 890         nullUsername := false
 891         c.Assert(err, check.IsNil)
 892         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
 893         for _, user := range lst.Items {
 894                 if user.Username == "" {
 895                         nullUsername = true
 896                         break
 897                 }
 898         }
 899         c.Assert(nullUsername, check.Equals, true)
 900
 901         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
 902         c.Assert(err, check.IsNil)
 903         c.Check(user1.IsActive, check.Equals, true)
 904
 905         // Ask for the user list on z3333 using z1111's system root token
 906         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 907         c.Assert(err, check.IsNil)
 908         found := false
 909         for _, user := range lst.Items {
 910                 if user.UUID == user1.UUID {
 911                         c.Check(user.IsActive, check.Equals, true)
 912                         found = true
 913                         break
 914                 }
 915         }
 916         c.Check(found, check.Equals, true)
 917
 918         // Deactivate user acct on z1111
 919         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
 920         c.Assert(err, check.IsNil)
 921
 922         // Get user list from z3333, check the returned z1111 user is
 923         // deactivated
 924         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 925         c.Assert(err, check.IsNil)
 926         found = false
 927         for _, user := range lst.Items {
 928                 if user.UUID == user1.UUID {
 929                         c.Check(user.IsActive, check.Equals, false)
 930                         found = true
 931                         break
 932                 }
 933         }
 934         c.Check(found, check.Equals, true)
 935
 936         // Deactivated user no longer has working token
 937         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
 938         c.Assert(err, check.ErrorMatches, `.*401 Unauthorized.*`)
 939 }
 940
 941 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
 942         conn1 := s.testClusters["z1111"].Conn()
 943         conn3 := s.testClusters["z3333"].Conn()
 944         rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients()
 945
 946         // Create user on LoginCluster z1111
 947         _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 948
 949         // Make a new root token (because rootClients() uses SystemRootToken)
 950         var outAuth arvados.APIClientAuthorization
 951         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
 952         c.Check(err, check.IsNil)
 953
 954         // Make a v2 root token to communicate with z3333
 955         rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2())
 956
 957         // Create VM on z3333
 958         var outVM arvados.VirtualMachine
 959         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
 960                 map[string]interface{}{"virtual_machine": map[string]interface{}{
 961                         "hostname": "example",
 962                 },
 963                 })
 964         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
 965         c.Check(err, check.IsNil)
 966
 967         // Make sure z3333 user list is up to date
 968         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
 969         c.Check(err, check.IsNil)
 970
 971         // Try to set up user on z3333 with the VM
 972         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
 973         c.Check(err, check.IsNil)
 974
 975         var outLinks arvados.LinkList
 976         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
 977                 arvados.ListOptions{
 978                         Limit: 1000,
 979                         Filters: []arvados.Filter{
 980                                 {
 981                                         Attr:     "tail_uuid",
 982                                         Operator: "=",
 983                                         Operand:  user.UUID,
 984                                 },
 985                                 {
 986                                         Attr:     "head_uuid",
 987                                         Operator: "=",
 988                                         Operand:  outVM.UUID,
 989                                 },
 990                                 {
 991                                         Attr:     "name",
 992                                         Operator: "=",
 993                                         Operand:  "can_login",
 994                                 },
 995                                 {
 996                                         Attr:     "link_class",
 997                                         Operator: "=",
 998                                         Operand:  "permission",
 999                                 }}})
1000         c.Check(err, check.IsNil)
1001
1002         c.Check(len(outLinks.Items), check.Equals, 1)
1003 }
1004
1005 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
1006         conn1 := s.testClusters["z1111"].Conn()
1007         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
1008         s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1009
1010         accesstoken := s.oidcprovider.ValidAccessToken()
1011
1012         for _, clusterID := range []string{"z1111", "z2222"} {
1013
1014                 var coll arvados.Collection
1015
1016                 // Write some file data and create a collection
1017                 {
1018                         c.Logf("save collection to %s", clusterID)
1019
1020                         conn := s.testClusters[clusterID].Conn()
1021                         ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
1022
1023                         fs, err := coll.FileSystem(ac, kc)
1024                         c.Assert(err, check.IsNil)
1025                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
1026                         c.Assert(err, check.IsNil)
1027                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
1028                         c.Assert(err, check.IsNil)
1029                         err = f.Close()
1030                         c.Assert(err, check.IsNil)
1031                         mtxt, err := fs.MarshalManifest(".")
1032                         c.Assert(err, check.IsNil)
1033                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
1034                                 "manifest_text": mtxt,
1035                         }})
1036                         c.Assert(err, check.IsNil)
1037                 }
1038
1039                 // Read the collection & file data -- both from the
1040                 // cluster where it was created, and from the other
1041                 // cluster.
1042                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
1043                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
1044
1045                         conn := s.testClusters[readClusterID].Conn()
1046                         ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken)
1047
1048                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
1049                         c.Assert(err, check.IsNil)
1050                         c.Check(user.FullName, check.Equals, "Example User")
1051                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
1052                         c.Assert(err, check.IsNil)
1053                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
1054                         fs, err := readcoll.FileSystem(ac, kc)
1055                         c.Assert(err, check.IsNil)
1056                         f, err := fs.Open("test.txt")
1057                         c.Assert(err, check.IsNil)
1058                         buf, err := ioutil.ReadAll(f)
1059                         c.Assert(err, check.IsNil)
1060                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
1061                 }
1062         }
1063 }
1064
1065 // z3333 should not forward a locally-issued container runtime token,
1066 // associated with a z1111 user, to its login cluster z1111. z1111
1067 // would only call back to z3333 and then reject the response because
1068 // the user ID does not match the token prefix. See
1069 // dev.arvados.org/issues/18346
1070 func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) {
1071         db3, db3conn := s.dbConn(c, "z3333")
1072         defer db3.Close()
1073         defer db3conn.Close()
1074         rootctx1, _, _ := s.testClusters["z1111"].RootClients()
1075         rootctx3, _, _ := s.testClusters["z3333"].RootClients()
1076         conn1 := s.testClusters["z1111"].Conn()
1077         conn3 := s.testClusters["z3333"].Conn()
1078         userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true)
1079
1080         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
1081         c.Assert(err, check.IsNil)
1082         c.Logf("user1 %+v", user1)
1083
1084         imageColl, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1085                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n",
1086         }})
1087         c.Assert(err, check.IsNil)
1088         c.Logf("imageColl %+v", imageColl)
1089
1090         cr, err := conn3.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1091                 "state":           "Committed",
1092                 "command":         []string{"echo"},
1093                 "container_image": imageColl.PortableDataHash,
1094                 "cwd":             "/",
1095                 "output_path":     "/",
1096                 "priority":        1,
1097                 "runtime_constraints": arvados.RuntimeConstraints{
1098                         VCPUs: 1,
1099                         RAM:   1000000000,
1100                 },
1101         }})
1102         c.Assert(err, check.IsNil)
1103         c.Logf("container request %+v", cr)
1104         ctr, err := conn3.ContainerLock(rootctx3, arvados.GetOptions{UUID: cr.ContainerUUID})
1105         c.Assert(err, check.IsNil)
1106         c.Logf("container %+v", ctr)
1107
1108         // We could use conn3.ContainerAuth() here, but that API
1109         // hasn't been added to sdk/go/arvados/api.go yet.
1110         row := db3conn.QueryRowContext(context.Background(), `SELECT api_token from api_client_authorizations where uuid=$1`, ctr.AuthUUID)
1111         c.Check(row, check.NotNil)
1112         var val sql.NullString
1113         row.Scan(&val)
1114         c.Assert(val.Valid, check.Equals, true)
1115         runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String
1116         ctrctx, _, _ := s.testClusters["z3333"].ClientsWithToken(runtimeToken)
1117         c.Logf("container runtime token %+v", runtimeToken)
1118
1119         _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID})
1120         c.Assert(err, check.NotNil)
1121         c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`)
1122         c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`)
1123 }