From cc45e3eb3609af1fd3b186e3aa6e806bcf1e92ae Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Tue, 5 Jan 2021 17:40:27 -0300 Subject: [PATCH] 16981: Extracts test federation building code to the arvadostest package. This is going to be reused on other integration test suites. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- lib/controller/integration_test.go | 194 ++++++----------------- sdk/go/arvadostest/federation_servers.go | 155 ++++++++++++++++++ 2 files changed, 200 insertions(+), 149 deletions(-) create mode 100644 sdk/go/arvadostest/federation_servers.go diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go index 0388f21bee..33c68ce6e3 100644 --- a/lib/controller/integration_test.go +++ b/lib/controller/integration_test.go @@ -6,7 +6,6 @@ package controller import ( "bytes" - "context" "encoding/json" "fmt" "io" @@ -14,36 +13,21 @@ import ( "math" "net" "net/http" - "net/url" "os" "os/exec" "path/filepath" "strconv" "strings" - "git.arvados.org/arvados.git/lib/boot" - "git.arvados.org/arvados.git/lib/config" - "git.arvados.org/arvados.git/lib/controller/rpc" - "git.arvados.org/arvados.git/lib/service" "git.arvados.org/arvados.git/sdk/go/arvados" - "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" - "git.arvados.org/arvados.git/sdk/go/auth" - "git.arvados.org/arvados.git/sdk/go/ctxlog" - "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) var _ = check.Suite(&IntegrationSuite{}) -type testCluster struct { - super boot.Supervisor - config arvados.Config - controllerURL *url.URL -} - type IntegrationSuite struct { - testClusters map[string]*testCluster + testClusters map[string]*arvadostest.TestCluster oidcprovider *arvadostest.OIDCProvider } @@ -62,7 +46,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { s.oidcprovider.ValidClientID = "clientid" s.oidcprovider.ValidClientSecret = "clientsecret" - s.testClusters = map[string]*testCluster{ + s.testClusters = map[string]*arvadostest.TestCluster{ "z1111": nil, "z2222": nil, "z3333": nil, @@ -139,118 +123,30 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { ` } - loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c)) - loader.Path = "-" - loader.SkipLegacy = true - loader.SkipAPICalls = true - cfg, err := loader.Load() + tc, err := arvadostest.NewTestCluster( + filepath.Join(cwd, "..", ".."), + id, yaml, "127.0.0."+id[3:], c.Log) c.Assert(err, check.IsNil) - s.testClusters[id] = &testCluster{ - super: boot.Supervisor{ - SourcePath: filepath.Join(cwd, "..", ".."), - ClusterType: "test", - ListenHost: "127.0.0." + id[3:], - ControllerAddr: ":0", - OwnTemporaryDatabase: true, - Stderr: &service.LogPrefixer{Writer: ctxlog.LogWriter(c.Log), Prefix: []byte("[" + id + "] ")}, - }, - config: *cfg, - } - s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config, "-") + s.testClusters[id] = tc + s.testClusters[id].Start() } for _, tc := range s.testClusters { - au, ok := tc.super.WaitReady() + ok := tc.WaitReady() c.Assert(ok, check.Equals, true) - u := url.URL(*au) - tc.controllerURL = &u } } func (s *IntegrationSuite) TearDownSuite(c *check.C) { for _, c := range s.testClusters { - c.super.Stop() + c.Super.Stop() } } -// Get rpc connection struct initialized to communicate with the -// specified cluster. -func (s *IntegrationSuite) conn(clusterID string) *rpc.Conn { - return rpc.NewConn(clusterID, s.testClusters[clusterID].controllerURL, true, rpc.PassthroughTokenProvider) -} - -// Return Context, Arvados.Client and keepclient structs initialized -// to connect to the specified cluster (by clusterID) using with the supplied -// Arvados token. -func (s *IntegrationSuite) clientsWithToken(clusterID string, token string) (context.Context, *arvados.Client, *keepclient.KeepClient) { - cl := s.testClusters[clusterID].config.Clusters[clusterID] - ctx := auth.NewContext(context.Background(), auth.NewCredentials(token)) - ac, err := arvados.NewClientFromConfig(&cl) - if err != nil { - panic(err) - } - ac.AuthToken = token - arv, err := arvadosclient.New(ac) - if err != nil { - panic(err) - } - kc := keepclient.New(arv) - return ctx, ac, kc -} - -// Log in as a user called "example", get the user's API token, -// initialize clients with the API token, set up the user and -// optionally activate the user. Return client structs for -// communicating with the cluster on behalf of the 'example' user. -func (s *IntegrationSuite) userClients(rootctx context.Context, c *check.C, conn *rpc.Conn, clusterID string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient, arvados.User) { - login, err := conn.UserSessionCreate(rootctx, rpc.UserSessionCreateOptions{ - ReturnTo: ",https://example.com", - AuthInfo: rpc.UserSessionAuthInfo{ - Email: "user@example.com", - FirstName: "Example", - LastName: "User", - Username: "example", - }, - }) - c.Assert(err, check.IsNil) - redirURL, err := url.Parse(login.RedirectLocation) - c.Assert(err, check.IsNil) - userToken := redirURL.Query().Get("api_token") - c.Logf("user token: %q", userToken) - ctx, ac, kc := s.clientsWithToken(clusterID, userToken) - user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{}) - c.Assert(err, check.IsNil) - _, err = conn.UserSetup(rootctx, arvados.UserSetupOptions{UUID: user.UUID}) - c.Assert(err, check.IsNil) - if activate { - _, err = conn.UserActivate(rootctx, arvados.UserActivateOptions{UUID: user.UUID}) - c.Assert(err, check.IsNil) - user, err = conn.UserGetCurrent(ctx, arvados.GetOptions{}) - c.Assert(err, check.IsNil) - c.Logf("user UUID: %q", user.UUID) - if !user.IsActive { - c.Fatalf("failed to activate user -- %#v", user) - } - } - return ctx, ac, kc, user -} - -// Return Context, arvados.Client and keepclient structs initialized -// to communicate with the cluster as the system root user. -func (s *IntegrationSuite) rootClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) { - return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].SystemRootToken) -} - -// Return Context, arvados.Client and keepclient structs initialized -// to communicate with the cluster as the anonymous user. -func (s *IntegrationSuite) anonymousClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) { - return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].Users.AnonymousUserToken) -} - func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { - conn1 := s.conn("z1111") - rootctx1, _, _ := s.rootClients("z1111") - conn3 := s.conn("z3333") - userctx1, ac1, kc1, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + conn1 := s.testClusters["z1111"].Conn() + rootctx1, _, _ := s.testClusters["z1111"].RootClients() + conn3 := s.testClusters["z3333"].Conn() + userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Create the collection to find its PDH (but don't save it // anywhere yet) @@ -292,13 +188,13 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { testText := "IntegrationSuite.TestS3WithFederatedToken" - conn1 := s.conn("z1111") - rootctx1, _, _ := s.rootClients("z1111") - userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) - conn3 := s.conn("z3333") + conn1 := s.testClusters["z1111"].Conn() + rootctx1, _, _ := s.testClusters["z1111"].RootClients() + userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + conn3 := s.testClusters["z3333"].Conn() createColl := func(clusterID string) arvados.Collection { - _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken) + _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken) var coll arvados.Collection fs, err := coll.FileSystem(ac, kc) c.Assert(err, check.IsNil) @@ -310,7 +206,7 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { c.Assert(err, check.IsNil) mtxt, err := fs.MarshalManifest(".") c.Assert(err, check.IsNil) - coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ + coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ "manifest_text": mtxt, }}) c.Assert(err, check.IsNil) @@ -358,10 +254,10 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { } func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { - conn1 := s.conn("z1111") - conn3 := s.conn("z3333") - rootctx1, rootac1, rootkc1 := s.rootClients("z1111") - anonctx3, anonac3, _ := s.anonymousClients("z3333") + conn1 := s.testClusters["z1111"].Conn() + conn3 := s.testClusters["z3333"].Conn() + rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients() + anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients() // Make sure anonymous token was set c.Assert(anonac3.AuthToken, check.Not(check.Equals), "") @@ -410,7 +306,7 @@ func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { c.Check(err, check.IsNil) // Make a v2 token of the z3 anonymous user, and use it on z1 - _, anonac1, _ := s.clientsWithToken("z1111", outAuth.TokenV2()) + _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2()) outUser2, err := anonac1.CurrentUser() c.Check(err, check.IsNil) // z3 anonymous user will be mapped to the z1 anonymous user @@ -425,14 +321,14 @@ func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { // Get a token from the login cluster (z1111), use it to submit a // container request on z2222. func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { - conn1 := s.conn("z1111") - rootctx1, _, _ := s.rootClients("z1111") - _, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + conn1 := s.testClusters["z1111"].Conn() + rootctx1, _, _ := s.testClusters["z1111"].RootClients() + _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Use ac2 to get the discovery doc with a blank token, so the // SDK doesn't magically pass the z1111 token to z2222 before // we're ready to start our test. - _, ac2, _ := s.clientsWithToken("z2222", "") + _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("") var dd map[string]interface{} err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil) c.Assert(err, check.IsNil) @@ -495,15 +391,15 @@ func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { // Test for bug #16263 func (s *IntegrationSuite) TestListUsers(c *check.C) { - rootctx1, _, _ := s.rootClients("z1111") - conn1 := s.conn("z1111") - conn3 := s.conn("z3333") - userctx1, _, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + rootctx1, _, _ := s.testClusters["z1111"].RootClients() + conn1 := s.testClusters["z1111"].Conn() + conn3 := s.testClusters["z3333"].Conn() + userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Make sure LoginCluster is properly configured for cls := range s.testClusters { c.Check( - s.testClusters[cls].config.Clusters[cls].Login.LoginCluster, + s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster, check.Equals, "z1111", check.Commentf("incorrect LoginCluster config on cluster %q", cls)) } @@ -564,12 +460,12 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) { } func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { - conn1 := s.conn("z1111") - conn3 := s.conn("z3333") - rootctx1, rootac1, _ := s.rootClients("z1111") + conn1 := s.testClusters["z1111"].Conn() + conn3 := s.testClusters["z3333"].Conn() + rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients() // Create user on LoginCluster z1111 - _, _, _, user := s.userClients(rootctx1, c, conn1, "z1111", false) + _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Make a new root token (because rootClients() uses SystemRootToken) var outAuth arvados.APIClientAuthorization @@ -577,7 +473,7 @@ func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { c.Check(err, check.IsNil) // Make a v2 root token to communicate with z3333 - rootctx3, rootac3, _ := s.clientsWithToken("z3333", outAuth.TokenV2()) + rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2()) // Create VM on z3333 var outVM arvados.VirtualMachine @@ -628,17 +524,17 @@ func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { } func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) { - conn1 := s.conn("z1111") - rootctx1, _, _ := s.rootClients("z1111") - s.userClients(rootctx1, c, conn1, "z1111", true) + conn1 := s.testClusters["z1111"].Conn() + rootctx1, _, _ := s.testClusters["z1111"].RootClients() + s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) accesstoken := s.oidcprovider.ValidAccessToken() - for _, clusterid := range []string{"z1111", "z2222"} { - c.Logf("trying clusterid %s", clusterid) + for _, clusterID := range []string{"z1111", "z2222"} { + c.Logf("trying clusterid %s", clusterID) - conn := s.conn(clusterid) - ctx, ac, kc := s.clientsWithToken(clusterid, accesstoken) + conn := s.testClusters[clusterID].Conn() + ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken) var coll arvados.Collection diff --git a/sdk/go/arvadostest/federation_servers.go b/sdk/go/arvadostest/federation_servers.go new file mode 100644 index 0000000000..da2a9d78c7 --- /dev/null +++ b/sdk/go/arvadostest/federation_servers.go @@ -0,0 +1,155 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package arvadostest + +import ( + "bytes" + "context" + "net/url" + + "git.arvados.org/arvados.git/lib/boot" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/lib/service" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/auth" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/keepclient" + "gopkg.in/check.v1" +) + +// TestCluster stores a working test cluster data +type TestCluster struct { + Super boot.Supervisor + Config arvados.Config + ControllerURL *url.URL + ClusterID string +} + +type logger struct { + loggerfunc func(...interface{}) +} + +func (l logger) Log(args ...interface{}) { + l.loggerfunc(args) +} + +// NewTestCluster loads the provided configuration, and sets up a test cluster +// ready for being started. +func NewTestCluster(srcPath string, clusterID, yamlConf, listenHost string, logWriter func(...interface{})) (*TestCluster, error) { + loader := config.NewLoader(bytes.NewBufferString(yamlConf), ctxlog.TestLogger(logger{logWriter})) + loader.Path = "-" + loader.SkipLegacy = true + loader.SkipAPICalls = true + cfg, err := loader.Load() + if err != nil { + return nil, err + } + return &TestCluster{ + Super: boot.Supervisor{ + SourcePath: srcPath, + ClusterType: "test", + ListenHost: listenHost, + ControllerAddr: ":0", + OwnTemporaryDatabase: true, + Stderr: &service.LogPrefixer{ + Writer: ctxlog.LogWriter(logWriter), + Prefix: []byte("[" + clusterID + "] ")}, + }, + Config: *cfg, + ClusterID: clusterID, + }, nil +} + +// Start the test cluster. +func (tc *TestCluster) Start() { + tc.Super.Start(context.Background(), &tc.Config, "-") +} + +// WaitReady waits for all components to report healthy, and finishes setting +// up the TestCluster struct. +func (tc *TestCluster) WaitReady() bool { + au, ok := tc.Super.WaitReady() + if !ok { + return ok + } + u := url.URL(*au) + tc.ControllerURL = &u + return ok +} + +// ClientsWithToken returns Context, Arvados.Client and keepclient structs +// initialized to connect to the cluster with the supplied Arvados token. +func (tc *TestCluster) ClientsWithToken(token string) (context.Context, *arvados.Client, *keepclient.KeepClient) { + cl := tc.Config.Clusters[tc.ClusterID] + ctx := auth.NewContext(context.Background(), auth.NewCredentials(token)) + ac, err := arvados.NewClientFromConfig(&cl) + if err != nil { + panic(err) + } + ac.AuthToken = token + arv, err := arvadosclient.New(ac) + if err != nil { + panic(err) + } + kc := keepclient.New(arv) + return ctx, ac, kc +} + +// UserClients logs in as a user called "example", get the user's API token, +// initialize clients with the API token, set up the user and +// optionally activate the user. Return client structs for +// communicating with the cluster on behalf of the 'example' user. +func (tc *TestCluster) UserClients(rootctx context.Context, c *check.C, conn *rpc.Conn, authEmail string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient, arvados.User) { + login, err := conn.UserSessionCreate(rootctx, rpc.UserSessionCreateOptions{ + ReturnTo: ",https://example.com", + AuthInfo: rpc.UserSessionAuthInfo{ + Email: authEmail, + FirstName: "Example", + LastName: "User", + Username: "example", + }, + }) + c.Assert(err, check.IsNil) + redirURL, err := url.Parse(login.RedirectLocation) + c.Assert(err, check.IsNil) + userToken := redirURL.Query().Get("api_token") + c.Logf("user token: %q", userToken) + ctx, ac, kc := tc.ClientsWithToken(userToken) + user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{}) + c.Assert(err, check.IsNil) + _, err = conn.UserSetup(rootctx, arvados.UserSetupOptions{UUID: user.UUID}) + c.Assert(err, check.IsNil) + if activate { + _, err = conn.UserActivate(rootctx, arvados.UserActivateOptions{UUID: user.UUID}) + c.Assert(err, check.IsNil) + user, err = conn.UserGetCurrent(ctx, arvados.GetOptions{}) + c.Assert(err, check.IsNil) + c.Logf("user UUID: %q", user.UUID) + if !user.IsActive { + c.Fatalf("failed to activate user -- %#v", user) + } + } + return ctx, ac, kc, user +} + +// RootClients returns Context, arvados.Client and keepclient structs initialized +// to communicate with the cluster as the system root user. +func (tc *TestCluster) RootClients() (context.Context, *arvados.Client, *keepclient.KeepClient) { + return tc.ClientsWithToken(tc.Config.Clusters[tc.ClusterID].SystemRootToken) +} + +// AnonymousClients returns Context, arvados.Client and keepclient structs initialized +// to communicate with the cluster as the anonymous user. +func (tc *TestCluster) AnonymousClients() (context.Context, *arvados.Client, *keepclient.KeepClient) { + return tc.ClientsWithToken(tc.Config.Clusters[tc.ClusterID].Users.AnonymousUserToken) +} + +// Conn gets rpc connection struct initialized to communicate with the +// specified cluster. +func (tc *TestCluster) Conn() *rpc.Conn { + return rpc.NewConn(tc.ClusterID, tc.ControllerURL, true, rpc.PassthroughTokenProvider) +} -- 2.30.2