X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/927524f1be454de021180b74999d682780b8cb6b..58e6402a72e9ac1a210b2d318591f973a37e1e57:/lib/controller/integration_test.go diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go index b7bda3dd16..67d60197e7 100644 --- a/lib/controller/integration_test.go +++ b/lib/controller/integration_test.go @@ -11,19 +11,19 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "io/ioutil" "math" "net" "net/http" "os" "os/exec" - "path/filepath" "strconv" "strings" "sync" + "time" "git.arvados.org/arvados.git/lib/boot" - "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/ctxlog" @@ -34,13 +34,11 @@ import ( var _ = check.Suite(&IntegrationSuite{}) type IntegrationSuite struct { - testClusters map[string]*boot.TestCluster + super *boot.Supervisor oidcprovider *arvadostest.OIDCProvider } func (s *IntegrationSuite) SetUpSuite(c *check.C) { - cwd, _ := os.Getwd() - s.oidcprovider = arvadostest.NewOIDCProvider(c) s.oidcprovider.AuthEmail = "user@example.com" s.oidcprovider.AuthEmailVerified = true @@ -48,13 +46,8 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { s.oidcprovider.ValidClientID = "clientid" s.oidcprovider.ValidClientSecret = "clientsecret" - s.testClusters = map[string]*boot.TestCluster{ - "z1111": nil, - "z2222": nil, - "z3333": nil, - } hostport := map[string]string{} - for id := range s.testClusters { + for _, id := range []string{"z1111", "z2222", "z3333"} { hostport[id] = func() string { // TODO: Instead of expecting random ports on // 127.0.0.11, 22, 33 to be race-safe, try @@ -68,8 +61,9 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { return "127.0.0." + id[3:] + ":" + port }() } - for id := range s.testClusters { - yaml := `Clusters: + yaml := "Clusters:\n" + for id := range hostport { + yaml += ` ` + id + `: Services: Controller: @@ -78,6 +72,18 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { Insecure: true SystemLogs: Format: text + Containers: + CloudVMs: + Enable: true + Driver: loopback + BootProbeCommand: "rm -f /var/lock/crunch-run-broken" + ProbeInterval: 1s + PollInterval: 5s + SyncInterval: 10s + TimeoutIdle: 1s + TimeoutBooting: 2s + RuntimeEngine: singularity + CrunchRunArgumentsList: ["--broken-node-hook", "true"] RemoteClusters: z1111: Host: ` + hostport["z1111"] + ` @@ -124,36 +130,35 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { LoginCluster: z1111 ` } - - loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c)) - loader.Path = "-" - loader.SkipLegacy = true - loader.SkipAPICalls = true - cfg, err := loader.Load() - c.Assert(err, check.IsNil) - tc := boot.NewTestCluster( - filepath.Join(cwd, "..", ".."), - id, cfg, "127.0.0."+id[3:], c.Log) - tc.Super.NoWorkbench1 = true - tc.Start() - s.testClusters[id] = tc } - for _, tc := range s.testClusters { - ok := tc.WaitReady() - c.Assert(ok, check.Equals, true) + s.super = &boot.Supervisor{ + ClusterType: "test", + ConfigYAML: yaml, + Stderr: ctxlog.LogWriter(c.Log), + NoWorkbench1: true, + NoWorkbench2: true, + OwnTemporaryDatabase: true, } + + // Give up if startup takes longer than 3m + timeout := time.AfterFunc(3*time.Minute, s.super.Stop) + defer timeout.Stop() + s.super.Start(context.Background()) + ok := s.super.WaitReady() + c.Assert(ok, check.Equals, true) } func (s *IntegrationSuite) TearDownSuite(c *check.C) { - for _, c := range s.testClusters { - c.Super.Stop() + if s.super != nil { + s.super.Stop() + s.super.Wait() } } func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) { - conn := s.testClusters["z1111"].Conn() - rootctx, _, _ := s.testClusters["z1111"].RootClients() - userctx, _, kc, _ := s.testClusters["z1111"].UserClients(rootctx, c, conn, s.oidcprovider.AuthEmail, true) + conn := s.super.Conn("z1111") + rootctx, _, _ := s.super.RootClients("z1111") + userctx, _, kc, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true) c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true) coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{}) c.Assert(err, check.IsNil) @@ -161,10 +166,10 @@ func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) { } func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - conn3 := s.testClusters["z3333"].Conn() - userctx1, ac1, kc1, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + conn3 := s.super.Conn("z3333") + userctx1, ac1, kc1, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Create the collection to find its PDH (but don't save it // anywhere yet) @@ -200,11 +205,11 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { // Tests bug #18004 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - rootctx2, _, _ := s.testClusters["z2222"].RootClients() - conn2 := s.testClusters["z2222"].Conn() - userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user2@example.com", true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + rootctx2, _, _ := s.super.RootClients("z2222") + conn2 := s.super.Conn("z2222") + userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user2@example.com", true) var wg1, wg2 sync.WaitGroup creqs := 100 @@ -249,13 +254,13 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { testText := "IntegrationSuite.TestS3WithFederatedToken" - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) - conn3 := s.testClusters["z3333"].Conn() + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + conn3 := s.super.Conn("z3333") createColl := func(clusterID string) arvados.Collection { - _, ac, kc := s.testClusters[clusterID].ClientsWithToken(ac1.AuthToken) + _, ac, kc := s.super.ClientsWithToken(clusterID, ac1.AuthToken) var coll arvados.Collection fs, err := coll.FileSystem(ac, kc) c.Assert(err, check.IsNil) @@ -267,7 +272,7 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { c.Assert(err, check.IsNil) mtxt, err := fs.MarshalManifest(".") c.Assert(err, check.IsNil) - coll, err = s.testClusters[clusterID].Conn().CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ + coll, err = s.super.Conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ "manifest_text": mtxt, }}) c.Assert(err, check.IsNil) @@ -315,10 +320,10 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { } func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - conn3 := s.testClusters["z3333"].Conn() - rootctx1, rootac1, rootkc1 := s.testClusters["z1111"].RootClients() - anonctx3, anonac3, _ := s.testClusters["z3333"].AnonymousClients() + conn1 := s.super.Conn("z1111") + conn3 := s.super.Conn("z3333") + rootctx1, rootac1, rootkc1 := s.super.RootClients("z1111") + anonctx3, anonac3, _ := s.super.AnonymousClients("z3333") // Make sure anonymous token was set c.Assert(anonac3.AuthToken, check.Not(check.Equals), "") @@ -367,7 +372,7 @@ func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { c.Check(err, check.IsNil) // Make a v2 token of the z3 anonymous user, and use it on z1 - _, anonac1, _ := s.testClusters["z1111"].ClientsWithToken(outAuth.TokenV2()) + _, anonac1, _ := s.super.ClientsWithToken("z1111", outAuth.TokenV2()) outUser2, err := anonac1.CurrentUser() c.Check(err, check.IsNil) // z3 anonymous user will be mapped to the z1 anonymous user @@ -379,17 +384,66 @@ func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { c.Check(coll.PortableDataHash, check.Equals, pdh) } +// z3333 should forward the locally-issued anonymous user token to its login +// cluster z1111. That is no problem because the login cluster controller will +// map any anonymous user token to its local anonymous user. +// +// This needs to work because wb1 has a tendency to slap the local anonymous +// user token on every request as a reader_token, which gets folded into the +// request token list controller. +// +// Use a z1111 user token and the anonymous token from z3333 passed in as a +// reader_token to do a request on z3333, asking for the z1111 anonymous user +// object. The request will be forwarded to the z1111 cluster. The presence of +// the z3333 anonymous user token should not prohibit the request from being +// forwarded. +func (s *IntegrationSuite) TestForwardAnonymousTokenToLoginCluster(c *check.C) { + conn1 := s.super.Conn("z1111") + + rootctx1, _, _ := s.super.RootClients("z1111") + _, anonac3, _ := s.super.AnonymousClients("z3333") + + // Make a user connection to z3333 (using a z1111 user, because that's the login cluster) + _, userac1, _, _ := s.super.UserClients("z3333", rootctx1, c, conn1, "user@example.com", true) + + // Get the anonymous user token for z3333 + var anon3Auth arvados.APIClientAuthorization + err := anonac3.RequestAndDecode(&anon3Auth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil) + c.Check(err, check.IsNil) + + var userList arvados.UserList + where := make(map[string]string) + where["uuid"] = "z1111-tpzed-anonymouspublic" + err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil, + map[string]interface{}{ + "reader_tokens": []string{anon3Auth.TokenV2()}, + "where": where, + }, + ) + // The local z3333 anonymous token must be allowed to be forwarded to the login cluster + c.Check(err, check.IsNil) + + userac1.AuthToken = "v2/z1111-gj3su-asdfasdfasdfasd/this-token-does-not-validate-so-anonymous-token-will-be-used-instead" + err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil, + map[string]interface{}{ + "reader_tokens": []string{anon3Auth.TokenV2()}, + "where": where, + }, + ) + c.Check(err, check.IsNil) +} + // Get a token from the login cluster (z1111), use it to submit a // container request on z2222. func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - _, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Use ac2 to get the discovery doc with a blank token, so the // SDK doesn't magically pass the z1111 token to z2222 before // we're ready to start our test. - _, ac2, _ := s.testClusters["z2222"].ClientsWithToken("") + _, ac2, _ := s.super.ClientsWithToken("z2222", "") var dd map[string]interface{} err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil) c.Assert(err, check.IsNil) @@ -451,9 +505,9 @@ func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { } func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - _, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + _, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) tests := []struct { name string @@ -488,9 +542,9 @@ func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) { } func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - userctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{}) c.Check(err, check.IsNil) @@ -562,7 +616,7 @@ func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { // to test tokens that are secret, so there is no API response that will give them back func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) { ctx := context.Background() - db, err := sql.Open("postgres", s.testClusters[clusterID].Super.Cluster().PostgreSQL.Connection.String()) + db, err := sql.Open("postgres", s.super.Cluster(clusterID).PostgreSQL.Connection.String()) c.Assert(err, check.IsNil) conn, err := db.Conn(ctx) @@ -582,9 +636,9 @@ func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) { db, dbconn := s.dbConn(c, "z1111") defer db.Close() defer dbconn.Close() - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - userctx1, ac1, _, au := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + userctx1, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) tests := []struct { name string @@ -634,9 +688,9 @@ func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) { // one cluster with another cluster as the destination // and check the tokens are being handled properly func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - uctx1, ac1, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + uctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) tests := []struct { name string @@ -666,20 +720,20 @@ func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) { // Test for #17785 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) { - rootctx1, rootclnt1, _ := s.testClusters["z1111"].RootClients() - conn1 := s.testClusters["z1111"].Conn() + rootctx1, rootclnt1, _ := s.super.RootClients("z1111") + conn1 := s.super.Conn("z1111") // Make sure LoginCluster is properly configured for _, cls := range []string{"z1111", "z3333"} { c.Check( - s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster, + s.super.Cluster(cls).Login.LoginCluster, check.Equals, "z1111", check.Commentf("incorrect LoginCluster config on cluster %q", cls)) } // Get user's UUID & attempt to create a token for it on the remote cluster - _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, + _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) - _, rootclnt3, _ := s.testClusters["z3333"].ClientsWithToken(rootclnt1.AuthToken) + _, rootclnt3, _ := s.super.ClientsWithToken("z3333", rootclnt1.AuthToken) var resp arvados.APIClientAuthorization err := rootclnt3.RequestAndDecode( &resp, "POST", "arvados/v1/api_client_authorizations", nil, @@ -690,6 +744,7 @@ func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) { }, ) c.Assert(err, check.IsNil) + c.Assert(resp.APIClientID, check.Not(check.Equals), 0) newTok := resp.TokenV2() c.Assert(newTok, check.Not(check.Equals), "") @@ -697,26 +752,34 @@ func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) { c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true) // Confirm the token works and is from the correct user - _, rootclnt3bis, _ := s.testClusters["z3333"].ClientsWithToken(newTok) + _, rootclnt3bis, _ := s.super.ClientsWithToken("z3333", newTok) var curUser arvados.User err = rootclnt3bis.RequestAndDecode( &curUser, "GET", "arvados/v1/users/current", nil, nil, ) c.Assert(err, check.IsNil) c.Assert(curUser.UUID, check.Equals, user.UUID) + + // Request the ApiClientAuthorization list using the new token + _, userClient, _ := s.super.ClientsWithToken("z3333", newTok) + var acaLst arvados.APIClientAuthorizationList + err = userClient.RequestAndDecode( + &acaLst, "GET", "arvados/v1/api_client_authorizations", nil, nil, + ) + c.Assert(err, check.IsNil) } // Test for bug #18076 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) { - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - _, rootclnt3, _ := s.testClusters["z3333"].RootClients() - conn1 := s.testClusters["z1111"].Conn() - conn3 := s.testClusters["z3333"].Conn() + rootctx1, _, _ := s.super.RootClients("z1111") + _, rootclnt3, _ := s.super.RootClients("z3333") + conn1 := s.super.Conn("z1111") + conn3 := s.super.Conn("z3333") // Make sure LoginCluster is properly configured for _, cls := range []string{"z1111", "z3333"} { c.Check( - s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster, + s.super.Cluster(cls).Login.LoginCluster, check.Equals, "z1111", check.Commentf("incorrect LoginCluster config on cluster %q", cls)) } @@ -732,7 +795,7 @@ func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) { // Create some users, request them on the federated cluster so they're cached. var users []arvados.User for userNr := 0; userNr < 2; userNr++ { - _, _, _, user := s.testClusters["z1111"].UserClients( + _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1, @@ -811,15 +874,15 @@ func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) { // Test for bug #16263 func (s *IntegrationSuite) TestListUsers(c *check.C) { - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - conn1 := s.testClusters["z1111"].Conn() - conn3 := s.testClusters["z3333"].Conn() - userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + rootctx1, _, _ := s.super.RootClients("z1111") + conn1 := s.super.Conn("z1111") + conn3 := s.super.Conn("z3333") + userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Make sure LoginCluster is properly configured - for cls := range s.testClusters { + for _, cls := range []string{"z1111", "z2222", "z3333"} { c.Check( - s.testClusters[cls].Config.Clusters[cls].Login.LoginCluster, + s.super.Cluster(cls).Login.LoginCluster, check.Equals, "z1111", check.Commentf("incorrect LoginCluster config on cluster %q", cls)) } @@ -879,12 +942,12 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) { } func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - conn3 := s.testClusters["z3333"].Conn() - rootctx1, rootac1, _ := s.testClusters["z1111"].RootClients() + conn1 := s.super.Conn("z1111") + conn3 := s.super.Conn("z3333") + rootctx1, rootac1, _ := s.super.RootClients("z1111") // Create user on LoginCluster z1111 - _, _, _, user := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) // Make a new root token (because rootClients() uses SystemRootToken) var outAuth arvados.APIClientAuthorization @@ -892,7 +955,7 @@ func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { c.Check(err, check.IsNil) // Make a v2 root token to communicate with z3333 - rootctx3, rootac3, _ := s.testClusters["z3333"].ClientsWithToken(outAuth.TokenV2()) + rootctx3, rootac3, _ := s.super.ClientsWithToken("z3333", outAuth.TokenV2()) // Create VM on z3333 var outVM arvados.VirtualMachine @@ -943,9 +1006,9 @@ func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { } func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) { - conn1 := s.testClusters["z1111"].Conn() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - s.testClusters["z1111"].UserClients(rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) accesstoken := s.oidcprovider.ValidAccessToken() @@ -957,8 +1020,8 @@ func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) { { c.Logf("save collection to %s", clusterID) - conn := s.testClusters[clusterID].Conn() - ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken) + conn := s.super.Conn(clusterID) + ctx, ac, kc := s.super.ClientsWithToken(clusterID, accesstoken) fs, err := coll.FileSystem(ac, kc) c.Assert(err, check.IsNil) @@ -982,8 +1045,8 @@ func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) { for _, readClusterID := range []string{"z1111", "z2222", "z3333"} { c.Logf("retrieve %s from %s", coll.UUID, readClusterID) - conn := s.testClusters[readClusterID].Conn() - ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken) + conn := s.super.Conn(readClusterID) + ctx, ac, kc := s.super.ClientsWithToken(readClusterID, accesstoken) user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{}) c.Assert(err, check.IsNil) @@ -1011,11 +1074,11 @@ func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) { db3, db3conn := s.dbConn(c, "z3333") defer db3.Close() defer db3conn.Close() - rootctx1, _, _ := s.testClusters["z1111"].RootClients() - rootctx3, _, _ := s.testClusters["z3333"].RootClients() - conn1 := s.testClusters["z1111"].Conn() - conn3 := s.testClusters["z3333"].Conn() - userctx1, _, _, _ := s.testClusters["z1111"].UserClients(rootctx1, c, conn1, "user@example.com", true) + rootctx1, _, _ := s.super.RootClients("z1111") + rootctx3, _, _ := s.super.RootClients("z3333") + conn1 := s.super.Conn("z1111") + conn3 := s.super.Conn("z3333") + userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true) user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{}) c.Assert(err, check.IsNil) @@ -1053,7 +1116,7 @@ func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) { row.Scan(&val) c.Assert(val.Valid, check.Equals, true) runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String - ctrctx, _, _ := s.testClusters["z3333"].ClientsWithToken(runtimeToken) + ctrctx, _, _ := s.super.ClientsWithToken("z3333", runtimeToken) c.Logf("container runtime token %+v", runtimeToken) _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID}) @@ -1061,3 +1124,100 @@ func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) { c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`) c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`) } + +func (s *IntegrationSuite) TestRunTrivialContainer(c *check.C) { + outcoll := s.runContainer(c, "z1111", map[string]interface{}{ + "command": []string{"sh", "-c", "touch \"/out/hello world\" /out/ohai"}, + "container_image": "busybox:uclibc", + "cwd": "/tmp", + "environment": map[string]string{}, + "mounts": map[string]arvados.Mount{"/out": {Kind: "tmp", Capacity: 10000}}, + "output_path": "/out", + "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1}, + "priority": 1, + "state": arvados.ContainerRequestStateCommitted, + }, 0) + c.Check(outcoll.ManifestText, check.Matches, `\. d41d8.* 0:0:hello\\040world 0:0:ohai\n`) + c.Check(outcoll.PortableDataHash, check.Equals, "8fa5dee9231a724d7cf377c5a2f4907c+65") +} + +func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, ctrSpec map[string]interface{}, expectExitCode int) arvados.Collection { + conn := s.super.Conn(clusterID) + rootctx, _, _ := s.super.RootClients(clusterID) + _, ac, kc, _ := s.super.UserClients(clusterID, rootctx, c, conn, s.oidcprovider.AuthEmail, true) + + c.Log("[docker load]") + out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput() + c.Logf("[docker load output] %s", out) + c.Check(err, check.IsNil) + + c.Log("[arv-keepdocker]") + akd := exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc") + akd.Env = append(os.Environ(), "ARVADOS_API_HOST="+ac.APIHost, "ARVADOS_API_HOST_INSECURE=1", "ARVADOS_API_TOKEN="+ac.AuthToken) + out, err = akd.CombinedOutput() + c.Logf("[arv-keepdocker output]\n%s", out) + c.Check(err, check.IsNil) + + var cr arvados.ContainerRequest + err = ac.RequestAndDecode(&cr, "POST", "/arvados/v1/container_requests", nil, map[string]interface{}{ + "container_request": ctrSpec, + }) + c.Assert(err, check.IsNil) + + showlogs := func(collectionID string) { + var logcoll arvados.Collection + err = ac.RequestAndDecode(&logcoll, "GET", "/arvados/v1/collections/"+collectionID, nil, nil) + c.Assert(err, check.IsNil) + cfs, err := logcoll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() || strings.HasPrefix(path, "/log for container") { + return nil + } + f, err := cfs.Open(path) + c.Assert(err, check.IsNil) + defer f.Close() + buf, err := ioutil.ReadAll(f) + c.Assert(err, check.IsNil) + c.Logf("=== %s\n%s\n", path, buf) + return nil + }) + } + + var ctr arvados.Container + var lastState arvados.ContainerState + deadline := time.Now().Add(time.Minute) +wait: + for ; ; lastState = ctr.State { + err = ac.RequestAndDecode(&ctr, "GET", "/arvados/v1/containers/"+cr.ContainerUUID, nil, nil) + c.Assert(err, check.IsNil) + switch ctr.State { + case lastState: + if time.Now().After(deadline) { + c.Errorf("timed out, container request state is %q", cr.State) + showlogs(ctr.Log) + c.FailNow() + } + time.Sleep(time.Second / 2) + case arvados.ContainerStateComplete: + break wait + case arvados.ContainerStateQueued, arvados.ContainerStateLocked, arvados.ContainerStateRunning: + c.Logf("container state changed to %q", ctr.State) + default: + c.Errorf("unexpected container state %q", ctr.State) + showlogs(ctr.Log) + c.FailNow() + } + } + c.Check(ctr.ExitCode, check.Equals, 0) + + err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil) + c.Assert(err, check.IsNil) + + showlogs(cr.LogUUID) + + var outcoll arvados.Collection + err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil) + c.Assert(err, check.IsNil) + return outcoll +}