X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6a5c119c5deba34d4b61af36edb5aa3a5dc1c70b..HEAD:/lib/controller/integration_test.go diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go index b0ec4293a3..18a0c1992b 100644 --- a/lib/controller/integration_test.go +++ b/lib/controller/integration_test.go @@ -28,6 +28,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/httpserver" + "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) @@ -72,6 +73,8 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { Insecure: true SystemLogs: Format: text + API: + MaxConcurrentRequests: 128 Containers: CloudVMs: Enable: true @@ -165,6 +168,20 @@ func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) { c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses) } +func (s *IntegrationSuite) createTestCollectionManifest(c *check.C, ac *arvados.Client, kc *keepclient.KeepClient, content string) string { + fs, err := (&arvados.Collection{}).FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, content) + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + return mtxt +} + func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { conn1 := s.super.Conn("z1111") rootctx1, _, _ := s.super.RootClients("z1111") @@ -173,34 +190,70 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { // Create the collection to find its PDH (but don't save it // anywhere yet) - var coll1 arvados.Collection - fs1, err := coll1.FileSystem(ac1, kc1) - c.Assert(err, check.IsNil) - f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777) - c.Assert(err, check.IsNil) - _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH") - c.Assert(err, check.IsNil) - err = f.Close() - c.Assert(err, check.IsNil) - mtxt, err := fs1.MarshalManifest(".") - c.Assert(err, check.IsNil) + mtxt := s.createTestCollectionManifest(c, ac1, kc1, c.TestName()) pdh := arvados.PortableDataHash(mtxt) // Looking up the PDH before saving returns 404 if cycle // detection is working. - _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh}) + _, err := conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh}) c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`) // Save the collection on cluster z1111. - coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ + _, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ "manifest_text": mtxt, }}) c.Assert(err, check.IsNil) // Retrieve the collection from cluster z3333. - coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh}) + coll2, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh}) c.Check(err, check.IsNil) - c.Check(coll.PortableDataHash, check.Equals, pdh) + c.Check(coll2.PortableDataHash, check.Equals, pdh) +} + +func (s *IntegrationSuite) TestFederation_Write1Read2(c *check.C) { + s.testFederationCollectionAccess(c, "z1111", "z2222") +} + +func (s *IntegrationSuite) TestFederation_Write2Read1(c *check.C) { + s.testFederationCollectionAccess(c, "z2222", "z1111") +} + +func (s *IntegrationSuite) TestFederation_Write2Read3(c *check.C) { + s.testFederationCollectionAccess(c, "z2222", "z3333") +} + +func (s *IntegrationSuite) testFederationCollectionAccess(c *check.C, writeCluster, readCluster string) { + conn1 := s.super.Conn("z1111") + rootctx1, _, _ := s.super.RootClients("z1111") + _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true) + + connW := s.super.Conn(writeCluster) + userctxW, acW, kcW := s.super.ClientsWithToken(writeCluster, ac1.AuthToken) + kcW.DiskCacheSize = keepclient.DiskCacheDisabled + connR := s.super.Conn(readCluster) + userctxR, acR, kcR := s.super.ClientsWithToken(readCluster, ac1.AuthToken) + kcR.DiskCacheSize = keepclient.DiskCacheDisabled + + filedata := fmt.Sprintf("%s: write to %s, read from %s", c.TestName(), writeCluster, readCluster) + mtxt := s.createTestCollectionManifest(c, acW, kcW, filedata) + collW, err := connW.CollectionCreate(userctxW, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + + collR, err := connR.CollectionGet(userctxR, arvados.GetOptions{UUID: collW.UUID}) + if !c.Check(err, check.IsNil) { + return + } + fsR, err := collR.FileSystem(acR, kcR) + if !c.Check(err, check.IsNil) { + return + } + buf, err := fs.ReadFile(arvados.FS(fsR), "test.txt") + if !c.Check(err, check.IsNil) { + return + } + c.Check(string(buf), check.Equals, filedata) } // Tests bug #18004 @@ -499,6 +552,7 @@ func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken) resp, err = arvados.InsecureHTTPClient.Do(req) c.Assert(err, check.IsNil) + defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&cr) c.Check(err, check.IsNil) c.Check(cr.UUID, check.Matches, "z2222-.*") @@ -536,8 +590,10 @@ func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) { c.Assert(err, check.IsNil) req.Header.Set("Content-Type", "application/json") resp, err := ac1.Do(req) - c.Assert(err, check.IsNil) - c.Assert(resp.StatusCode, check.Equals, tt.expectedCode) + if c.Check(err, check.IsNil) { + c.Assert(resp.StatusCode, check.Equals, tt.expectedCode) + resp.Body.Close() + } } } @@ -548,8 +604,6 @@ func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{}) c.Check(err, check.IsNil) - specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{}) - c.Check(err, check.IsNil) tests := []struct { path string @@ -562,8 +616,6 @@ func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { {"/arvados/v1/nonexistant", true, true}, {"/arvados/v1/collections/" + coll.UUID, false, false}, {"/arvados/v1/collections/" + coll.UUID, true, false}, - {"/arvados/v1/specimens/" + specimen.UUID, false, false}, - {"/arvados/v1/specimens/" + specimen.UUID, true, false}, // new code path (lib/controller/router etc) - single-cluster request {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true}, {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true}, @@ -571,8 +623,8 @@ func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", false, true}, {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", true, true}, // old code path (proxyRailsAPI) - single-cluster request - {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true}, - {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true}, + {"/arvados/v1/containers/z1111-dz642-0123456789abcde", false, true}, + {"/arvados/v1/containers/z1111-dz642-0123456789abcde", true, true}, // old code path (setupProxyRemoteCluster) - federated request {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", false, true}, {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", true, true}, @@ -605,9 +657,11 @@ func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) { var jresp httpserver.ErrorResponse err := json.NewDecoder(resp.Body).Decode(&jresp) c.Check(err, check.IsNil) - c.Assert(jresp.Errors, check.HasLen, 1) - c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`) + if c.Check(jresp.Errors, check.HasLen, 1) { + c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`) + } } + resp.Body.Close() } } @@ -772,7 +826,6 @@ func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) { // Test for bug #18076 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) { rootctx1, _, _ := s.super.RootClients("z1111") - _, rootclnt3, _ := s.super.RootClients("z3333") conn1 := s.super.Conn("z1111") conn3 := s.super.Conn("z3333") @@ -784,92 +837,69 @@ func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) { check.Commentf("incorrect LoginCluster config on cluster %q", cls)) } - for testCaseNr, testCase := range []struct { - name string - withRepository bool - }{ - {"User without local repository", false}, - {"User with local repository", true}, - } { - c.Log(c.TestName() + " " + testCase.name) - // Create some users, request them on the federated cluster so they're cached. - var users []arvados.User - for userNr := 0; userNr < 2; userNr++ { - _, _, _, user := s.super.UserClients("z1111", - rootctx1, - c, - conn1, - fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr), - true) - c.Assert(user.Username, check.Not(check.Equals), "") - users = append(users, user) - - lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1}) - c.Assert(err, check.Equals, nil) - userFound := false - for _, fedUser := range lst.Items { - if fedUser.UUID == user.UUID { - c.Assert(fedUser.Username, check.Equals, user.Username) - userFound = true - break - } - } - c.Assert(userFound, check.Equals, true) - - if testCase.withRepository { - var repo interface{} - err = rootclnt3.RequestAndDecode( - &repo, "POST", "arvados/v1/repositories", nil, - map[string]interface{}{ - "repository": map[string]string{ - "name": fmt.Sprintf("%s/test", user.Username), - "owner_uuid": user.UUID, - }, - }, - ) - c.Assert(err, check.IsNil) - } - } + // Create some users, request them on the federated cluster so they're cached. + var users []arvados.User + for userNr := 0; userNr < 2; userNr++ { + _, _, _, user := s.super.UserClients("z1111", + rootctx1, + c, + conn1, + fmt.Sprintf("user0%d@example.com", userNr), + true) + c.Assert(user.Username, check.Not(check.Equals), "") + users = append(users, user) - // Swap the usernames - _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ - UUID: users[0].UUID, - Attrs: map[string]interface{}{ - "username": "", - }, - }) - c.Assert(err, check.Equals, nil) - _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ - UUID: users[1].UUID, - Attrs: map[string]interface{}{ - "username": users[0].Username, - }, - }) - c.Assert(err, check.Equals, nil) - _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ - UUID: users[0].UUID, - Attrs: map[string]interface{}{ - "username": users[1].Username, - }, - }) - c.Assert(err, check.Equals, nil) - - // Re-request the list on the federated cluster & check for updates lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1}) c.Assert(err, check.Equals, nil) - var user0Found, user1Found bool - for _, user := range lst.Items { - if user.UUID == users[0].UUID { - user0Found = true - c.Assert(user.Username, check.Equals, users[1].Username) - } else if user.UUID == users[1].UUID { - user1Found = true - c.Assert(user.Username, check.Equals, users[0].Username) + userFound := false + for _, fedUser := range lst.Items { + if fedUser.UUID == user.UUID { + c.Assert(fedUser.Username, check.Equals, user.Username) + userFound = true + break } } - c.Assert(user0Found, check.Equals, true) - c.Assert(user1Found, check.Equals, true) + c.Assert(userFound, check.Equals, true) } + + // Swap the usernames + _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ + UUID: users[0].UUID, + Attrs: map[string]interface{}{ + "username": "", + }, + }) + c.Assert(err, check.Equals, nil) + _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ + UUID: users[1].UUID, + Attrs: map[string]interface{}{ + "username": users[0].Username, + }, + }) + c.Assert(err, check.Equals, nil) + _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{ + UUID: users[0].UUID, + Attrs: map[string]interface{}{ + "username": users[1].Username, + }, + }) + c.Assert(err, check.Equals, nil) + + // Re-request the list on the federated cluster & check for updates + lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1}) + c.Assert(err, check.Equals, nil) + var user0Found, user1Found bool + for _, user := range lst.Items { + if user.UUID == users[0].UUID { + user0Found = true + c.Assert(user.Username, check.Equals, users[1].Username) + } else if user.UUID == users[1].UUID { + user1Found = true + c.Assert(user.Username, check.Equals, users[0].Username) + } + } + c.Assert(user0Found, check.Equals, true) + c.Assert(user1Found, check.Equals, true) } // Test for bug #16263 @@ -964,8 +994,8 @@ func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { "hostname": "example", }, }) + c.Assert(err, check.IsNil) c.Check(outVM.UUID[0:5], check.Equals, "z3333") - c.Check(err, check.IsNil) // Make sure z3333 user list is up to date _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000}) @@ -1133,7 +1163,7 @@ func (s *IntegrationSuite) TestRunTrivialContainer(c *check.C) { "environment": map[string]string{}, "mounts": map[string]arvados.Mount{"/out": {Kind: "tmp", Capacity: 10000}}, "output_path": "/out", - "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1}, + "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26}, "priority": 1, "state": arvados.ContainerRequestStateCommitted, }, 0) @@ -1160,7 +1190,7 @@ func (s *IntegrationSuite) TestContainerInputOnDifferentCluster(c *check.C) { "/out": {Kind: "tmp", Capacity: 10000}, }, "output_path": "/out", - "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1}, + "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26}, "priority": 1, "state": arvados.ContainerRequestStateCommitted, "container_count_max": 1, @@ -1224,12 +1254,35 @@ func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, token stri return cfs } + checkwebdavlogs := func(cr arvados.ContainerRequest) { + req, err := http.NewRequest("OPTIONS", "https://"+ac.APIHost+"/arvados/v1/container_requests/"+cr.UUID+"/log/"+cr.ContainerUUID+"/", nil) + c.Assert(err, check.IsNil) + req.Header.Set("Origin", "http://example.example") + resp, err := ac.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + // Check for duplicate headers -- must use Header[], not Header.Get() + c.Check(resp.Header["Access-Control-Allow-Origin"], check.DeepEquals, []string{"*"}) + } + var ctr arvados.Container var lastState arvados.ContainerState + var status, lastStatus arvados.ContainerStatus + var allStatus string + checkstatus := func() { + err := ac.RequestAndDecode(&status, "GET", "/arvados/v1/container_requests/"+cr.UUID+"/container_status", nil, nil) + c.Assert(err, check.IsNil) + if status != lastStatus { + c.Logf("container status: %s, %s", status.State, status.SchedulingStatus) + allStatus += fmt.Sprintf("%s, %s\n", status.State, status.SchedulingStatus) + lastStatus = status + } + } deadline := time.Now().Add(time.Minute) - for cr.State != arvados.ContainerRequestStateFinal { + for cr.State != arvados.ContainerRequestStateFinal || (lastStatus.State != arvados.ContainerStateComplete && lastStatus.State != arvados.ContainerStateCancelled) { err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil) c.Assert(err, check.IsNil) + checkstatus() err = ac.RequestAndDecode(&ctr, "GET", "/arvados/v1/containers/"+cr.ContainerUUID, nil, nil) if err != nil { c.Logf("error getting container state: %s", err) @@ -1239,18 +1292,35 @@ func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, token stri } else { if time.Now().After(deadline) { c.Errorf("timed out, container state is %q", cr.State) - showlogs(ctr.Log) + if ctr.Log == "" { + c.Logf("=== NO LOG COLLECTION saved for container") + } else { + showlogs(ctr.Log) + } c.FailNow() } time.Sleep(time.Second / 2) } } + checkstatus() + c.Logf("cr.CumulativeCost == %f", cr.CumulativeCost) + c.Check(cr.CumulativeCost, check.Not(check.Equals), 0.0) if expectExitCode >= 0 { c.Check(ctr.State, check.Equals, arvados.ContainerStateComplete) c.Check(ctr.ExitCode, check.Equals, expectExitCode) err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil) c.Assert(err, check.IsNil) + c.Check(allStatus, check.Matches, `Queued, waiting for dispatch\n`+ + // Occasionally the dispatcher will + // unlock/retry, and we get state/status from + // database/dispatcher via separate API calls, + // so we can also see "Queued, preparing + // runtime environment". + `((Queued|Locked), (waiting .*|preparing runtime environment)\n)*`+ + `(Running, \n)?`+ + `Complete, \n`) } logcfs = showlogs(cr.LogUUID) + checkwebdavlogs(cr) return outcoll, logcfs }