X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4c1281bba5d3e01d677165b6e2fa7d9209e233b5..ae92d144610446849eb568247a44f02ae985c281:/services/crunch-dispatch-local/crunch-dispatch-local_test.go diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go index ef8cb836c9..e5ce5c66c5 100644 --- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go +++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go @@ -1,19 +1,26 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - - "io/ioutil" - "log" + "bytes" + "context" + "io" "net/http" "net/http/httptest" "os" - "strings" - "syscall" + "os/exec" + "regexp" "testing" "time" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/dispatch" . "gopkg.in/check.v1" ) @@ -32,26 +39,19 @@ var initialArgs []string func (s *TestSuite) SetUpSuite(c *C) { initialArgs = os.Args - arvadostest.StartAPI() -} - -func (s *TestSuite) TearDownSuite(c *C) { - arvadostest.StopAPI() + runningCmds = make(map[string]*exec.Cmd) } func (s *TestSuite) SetUpTest(c *C) { + arvadostest.ResetDB(c) + arvadostest.ResetEnv() args := []string{"crunch-dispatch-local"} os.Args = args - - var err error - arv, err = arvadosclient.MakeArvadosClient() - if err != nil { - c.Fatalf("Error making arvados client: %s", err) - } } func (s *TestSuite) TearDownTest(c *C) { arvadostest.ResetEnv() + arvadostest.ResetDB(c) os.Args = initialArgs } @@ -59,79 +59,105 @@ func (s *MockArvadosServerSuite) TearDownTest(c *C) { arvadostest.ResetEnv() } -func (s *TestSuite) Test_doMain(c *C) { - args := []string{"-poll-interval", "2", "-container-priority-poll-interval", "1", "-crunch-run-command", "echo"} - os.Args = append(os.Args, args...) +func (s *TestSuite) TestIntegration(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, IsNil) - go func() { - time.Sleep(5 * time.Second) - sigChan <- syscall.SIGINT - }() + echo := "echo" + crunchRunCommand = echo - err := doMain() - c.Check(err, IsNil) + ctx, cancel := context.WithCancel(ctxlog.Context(context.Background(), ctxlog.TestLogger(c))) + dispatcher := dispatch.Dispatcher{ + Arv: arv, + PollPeriod: time.Second, + } + + startCmd := func(container arvados.Container, cmd *exec.Cmd) error { + dispatcher.UpdateState(container.UUID, "Running") + dispatcher.UpdateState(container.UUID, "Complete") + return cmd.Start() + } + + cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}} + + dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) error { + defer cancel() + return (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s) + } + + err = dispatcher.Run(ctx) + c.Assert(err, Equals, context.Canceled) + + // Wait for all running crunch jobs to complete / terminate + waitGroup.Wait() // There should be no queued containers now params := arvadosclient.Dict{ - "filters": [][]string{[]string{"state", "=", "Queued"}}, + "filters": [][]string{{"state", "=", "Queued"}}, } - var containers ContainerList + var containers arvados.ContainerList err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Assert(len(containers.Items), Equals, 0) // Previously "Queued" container should now be in "Complete" state - var container Container + var container arvados.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) - c.Check(container.State, Equals, "Complete") + c.Check(string(container.State), Equals, "Complete") } func (s *MockArvadosServerSuite) Test_APIErrorGettingContainers(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)} - testWithServerStub(c, apiStubResponses, "echo", "Error getting list of queued containers") + testWithServerStub(c, apiStubResponses, "echo", "error getting count of containers") } func (s *MockArvadosServerSuite) Test_APIErrorUpdatingContainerState(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx1"}]}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx1","State":"Queued","Priority":1}]}`)} apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx1"] = arvadostest.StubResponse{500, string(`{}`)} - testWithServerStub(c, apiStubResponses, "echo", "Error updating container state") + testWithServerStub(c, apiStubResponses, "echo", "error locking container zzzzz-dz642-xxxxxxxxxxxxxx1") } func (s *MockArvadosServerSuite) Test_ContainerStillInRunningAfterRun(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2"}]}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2","State":"Queued","Priority":1}]}`)} + apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx2/lock"] = + arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2", "state":"Locked", "priority":1, "locked_by_uuid": "` + arvadostest.Dispatch1AuthUUID + `"}`)} apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx2"] = - arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2", "state":"Running", "priority":1}`)} + arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2", "state":"Running", "priority":1, "locked_by_uuid": "` + arvadostest.Dispatch1AuthUUID + `"}`)} testWithServerStub(c, apiStubResponses, "echo", - "After crunch-run process termination, the state is still 'Running' for zzzzz-dz642-xxxxxxxxxxxxxx2") + `after \\"echo\\" process termination, container state for zzzzz-dz642-xxxxxxxxxxxxxx2 is \\"Running\\"; updating it to \\"Cancelled\\"`) } func (s *MockArvadosServerSuite) Test_ErrorRunningContainer(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3"}]}`)} - apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx3"] = - arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3", "state":"Running", "priority":1}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3","State":"Queued","Priority":1}]}`)} + + apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx3/lock"] = + arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3", "state":"Locked", "priority":1}`)} - testWithServerStub(c, apiStubResponses, "nosuchcommand", "Error running container for zzzzz-dz642-xxxxxxxxxxxxxx3") + testWithServerStub(c, apiStubResponses, "nosuchcommand", `error starting \\"nosuchcommand\\" for zzzzz-dz642-xxxxxxxxxxxxxx3`) } func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { + apiStubResponses["/arvados/v1/api_client_authorizations/current"] = + arvadostest.StubResponse{200, string(`{"uuid": "` + arvadostest.Dispatch1AuthUUID + `", "api_token": "xyz"}`)} + apiStub := arvadostest.ServerStub{apiStubResponses} api := httptest.NewServer(&apiStub) defer api.Close() - arv = arvadosclient.ArvadosClient{ + arv := &arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", @@ -139,21 +165,45 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon Retries: 0, } - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - defer os.Remove(tempfile.Name()) - log.SetOutput(tempfile) + buf := bytes.NewBuffer(nil) + logger := ctxlog.TestLogger(c) + logger.SetOutput(io.MultiWriter(buf, logger.Out)) + crunchRunCommand = crunchCmd + + ctx, cancel := context.WithCancel(ctxlog.Context(context.Background(), logger)) + defer cancel() + dispatcher := dispatch.Dispatcher{ + Logger: logger, + Arv: arv, + PollPeriod: time.Second, + } + + startCmd := func(container arvados.Container, cmd *exec.Cmd) error { + dispatcher.UpdateState(container.UUID, "Running") + dispatcher.UpdateState(container.UUID, "Complete") + return cmd.Start() + } + + cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}} + + dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) error { + defer cancel() + return (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s) + } + + re := regexp.MustCompile(`(?ms).*` + expected + `.*`) go func() { - time.Sleep(1 * time.Second) - sigChan <- syscall.SIGTERM + for i := 0; i < 80 && !re.MatchString(buf.String()); i++ { + time.Sleep(100 * time.Millisecond) + } + cancel() }() - runQueuedContainers(1, 1, crunchCmd) - - // Give some time for run goroutine to complete - time.Sleep(5 * time.Second) + err := dispatcher.Run(ctx) + c.Assert(err, Equals, context.Canceled) + c.Check(buf.String(), Matches, `(?ms).*`+expected+`.*`) - buf, _ := ioutil.ReadFile(tempfile.Name()) - c.Check(strings.Contains(string(buf), expected), Equals, true) + c.Logf("test finished, waiting for running crunch jobs to complete / terminate") + waitGroup.Wait() }