X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0c5e684422234d62870b3c06b9e1014988878099..4153cb6cfad920ed0b1a4b818d3bcc8de492d134:/services/crunch-dispatch-local/crunch-dispatch-local_test.go diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go index e3ab3a4e1d..aca60e9b7b 100644 --- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go +++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go @@ -1,19 +1,20 @@ package main import ( + "bytes" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" - - "bytes" + "git.curoverse.com/arvados.git/sdk/go/dispatch" + . "gopkg.in/check.v1" + "io" "log" "net/http" "net/http/httptest" "os" - "syscall" + "os/exec" + "strings" "testing" "time" - - . "gopkg.in/check.v1" ) // Gocheck boilerplate @@ -32,6 +33,7 @@ var initialArgs []string func (s *TestSuite) SetUpSuite(c *C) { initialArgs = os.Args arvadostest.StartAPI() + runningCmds = make(map[string]*exec.Cmd) } func (s *TestSuite) TearDownSuite(c *C) { @@ -41,12 +43,6 @@ func (s *TestSuite) TearDownSuite(c *C) { func (s *TestSuite) SetUpTest(c *C) { args := []string{"crunch-dispatch-local"} os.Args = args - - var err error - arv, err = arvadosclient.MakeArvadosClient() - if err != nil { - c.Fatalf("Error making arvados client: %s", err) - } } func (s *TestSuite) TearDownTest(c *C) { @@ -58,29 +54,48 @@ func (s *MockArvadosServerSuite) TearDownTest(c *C) { arvadostest.ResetEnv() } -func (s *TestSuite) Test_doMain(c *C) { - args := []string{"-poll-interval", "2", "-container-priority-poll-interval", "1", "-crunch-run-command", "echo"} - os.Args = append(os.Args, args...) +func (s *TestSuite) TestIntegration(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, IsNil) + + echo := "echo" + crunchRunCommand = &echo + + doneProcessing := make(chan struct{}) + dispatcher := dispatch.Dispatcher{ + Arv: arv, + PollInterval: time.Duration(1) * time.Second, + RunContainer: func(dispatcher *dispatch.Dispatcher, + container dispatch.Container, + status chan dispatch.Container) { + run(dispatcher, container, status) + doneProcessing <- struct{}{} + }, + DoneProcessing: doneProcessing} + + startCmd = func(container dispatch.Container, cmd *exec.Cmd) error { + dispatcher.UpdateState(container.UUID, "Running") + dispatcher.UpdateState(container.UUID, "Complete") + return cmd.Start() + } - go func() { - time.Sleep(5 * time.Second) - sigChan <- syscall.SIGINT - }() + err = dispatcher.RunDispatcher() + c.Assert(err, IsNil) - err := doMain() - c.Check(err, IsNil) + // Wait for all running crunch jobs to complete / terminate + waitGroup.Wait() // There should be no queued containers now params := arvadosclient.Dict{ "filters": [][]string{[]string{"state", "=", "Queued"}}, } - var containers ContainerList + var containers dispatch.ContainerList err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Assert(len(containers.Items), Equals, 0) // Previously "Queued" container should now be in "Complete" state - var container Container + var container dispatch.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) c.Check(container.State, Equals, "Complete") @@ -90,13 +105,13 @@ func (s *MockArvadosServerSuite) Test_APIErrorGettingContainers(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)} - testWithServerStub(c, apiStubResponses, "echo", "Error getting list of queued containers") + testWithServerStub(c, apiStubResponses, "echo", "Error getting list of containers") } func (s *MockArvadosServerSuite) Test_APIErrorUpdatingContainerState(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx1"}]}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx1","State":"Queued"}]}`)} apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx1"] = arvadostest.StubResponse{500, string(`{}`)} @@ -106,31 +121,35 @@ func (s *MockArvadosServerSuite) Test_APIErrorUpdatingContainerState(c *C) { func (s *MockArvadosServerSuite) Test_ContainerStillInRunningAfterRun(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2"}]}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2","State":"Queued"}]}`)} apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx2"] = arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx2", "state":"Running", "priority":1}`)} testWithServerStub(c, apiStubResponses, "echo", - "After crunch-run process termination, the state is still 'Running' for zzzzz-dz642-xxxxxxxxxxxxxx2") + `After echo process termination, container state for Running is "zzzzz-dz642-xxxxxxxxxxxxxx2". Updating it to "Cancelled"`) } func (s *MockArvadosServerSuite) Test_ErrorRunningContainer(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/containers"] = - arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3"}]}`)} + arvadostest.StubResponse{200, string(`{"items_available":1, "items":[{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3","State":"Queued"}]}`)} + apiStubResponses["/arvados/v1/containers/zzzzz-dz642-xxxxxxxxxxxxxx3"] = arvadostest.StubResponse{200, string(`{"uuid":"zzzzz-dz642-xxxxxxxxxxxxxx3", "state":"Running", "priority":1}`)} - testWithServerStub(c, apiStubResponses, "nosuchcommand", "Error starting crunch-run for zzzzz-dz642-xxxxxxxxxxxxxx3") + testWithServerStub(c, apiStubResponses, "nosuchcommand", "Error starting nosuchcommand for zzzzz-dz642-xxxxxxxxxxxxxx3") } func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { + apiStubResponses["/arvados/v1/api_client_authorizations/current"] = + arvadostest.StubResponse{200, string(`{"uuid": "abc", "api_token": "xyz"}`)} + apiStub := arvadostest.ServerStub{apiStubResponses} api := httptest.NewServer(&apiStub) defer api.Close() - arv = arvadosclient.ArvadosClient{ + arv := arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", @@ -139,15 +158,38 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon } buf := bytes.NewBuffer(nil) - log.SetOutput(buf) + log.SetOutput(io.MultiWriter(buf, os.Stderr)) defer log.SetOutput(os.Stderr) + *crunchRunCommand = crunchCmd + + doneProcessing := make(chan struct{}) + dispatcher := dispatch.Dispatcher{ + Arv: arv, + PollInterval: time.Duration(1) * time.Second, + RunContainer: func(dispatcher *dispatch.Dispatcher, + container dispatch.Container, + status chan dispatch.Container) { + run(dispatcher, container, status) + doneProcessing <- struct{}{} + }, + DoneProcessing: doneProcessing} + + startCmd = func(container dispatch.Container, cmd *exec.Cmd) error { + dispatcher.UpdateState(container.UUID, "Running") + dispatcher.UpdateState(container.UUID, "Complete") + return cmd.Start() + } + go func() { - time.Sleep(2 * time.Second) - sigChan <- syscall.SIGTERM + for i := 0; i < 80 && !strings.Contains(buf.String(), expected); i++ { + time.Sleep(100 * time.Millisecond) + } + dispatcher.DoneProcessing <- struct{}{} }() - runQueuedContainers(time.Second, time.Second, crunchCmd) + err := dispatcher.RunDispatcher() + c.Assert(err, IsNil) // Wait for all running crunch jobs to complete / terminate waitGroup.Wait()