X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f1adedeba07502273d39084d4ff3645b30067579..a74c81b035c67d299e2a7298f8db3d368a578510:/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index 82675b26b2..d8b94f98d2 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -1,19 +1,19 @@ package main import ( + "bytes" + "fmt" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" - - "fmt" - "io/ioutil" + "git.curoverse.com/arvados.git/sdk/go/dispatch" + "io" "log" "net/http" "net/http/httptest" "os" "os/exec" - "strconv" "strings" - "syscall" "testing" "time" @@ -35,92 +35,138 @@ var initialArgs []string func (s *TestSuite) SetUpSuite(c *C) { initialArgs = os.Args - arvadostest.StartAPI() } func (s *TestSuite) TearDownSuite(c *C) { - arvadostest.StopAPI() } func (s *TestSuite) SetUpTest(c *C) { args := []string{"crunch-dispatch-slurm"} os.Args = args - var err error - arv, err = arvadosclient.MakeArvadosClient() - if err != nil { - c.Fatalf("Error making arvados client: %s", err) - } + arvadostest.StartAPI() + os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token) } func (s *TestSuite) TearDownTest(c *C) { - arvadostest.ResetEnv() os.Args = initialArgs + arvadostest.StopAPI() } func (s *MockArvadosServerSuite) TearDownTest(c *C) { arvadostest.ResetEnv() } -func (s *TestSuite) Test_doMain(c *C) { - args := []string{"-poll-interval", "2", "-container-priority-poll-interval", "1", "-crunch-run-command", "echo"} - os.Args = append(os.Args, args...) +func (s *TestSuite) TestIntegrationNormal(c *C) { + container := s.integrationTest(c, func() *exec.Cmd { return exec.Command("echo", "zzzzz-dz642-queuedcontainer") }, + []string(nil), + func(dispatcher *dispatch.Dispatcher, container arvados.Container) { + dispatcher.UpdateState(container.UUID, dispatch.Running) + time.Sleep(3 * time.Second) + dispatcher.UpdateState(container.UUID, dispatch.Complete) + }) + c.Check(container.State, Equals, arvados.ContainerStateComplete) +} - var sbatchCmdLine []string - var striggerCmdLine []string +func (s *TestSuite) TestIntegrationCancel(c *C) { // Override sbatchCmd - defer func(orig func(Container) *exec.Cmd) { - sbatchCmd = orig - }(sbatchCmd) - sbatchCmd = func(container Container) *exec.Cmd { - sbatchCmdLine = sbatchFunc(container).Args - return exec.Command("echo", container.UUID) + var scancelCmdLine []string + defer func(orig func(arvados.Container) *exec.Cmd) { + scancelCmd = orig + }(scancelCmd) + scancelCmd = func(container arvados.Container) *exec.Cmd { + scancelCmdLine = scancelFunc(container).Args + return exec.Command("echo") } - // Override striggerCmd - defer func(orig func(jobid, containerUUID, finishCommand, - apiHost, apiToken, apiInsecure string) *exec.Cmd) { - striggerCmd = orig - }(striggerCmd) - striggerCmd = func(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) *exec.Cmd { - striggerCmdLine = striggerFunc(jobid, containerUUID, finishCommand, - apiHost, apiToken, apiInsecure).Args - go func() { - time.Sleep(5 * time.Second) - arv.Update("containers", containerUUID, + container := s.integrationTest(c, func() *exec.Cmd { return exec.Command("echo", "zzzzz-dz642-queuedcontainer") }, + []string(nil), + func(dispatcher *dispatch.Dispatcher, container arvados.Container) { + dispatcher.UpdateState(container.UUID, dispatch.Running) + time.Sleep(1 * time.Second) + dispatcher.Arv.Update("containers", container.UUID, arvadosclient.Dict{ - "container": arvadosclient.Dict{"state": "Complete"}}, + "container": arvadosclient.Dict{"priority": 0}}, nil) - }() - return exec.Command("echo", "strigger") + }) + c.Check(container.State, Equals, arvados.ContainerStateCancelled) + c.Check(scancelCmdLine, DeepEquals, []string{"scancel", "--name=zzzzz-dz642-queuedcontainer"}) +} + +func (s *TestSuite) TestIntegrationMissingFromSqueue(c *C) { + container := s.integrationTest(c, func() *exec.Cmd { return exec.Command("echo") }, []string{"sbatch", "--share", "--parsable", + fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"), + fmt.Sprintf("--mem-per-cpu=%d", 2862), + fmt.Sprintf("--cpus-per-task=%d", 4), + fmt.Sprintf("--priority=%d", 1)}, + func(dispatcher *dispatch.Dispatcher, container arvados.Container) { + dispatcher.UpdateState(container.UUID, dispatch.Running) + time.Sleep(3 * time.Second) + dispatcher.UpdateState(container.UUID, dispatch.Complete) + }) + c.Check(container.State, Equals, arvados.ContainerStateCancelled) +} + +func (s *TestSuite) integrationTest(c *C, + newSqueueCmd func() *exec.Cmd, + sbatchCmdComps []string, + runContainer func(*dispatch.Dispatcher, arvados.Container)) arvados.Container { + arvadostest.ResetEnv() + + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, IsNil) + + var sbatchCmdLine []string + + // Override sbatchCmd + defer func(orig func(arvados.Container) *exec.Cmd) { + sbatchCmd = orig + }(sbatchCmd) + sbatchCmd = func(container arvados.Container) *exec.Cmd { + sbatchCmdLine = sbatchFunc(container).Args + return exec.Command("sh") } - go func() { - time.Sleep(8 * time.Second) - sigChan <- syscall.SIGINT - }() + // Override squeueCmd + defer func(orig func() *exec.Cmd) { + squeueCmd = orig + }(squeueCmd) + squeueCmd = newSqueueCmd // There should be no queued containers now params := arvadosclient.Dict{ - "filters": [][]string{[]string{"state", "=", "Queued"}}, + "filters": [][]string{{"state", "=", "Queued"}}, } - var containers ContainerList - err := arv.List("containers", params, &containers) + var containers arvados.ContainerList + err = arv.List("containers", params, &containers) c.Check(err, IsNil) c.Check(len(containers.Items), Equals, 1) - err = doMain() - c.Check(err, IsNil) + echo := "echo" + crunchRunCommand = &echo - sbatchCmdComps := []string{"sbatch", "--share", "--parsable", - fmt.Sprintf("--job-name=%s", containers.Items[0].UUID), - fmt.Sprintf("--mem=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["ram"])), - fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["vcpus"]))} - c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps) + doneProcessing := make(chan struct{}) + dispatcher := dispatch.Dispatcher{ + Arv: arv, + PollInterval: time.Duration(1) * time.Second, + RunContainer: func(dispatcher *dispatch.Dispatcher, + container arvados.Container, + status chan arvados.Container) { + go runContainer(dispatcher, container) + run(dispatcher, container, status) + doneProcessing <- struct{}{} + }, + DoneProcessing: doneProcessing} + + squeueUpdater.StartMonitor(time.Duration(500) * time.Millisecond) - c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini", - "--program=/usr/bin/crunch-finish-slurm.sh " + os.Getenv("ARVADOS_API_HOST") + " 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h 1 zzzzz-dz642-queuedcontainer"}) + err = dispatcher.RunDispatcher() + c.Assert(err, IsNil) + + squeueUpdater.Done() + + c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps) // There should be no queued containers now err = arv.List("containers", params, &containers) @@ -128,17 +174,18 @@ func (s *TestSuite) Test_doMain(c *C) { c.Check(len(containers.Items), Equals, 0) // Previously "Queued" container should now be in "Complete" state - var container Container + var container arvados.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) - c.Check(container.State, Equals, "Complete") + return container } func (s *MockArvadosServerSuite) Test_APIErrorGettingContainers(c *C) { apiStubResponses := make(map[string]arvadostest.StubResponse) + apiStubResponses["/arvados/v1/api_client_authorizations/current"] = arvadostest.StubResponse{200, `{"uuid":"` + arvadostest.Dispatch1AuthUUID + `"}`} apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)} - testWithServerStub(c, apiStubResponses, "echo", "Error getting list of queued containers") + testWithServerStub(c, apiStubResponses, "echo", "Error getting list of containers") } func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { @@ -147,7 +194,7 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon api := httptest.NewServer(&apiStub) defer api.Close() - arv = arvadosclient.ArvadosClient{ + arv := arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", @@ -155,18 +202,38 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon Retries: 0, } - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - defer os.Remove(tempfile.Name()) - log.SetOutput(tempfile) + buf := bytes.NewBuffer(nil) + log.SetOutput(io.MultiWriter(buf, os.Stderr)) + defer log.SetOutput(os.Stderr) + + crunchRunCommand = &crunchCmd + + doneProcessing := make(chan struct{}) + dispatcher := dispatch.Dispatcher{ + Arv: arv, + PollInterval: time.Duration(1) * time.Second, + RunContainer: func(dispatcher *dispatch.Dispatcher, + container arvados.Container, + status chan arvados.Container) { + go func() { + time.Sleep(1 * time.Second) + dispatcher.UpdateState(container.UUID, dispatch.Running) + dispatcher.UpdateState(container.UUID, dispatch.Complete) + }() + run(dispatcher, container, status) + doneProcessing <- struct{}{} + }, + DoneProcessing: doneProcessing} go func() { - time.Sleep(2 * time.Second) - sigChan <- syscall.SIGTERM + for i := 0; i < 80 && !strings.Contains(buf.String(), expected); i++ { + time.Sleep(100 * time.Millisecond) + } + dispatcher.DoneProcessing <- struct{}{} }() - runQueuedContainers(2, 1, crunchCmd, crunchCmd) + err := dispatcher.RunDispatcher() + c.Assert(err, IsNil) - buf, _ := ioutil.ReadFile(tempfile.Name()) - c.Check(strings.Contains(string(buf), expected), Equals, true) + c.Check(buf.String(), Matches, `(?ms).*`+expected+`.*`) }