X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6d1ebf894a02151f751686003dc67ed4788d6c10..f739d736bbb60a8463f04f5d56c18d09157d820e:/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index 4211026a67..b76ece314d 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/http/httptest" "os" @@ -25,6 +24,7 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/dispatch" + "github.com/Sirupsen/logrus" . "gopkg.in/check.v1" ) @@ -55,11 +55,12 @@ func (s *IntegrationSuite) TearDownTest(c *C) { } type slurmFake struct { - didBatch [][]string - didCancel []string - didRelease []string - didRenice [][]string - queue string + didBatch [][]string + didCancel []string + didRelease []string + didRenice [][]string + queue string + rejectNice10K bool // If non-nil, run this func during the 2nd+ call to Cancel() onCancel func() // Error returned by Batch() @@ -82,6 +83,9 @@ func (sf *slurmFake) Release(name string) error { func (sf *slurmFake) Renice(name string, nice int64) error { sf.didRenice = append(sf.didRenice, []string{name, fmt.Sprintf("%d", nice)}) + if sf.rejectNice10K && nice > 10000 { + return errors.New("scontrol: error: Invalid nice value, must be between -10000 and 10000") + } return nil } @@ -112,7 +116,7 @@ func (s *IntegrationSuite) integrationTest(c *C, var containers arvados.ContainerList err = arv.List("containers", params, &containers) c.Check(err, IsNil) - c.Check(len(containers.Items), Equals, 1) + c.Assert(len(containers.Items), Equals, 1) s.disp.CrunchRunCommand = []string{"echo"} @@ -134,7 +138,11 @@ func (s *IntegrationSuite) integrationTest(c *C, } s.disp.slurm = &s.slurm - s.disp.sqCheck = &SqueueChecker{Period: 500 * time.Millisecond, Slurm: s.disp.slurm} + s.disp.sqCheck = &SqueueChecker{ + Logger: logrus.StandardLogger(), + Period: 500 * time.Millisecond, + Slurm: s.disp.slurm, + } err = s.disp.Dispatcher.Run(ctx) <-doneRun @@ -193,10 +201,11 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { container := s.integrationTest(c, [][]string{{ fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"), + fmt.Sprintf("--nice=%d", 10000), fmt.Sprintf("--mem=%d", 11445), fmt.Sprintf("--cpus-per-task=%d", 4), fmt.Sprintf("--tmp=%d", 45777), - fmt.Sprintf("--nice=%d", 10000)}}, + }}, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) time.Sleep(3 * time.Second) @@ -208,7 +217,7 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { func (s *IntegrationSuite) TestSbatchFail(c *C) { s.slurm = slurmFake{errBatch: errors.New("something terrible happened")} container := s.integrationTest(c, - [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--mem=11445", "--cpus-per-task=4", "--tmp=45777", "--nice=10000"}}, + [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--nice=10000", "--mem=11445", "--cpus-per-task=4", "--tmp=45777"}}, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) dispatcher.UpdateState(container.UUID, dispatch.Complete) @@ -223,6 +232,7 @@ func (s *IntegrationSuite) TestSbatchFail(c *C) { {"object_uuid", "=", container.UUID}, {"event_type", "=", "dispatch"}, }}, &ll) + c.Assert(err, IsNil) c.Assert(len(ll.Items), Equals, 1) } @@ -240,7 +250,7 @@ func (s *StubbedSuite) TestAPIErrorGettingContainers(c *C) { apiStubResponses["/arvados/v1/api_client_authorizations/current"] = arvadostest.StubResponse{200, `{"uuid":"` + arvadostest.Dispatch1AuthUUID + `"}`} apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)} - s.testWithServerStub(c, apiStubResponses, "echo", "Error getting list of containers") + s.testWithServerStub(c, apiStubResponses, "echo", "error getting count of containers") } func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) { @@ -258,8 +268,8 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva } buf := bytes.NewBuffer(nil) - log.SetOutput(io.MultiWriter(buf, os.Stderr)) - defer log.SetOutput(os.Stderr) + logrus.SetOutput(io.MultiWriter(buf, os.Stderr)) + defer logrus.SetOutput(os.Stderr) s.disp.CrunchRunCommand = []string{crunchCmd} @@ -352,7 +362,7 @@ func (s *StubbedSuite) TestSbatchArgs(c *C) { s.disp.SbatchArguments = defaults args, err := s.disp.sbatchArgs(container) - c.Check(args, DeepEquals, append(defaults, "--job-name=123", "--mem=239", "--cpus-per-task=2", "--tmp=0", "--nice=10000")) + c.Check(args, DeepEquals, append(defaults, "--job-name=123", "--nice=10000", "--mem=239", "--cpus-per-task=2", "--tmp=0")) c.Check(err, IsNil) } } @@ -365,40 +375,42 @@ func (s *StubbedSuite) TestSbatchInstanceTypeConstraint(c *C) { } for _, trial := range []struct { - types []arvados.InstanceType + types map[string]arvados.InstanceType sbatchArgs []string err error }{ // Choose node type => use --constraint arg { - types: []arvados.InstanceType{ - {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1}, - {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2}, - {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4}, - {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8}, + types: map[string]arvados.InstanceType{ + "a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1}, + "a1.small": {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2}, + "a1.medium": {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4}, + "a1.large": {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8}, }, sbatchArgs: []string{"--constraint=instancetype=a1.medium"}, }, // No node types configured => no slurm constraint { types: nil, - sbatchArgs: nil, + sbatchArgs: []string{"--mem=239", "--cpus-per-task=2", "--tmp=0"}, }, // No node type is big enough => error { - types: []arvados.InstanceType{ - {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1}, + types: map[string]arvados.InstanceType{ + "a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1}, }, - err: dispatchcloud.ErrConstraintsNotSatisfiable, + err: dispatchcloud.ConstraintsNotSatisfiableError{}, }, } { c.Logf("%#v", trial) s.disp.cluster = &arvados.Cluster{InstanceTypes: trial.types} args, err := s.disp.sbatchArgs(container) - c.Check(err, Equals, trial.err) + c.Check(err == nil, Equals, trial.err == nil) if trial.err == nil { - c.Check(args, DeepEquals, append([]string{"--job-name=123", "--mem=239", "--cpus-per-task=2", "--tmp=0", "--nice=10000"}, trial.sbatchArgs...)) + c.Check(args, DeepEquals, append([]string{"--job-name=123", "--nice=10000"}, trial.sbatchArgs...)) + } else { + c.Check(len(err.(dispatchcloud.ConstraintsNotSatisfiableError).AvailableTypes), Equals, len(trial.types)) } } } @@ -413,7 +425,8 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) { args, err := s.disp.sbatchArgs(container) c.Check(args, DeepEquals, []string{ - "--job-name=123", "--mem=239", "--cpus-per-task=1", "--tmp=0", "--nice=10000", + "--job-name=123", "--nice=10000", + "--mem=239", "--cpus-per-task=1", "--tmp=0", "--partition=blurb,b2", }) c.Check(err, IsNil)