refs #13667 "Merge branch '13667-add-version-file'".

[arvados.git] / services / crunch-dispatch-slurm / crunch-dispatch-slurm_test.go

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 371f7db31ec20ababf00100d54355a1052e78bc1..719ec98d27aa19d65eceb7d3db3a46f506aed2f0 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -55,10 +55,12 @@ func (s *IntegrationSuite) TearDownTest(c *C) {
 }
 
 type slurmFake struct {
-       didBatch  [][]string
-       didCancel []string
-       didRenice [][]string
-       queue     string
+       didBatch      [][]string
+       didCancel     []string
+       didRelease    []string
+       didRenice     [][]string
+       queue         string
+       rejectNice10K bool
        // If non-nil, run this func during the 2nd+ call to Cancel()
        onCancel func()
        // Error returned by Batch()
@@ -74,8 +76,16 @@ func (sf *slurmFake) QueueCommand(args []string) *exec.Cmd {
        return exec.Command("echo", sf.queue)
 }
 
+func (sf *slurmFake) Release(name string) error {
+       sf.didRelease = append(sf.didRelease, name)
+       return nil
+}
+
 func (sf *slurmFake) Renice(name string, nice int64) error {
        sf.didRenice = append(sf.didRenice, []string{name, fmt.Sprintf("%d", nice)})
+       if sf.rejectNice10K && nice > 10000 {
+               return errors.New("scontrol: error: Invalid nice value, must be between -10000 and 10000")
+       }
        return nil
 }
 
@@ -115,7 +125,7 @@ func (s *IntegrationSuite) integrationTest(c *C,
 
        s.disp.Dispatcher = &dispatch.Dispatcher{
                Arv:        arv,
-               PollPeriod: time.Duration(1) * time.Second,
+               PollPeriod: time.Second,
                RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) {
                        go func() {
                                runContainer(disp, ctr)
@@ -151,7 +161,7 @@ func (s *IntegrationSuite) integrationTest(c *C,
 }
 
 func (s *IntegrationSuite) TestNormal(c *C) {
-       s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100\n"}
+       s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"}
        container := s.integrationTest(c,
                nil,
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
@@ -163,7 +173,7 @@ func (s *IntegrationSuite) TestNormal(c *C) {
 }
 
 func (s *IntegrationSuite) TestCancel(c *C) {
-       s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100\n"}
+       s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"}
        readyToCancel := make(chan bool)
        s.slurm.onCancel = func() { <-readyToCancel }
        container := s.integrationTest(c,
@@ -187,10 +197,11 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) {
        container := s.integrationTest(c,
                [][]string{{
                        fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"),
+                       fmt.Sprintf("--nice=%d", 10000),
                        fmt.Sprintf("--mem=%d", 11445),
                        fmt.Sprintf("--cpus-per-task=%d", 4),
                        fmt.Sprintf("--tmp=%d", 45777),
-                       fmt.Sprintf("--nice=%d", 10000)}},
+               }},
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
                        dispatcher.UpdateState(container.UUID, dispatch.Running)
                        time.Sleep(3 * time.Second)
@@ -202,7 +213,7 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) {
 func (s *IntegrationSuite) TestSbatchFail(c *C) {
        s.slurm = slurmFake{errBatch: errors.New("something terrible happened")}
        container := s.integrationTest(c,
-               [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--mem=11445", "--cpus-per-task=4", "--tmp=45777", "--nice=10000"}},
+               [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--nice=10000", "--mem=11445", "--cpus-per-task=4", "--tmp=45777"}},
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
                        dispatcher.UpdateState(container.UUID, dispatch.Running)
                        dispatcher.UpdateState(container.UUID, dispatch.Complete)
@@ -217,6 +228,7 @@ func (s *IntegrationSuite) TestSbatchFail(c *C) {
                {"object_uuid", "=", container.UUID},
                {"event_type", "=", "dispatch"},
        }}, &ll)
+       c.Assert(err, IsNil)
        c.Assert(len(ll.Items), Equals, 1)
 }
 
@@ -260,10 +272,10 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva
        ctx, cancel := context.WithCancel(context.Background())
        dispatcher := dispatch.Dispatcher{
                Arv:        arv,
-               PollPeriod: time.Duration(1) * time.Second,
+               PollPeriod: time.Second,
                RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) {
                        go func() {
-                               time.Sleep(1 * time.Second)
+                               time.Sleep(time.Second)
                                disp.UpdateState(ctr.UUID, dispatch.Running)
                                disp.UpdateState(ctr.UUID, dispatch.Complete)
                        }()
@@ -346,7 +358,7 @@ func (s *StubbedSuite) TestSbatchArgs(c *C) {
                s.disp.SbatchArguments = defaults
 
                args, err := s.disp.sbatchArgs(container)
-               c.Check(args, DeepEquals, append(defaults, "--job-name=123", "--mem=239", "--cpus-per-task=2", "--tmp=0", "--nice=10000"))
+               c.Check(args, DeepEquals, append(defaults, "--job-name=123", "--nice=10000", "--mem=239", "--cpus-per-task=2", "--tmp=0"))
                c.Check(err, IsNil)
        }
 }
@@ -359,40 +371,42 @@ func (s *StubbedSuite) TestSbatchInstanceTypeConstraint(c *C) {
        }
 
        for _, trial := range []struct {
-               types      []arvados.InstanceType
+               types      map[string]arvados.InstanceType
                sbatchArgs []string
                err        error
        }{
                // Choose node type => use --constraint arg
                {
-                       types: []arvados.InstanceType{
-                               {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
-                               {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
-                               {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
-                               {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
+                       types: map[string]arvados.InstanceType{
+                               "a1.tiny":   {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+                               "a1.small":  {Name: "a1.small", Price: 0.04, RAM: 256000000, VCPUs: 2},
+                               "a1.medium": {Name: "a1.medium", Price: 0.08, RAM: 512000000, VCPUs: 4},
+                               "a1.large":  {Name: "a1.large", Price: 0.16, RAM: 1024000000, VCPUs: 8},
                        },
                        sbatchArgs: []string{"--constraint=instancetype=a1.medium"},
                },
                // No node types configured => no slurm constraint
                {
                        types:      nil,
-                       sbatchArgs: nil,
+                       sbatchArgs: []string{"--mem=239", "--cpus-per-task=2", "--tmp=0"},
                },
                // No node type is big enough => error
                {
-                       types: []arvados.InstanceType{
-                               {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
+                       types: map[string]arvados.InstanceType{
+                               "a1.tiny": {Name: "a1.tiny", Price: 0.02, RAM: 128000000, VCPUs: 1},
                        },
-                       err: dispatchcloud.ErrConstraintsNotSatisfiable,
+                       err: dispatchcloud.ConstraintsNotSatisfiableError{},
                },
        } {
                c.Logf("%#v", trial)
                s.disp.cluster = &arvados.Cluster{InstanceTypes: trial.types}
 
                args, err := s.disp.sbatchArgs(container)
-               c.Check(err, Equals, trial.err)
+               c.Check(err == nil, Equals, trial.err == nil)
                if trial.err == nil {
-                       c.Check(args, DeepEquals, append([]string{"--job-name=123", "--mem=239", "--cpus-per-task=2", "--tmp=0", "--nice=10000"}, trial.sbatchArgs...))
+                       c.Check(args, DeepEquals, append([]string{"--job-name=123", "--nice=10000"}, trial.sbatchArgs...))
+               } else {
+                       c.Check(len(err.(dispatchcloud.ConstraintsNotSatisfiableError).AvailableTypes), Equals, len(trial.types))
                }
        }
 }
@@ -407,7 +421,8 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) {
 
        args, err := s.disp.sbatchArgs(container)
        c.Check(args, DeepEquals, []string{
-               "--job-name=123", "--mem=239", "--cpus-per-task=1", "--tmp=0", "--nice=10000",
+               "--job-name=123", "--nice=10000",
+               "--mem=239", "--cpus-per-task=1", "--tmp=0",
                "--partition=blurb,b2",
        })
        c.Check(err, IsNil)