Merge branch '8784-dir-listings'

[arvados.git] / services / crunch-dispatch-slurm / crunch-dispatch-slurm_test.go

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index e6abb1650ec1007d6f44906d7fdeba7c977318f9..5ab0e4fcb3092aa74743f58bdec05bc104359eb1 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
@@ -52,6 +56,7 @@ func (s *TestSuite) SetUpTest(c *C) {
 
 func (s *TestSuite) TearDownTest(c *C) {
        os.Args = initialArgs
+       arvadostest.ResetEnv()
        arvadostest.StopAPI()
 }
 
@@ -69,6 +74,8 @@ func (s *TestSuite) TestIntegrationNormal(c *C) {
                                return exec.Command("echo", "zzzzz-dz642-queuedcontainer")
                        }
                },
+               nil,
+               nil,
                []string(nil),
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
                        dispatcher.UpdateState(container.UUID, dispatch.Running)
@@ -82,19 +89,7 @@ func (s *TestSuite) TestIntegrationNormal(c *C) {
 func (s *TestSuite) TestIntegrationCancel(c *C) {
        var cmd *exec.Cmd
        var scancelCmdLine []string
-       defer func(orig func(arvados.Container) *exec.Cmd) {
-               scancelCmd = orig
-       }(scancelCmd)
        attempt := 0
-       scancelCmd = func(container arvados.Container) *exec.Cmd {
-               if attempt++; attempt == 1 {
-                       return exec.Command("false")
-               } else {
-                       scancelCmdLine = scancelFunc(container).Args
-                       cmd = exec.Command("echo")
-                       return cmd
-               }
-       }
 
        container := s.integrationTest(c,
                func() *exec.Cmd {
@@ -104,6 +99,16 @@ func (s *TestSuite) TestIntegrationCancel(c *C) {
                                return exec.Command("echo", "zzzzz-dz642-queuedcontainer")
                        }
                },
+               func(container arvados.Container) *exec.Cmd {
+                       if attempt++; attempt == 1 {
+                               return exec.Command("false")
+                       } else {
+                               scancelCmdLine = scancelFunc(container).Args
+                               cmd = exec.Command("echo")
+                               return cmd
+                       }
+               },
+               nil,
                []string(nil),
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
                        dispatcher.UpdateState(container.UUID, dispatch.Running)
@@ -118,10 +123,15 @@ func (s *TestSuite) TestIntegrationCancel(c *C) {
 }
 
 func (s *TestSuite) TestIntegrationMissingFromSqueue(c *C) {
-       container := s.integrationTest(c, func() *exec.Cmd { return exec.Command("echo") }, []string{"sbatch", "--share",
-               fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"),
-               fmt.Sprintf("--mem-per-cpu=%d", 2862),
-               fmt.Sprintf("--cpus-per-task=%d", 4)},
+       container := s.integrationTest(c,
+               func() *exec.Cmd { return exec.Command("echo") },
+               nil,
+               nil,
+               []string{"sbatch",
+                       fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"),
+                       fmt.Sprintf("--mem=%d", 11445),
+                       fmt.Sprintf("--cpus-per-task=%d", 4),
+                       fmt.Sprintf("--tmp=%d", 45777)},
                func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
                        dispatcher.UpdateState(container.UUID, dispatch.Running)
                        time.Sleep(3 * time.Second)
@@ -130,8 +140,35 @@ func (s *TestSuite) TestIntegrationMissingFromSqueue(c *C) {
        c.Check(container.State, Equals, arvados.ContainerStateCancelled)
 }
 
+func (s *TestSuite) TestSbatchFail(c *C) {
+       container := s.integrationTest(c,
+               func() *exec.Cmd { return exec.Command("echo") },
+               nil,
+               func(container arvados.Container) *exec.Cmd {
+                       return exec.Command("false")
+               },
+               []string(nil),
+               func(dispatcher *dispatch.Dispatcher, container arvados.Container) {
+                       dispatcher.UpdateState(container.UUID, dispatch.Running)
+                       dispatcher.UpdateState(container.UUID, dispatch.Complete)
+               })
+       c.Check(container.State, Equals, arvados.ContainerStateComplete)
+
+       arv, err := arvadosclient.MakeArvadosClient()
+       c.Assert(err, IsNil)
+
+       var ll arvados.LogList
+       err = arv.List("logs", arvadosclient.Dict{"filters": [][]string{
+               []string{"object_uuid", "=", container.UUID},
+               []string{"event_type", "=", "dispatch"},
+       }}, &ll)
+       c.Assert(len(ll.Items), Equals, 1)
+}
+
 func (s *TestSuite) integrationTest(c *C,
        newSqueueCmd func() *exec.Cmd,
+       newScancelCmd func(arvados.Container) *exec.Cmd,
+       newSbatchCmd func(arvados.Container) *exec.Cmd,
        sbatchCmdComps []string,
        runContainer func(*dispatch.Dispatcher, arvados.Container)) arvados.Container {
        arvadostest.ResetEnv()
@@ -145,9 +182,14 @@ func (s *TestSuite) integrationTest(c *C,
        defer func(orig func(arvados.Container) *exec.Cmd) {
                sbatchCmd = orig
        }(sbatchCmd)
-       sbatchCmd = func(container arvados.Container) *exec.Cmd {
-               sbatchCmdLine = sbatchFunc(container).Args
-               return exec.Command("sh")
+
+       if newSbatchCmd != nil {
+               sbatchCmd = newSbatchCmd
+       } else {
+               sbatchCmd = func(container arvados.Container) *exec.Cmd {
+                       sbatchCmdLine = sbatchFunc(container).Args
+                       return exec.Command("sh")
+               }
        }
 
        // Override squeueCmd
@@ -156,6 +198,12 @@ func (s *TestSuite) integrationTest(c *C,
        }(squeueCmd)
        squeueCmd = newSqueueCmd
 
+       // Override scancel
+       defer func(orig func(arvados.Container) *exec.Cmd) {
+               scancelCmd = orig
+       }(scancelCmd)
+       scancelCmd = newScancelCmd
+
        // There should be one queued container
        params := arvadosclient.Dict{
                "filters": [][]string{{"state", "=", "Queued"}},
@@ -168,11 +216,16 @@ func (s *TestSuite) integrationTest(c *C,
        theConfig.CrunchRunCommand = []string{"echo"}
 
        ctx, cancel := context.WithCancel(context.Background())
+       doneRun := make(chan struct{})
+
        dispatcher := dispatch.Dispatcher{
                Arv:        arv,
                PollPeriod: time.Duration(1) * time.Second,
                RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) {
-                       go runContainer(disp, ctr)
+                       go func() {
+                               runContainer(disp, ctr)
+                               doneRun <- struct{}{}
+                       }()
                        run(disp, ctr, status)
                        cancel()
                },
@@ -181,6 +234,7 @@ func (s *TestSuite) integrationTest(c *C,
        sqCheck = &SqueueChecker{Period: 500 * time.Millisecond}
 
        err = dispatcher.Run(ctx)
+       <-doneRun
        c.Assert(err, Equals, context.Canceled)
 
        sqCheck.Stop()
@@ -327,9 +381,9 @@ func testSbatchFuncWithArgs(c *C, args []string) {
        sbatchCmd := sbatchFunc(container)
 
        var expected []string
-       expected = append(expected, "sbatch", "--share")
+       expected = append(expected, "sbatch")
        expected = append(expected, theConfig.SbatchArguments...)
-       expected = append(expected, "--job-name=123", "--mem-per-cpu=120", "--cpus-per-task=2")
+       expected = append(expected, "--job-name=123", "--mem=239", "--cpus-per-task=2", "--tmp=0")
 
        c.Check(sbatchCmd.Args, DeepEquals, expected)
 }
@@ -340,8 +394,8 @@ func (s *MockArvadosServerSuite) TestSbatchPartition(c *C) {
        sbatchCmd := sbatchFunc(container)
 
        var expected []string
-       expected = append(expected, "sbatch", "--share")
-       expected = append(expected, "--job-name=123", "--mem-per-cpu=239", "--cpus-per-task=1", "--partition=blurb,b2")
+       expected = append(expected, "sbatch")
+       expected = append(expected, "--job-name=123", "--mem=239", "--cpus-per-task=1", "--tmp=0", "--partition=blurb,b2")
 
        c.Check(sbatchCmd.Args, DeepEquals, expected)
 }