X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dc021c3b57dcdebe464c148d55f9990a74e8246b..1f92b08e21c19dfa0c6d2260fe2fd8b24529e9e3:/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index 3c14f7044c..fb433e65cd 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -2,12 +2,13 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package dispatchslurm import ( "bytes" "context" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -19,10 +20,13 @@ import ( "testing" "time" + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/lib/dispatchcloud" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/dispatch" "github.com/sirupsen/logrus" . "gopkg.in/check.v1" @@ -42,7 +46,8 @@ type IntegrationSuite struct { } func (s *IntegrationSuite) SetUpTest(c *C) { - arvadostest.StartAPI() + arvadostest.ResetEnv() + arvadostest.ResetDB(c) os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token) s.disp = Dispatcher{} s.disp.cluster = &arvados.Cluster{} @@ -52,7 +57,7 @@ func (s *IntegrationSuite) SetUpTest(c *C) { func (s *IntegrationSuite) TearDownTest(c *C) { arvadostest.ResetEnv() - arvadostest.StopAPI() + arvadostest.ResetDB(c) } type slurmFake struct { @@ -104,7 +109,7 @@ func (sf *slurmFake) Cancel(name string) error { func (s *IntegrationSuite) integrationTest(c *C, expectBatch [][]string, - runContainer func(*dispatch.Dispatcher, arvados.Container)) arvados.Container { + runContainer func(*dispatch.Dispatcher, arvados.Container)) (arvados.Container, error) { arvadostest.ResetEnv() arv, err := arvadosclient.MakeArvadosClient() @@ -123,18 +128,21 @@ func (s *IntegrationSuite) integrationTest(c *C, ctx, cancel := context.WithCancel(context.Background()) doneRun := make(chan struct{}) + doneDispatch := make(chan error) s.disp.Dispatcher = &dispatch.Dispatcher{ Arv: arv, PollPeriod: time.Second, - RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) { + RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) error { go func() { runContainer(disp, ctr) s.slurm.queue = "" doneRun <- struct{}{} }() - s.disp.runContainer(disp, ctr, status) + err := s.disp.runContainer(disp, ctr, status) cancel() + doneDispatch <- err + return nil }, } @@ -148,6 +156,7 @@ func (s *IntegrationSuite) integrationTest(c *C, err = s.disp.Dispatcher.Run(ctx) <-doneRun c.Assert(err, Equals, context.Canceled) + errDispatch := <-doneDispatch s.disp.sqCheck.Stop() @@ -162,12 +171,12 @@ func (s *IntegrationSuite) integrationTest(c *C, var container arvados.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) - return container + return container, errDispatch } func (s *IntegrationSuite) TestNormal(c *C) { s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"} - container := s.integrationTest(c, + container, _ := s.integrationTest(c, nil, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) @@ -181,7 +190,7 @@ func (s *IntegrationSuite) TestCancel(c *C) { s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"} readyToCancel := make(chan bool) s.slurm.onCancel = func() { <-readyToCancel } - container := s.integrationTest(c, + container, _ := s.integrationTest(c, nil, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) @@ -199,7 +208,7 @@ func (s *IntegrationSuite) TestCancel(c *C) { } func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { - container := s.integrationTest(c, + container, _ := s.integrationTest(c, [][]string{{ fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"), fmt.Sprintf("--nice=%d", 10000), @@ -218,24 +227,14 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { func (s *IntegrationSuite) TestSbatchFail(c *C) { s.slurm = slurmFake{errBatch: errors.New("something terrible happened")} - container := s.integrationTest(c, + container, err := s.integrationTest(c, [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--nice=10000", "--no-requeue", "--mem=11445", "--cpus-per-task=4", "--tmp=45777"}}, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) dispatcher.UpdateState(container.UUID, dispatch.Complete) }) c.Check(container.State, Equals, arvados.ContainerStateComplete) - - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, IsNil) - - var ll arvados.LogList - err = arv.List("logs", arvadosclient.Dict{"filters": [][]string{ - {"object_uuid", "=", container.UUID}, - {"event_type", "=", "dispatch"}, - }}, &ll) - c.Assert(err, IsNil) - c.Assert(len(ll.Items), Equals, 1) + c.Check(err, ErrorMatches, `something terrible happened`) } type StubbedSuite struct { @@ -280,7 +279,7 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva dispatcher := dispatch.Dispatcher{ Arv: arv, PollPeriod: time.Second, - RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) { + RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) error { go func() { time.Sleep(time.Second) disp.UpdateState(ctr.UUID, dispatch.Running) @@ -288,6 +287,7 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva }() s.disp.runContainer(disp, ctr, status) cancel() + return nil }, } @@ -391,6 +391,7 @@ func (s *StubbedSuite) TestSbatchPartition(c *C) { } func (s *StubbedSuite) TestLoadLegacyConfig(c *C) { + log := ctxlog.TestLogger(c) content := []byte(` Client: APIHost: example.com @@ -406,36 +407,42 @@ ReserveExtraRAM: 12345 MinRetryPeriod: 13s BatchSize: 99 `) - tmpfile, err := ioutil.TempFile("", "example") - if err != nil { - c.Error(err) - } - - defer os.Remove(tmpfile.Name()) // clean up - - if _, err := tmpfile.Write(content); err != nil { - c.Error(err) - } - if err := tmpfile.Close(); err != nil { - c.Error(err) + tmpfile := c.MkDir() + "/config.yml" + err := ioutil.WriteFile(tmpfile, content, 0777) + c.Assert(err, IsNil) - } os.Setenv("ARVADOS_KEEP_SERVICES", "") - err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()}) - c.Check(err, IsNil) - c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"}) - c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg") - c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) - c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) - c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42)) - c.Check(s.disp.cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run") - c.Check(s.disp.cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"}) - c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345)) - c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second)) - c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99) - c.Check(s.disp.cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{ + flags := flag.NewFlagSet("", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + loader := config.NewLoader(&bytes.Buffer{}, log) + loader.SetupFlags(flags) + args := loader.MungeLegacyConfigArgs(log, []string{"-config", tmpfile}, "-legacy-"+string(arvados.ServiceNameDispatchSLURM)+"-config") + ok, _ := cmd.ParseFlags(flags, "crunch-dispatch-slurm", args, "", os.Stderr) + c.Check(ok, Equals, true) + cfg, err := loader.Load() + c.Assert(err, IsNil) + cluster, err := cfg.GetCluster("") + c.Assert(err, IsNil) + + c.Check(cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"}) + c.Check(cluster.SystemRootToken, Equals, "abcdefg") + c.Check(cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) + c.Check(cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) + c.Check(cluster.Containers.SLURM.PrioritySpread, Equals, int64(42)) + c.Check(cluster.Containers.CrunchRunCommand, Equals, "x-crunch-run") + c.Check(cluster.Containers.CrunchRunArgumentsList, DeepEquals, []string{"--cgroup-parent-subsystem=memory"}) + c.Check(cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345)) + c.Check(cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second)) + c.Check(cluster.API.MaxItemsPerResponse, Equals, 99) + c.Check(cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{ "ARVADOS_KEEP_SERVICES": "https://example.com/keep1 https://example.com/keep2", }) + + // Ensure configure() copies SbatchEnvironmentVariables into + // the current process's environment (that's how they end up + // getting passed to sbatch). + s.disp.cluster = cluster + s.disp.configure() c.Check(os.Getenv("ARVADOS_KEEP_SERVICES"), Equals, "https://example.com/keep1 https://example.com/keep2") }