X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a93774da6449567a2759672cf329be6ec96f91e3..6d8326a0f1521b0d7c49cd90dc775d409906bc49:/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index 6007c6d4a8..cf83257dad 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "net/http/httptest" "os" @@ -20,11 +19,11 @@ import ( "testing" "time" - "git.curoverse.com/arvados.git/lib/dispatchcloud" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/dispatch" + "git.arvados.org/arvados.git/lib/dispatchcloud" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/dispatch" "github.com/sirupsen/logrus" . "gopkg.in/check.v1" ) @@ -43,7 +42,8 @@ type IntegrationSuite struct { } func (s *IntegrationSuite) SetUpTest(c *C) { - arvadostest.StartAPI() + arvadostest.ResetEnv() + arvadostest.ResetDB(c) os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token) s.disp = Dispatcher{} s.disp.cluster = &arvados.Cluster{} @@ -53,7 +53,7 @@ func (s *IntegrationSuite) SetUpTest(c *C) { func (s *IntegrationSuite) TearDownTest(c *C) { arvadostest.ResetEnv() - arvadostest.StopAPI() + arvadostest.ResetDB(c) } type slurmFake struct { @@ -105,7 +105,7 @@ func (sf *slurmFake) Cancel(name string) error { func (s *IntegrationSuite) integrationTest(c *C, expectBatch [][]string, - runContainer func(*dispatch.Dispatcher, arvados.Container)) arvados.Container { + runContainer func(*dispatch.Dispatcher, arvados.Container)) (arvados.Container, error) { arvadostest.ResetEnv() arv, err := arvadosclient.MakeArvadosClient() @@ -124,18 +124,21 @@ func (s *IntegrationSuite) integrationTest(c *C, ctx, cancel := context.WithCancel(context.Background()) doneRun := make(chan struct{}) + doneDispatch := make(chan error) s.disp.Dispatcher = &dispatch.Dispatcher{ Arv: arv, PollPeriod: time.Second, - RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) { + RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) error { go func() { runContainer(disp, ctr) s.slurm.queue = "" doneRun <- struct{}{} }() - s.disp.runContainer(disp, ctr, status) + err := s.disp.runContainer(disp, ctr, status) cancel() + doneDispatch <- err + return nil }, } @@ -149,6 +152,7 @@ func (s *IntegrationSuite) integrationTest(c *C, err = s.disp.Dispatcher.Run(ctx) <-doneRun c.Assert(err, Equals, context.Canceled) + errDispatch := <-doneDispatch s.disp.sqCheck.Stop() @@ -163,12 +167,12 @@ func (s *IntegrationSuite) integrationTest(c *C, var container arvados.Container err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container) c.Check(err, IsNil) - return container + return container, errDispatch } func (s *IntegrationSuite) TestNormal(c *C) { s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"} - container := s.integrationTest(c, + container, _ := s.integrationTest(c, nil, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) @@ -182,7 +186,7 @@ func (s *IntegrationSuite) TestCancel(c *C) { s.slurm = slurmFake{queue: "zzzzz-dz642-queuedcontainer 10000 100 PENDING Resources\n"} readyToCancel := make(chan bool) s.slurm.onCancel = func() { <-readyToCancel } - container := s.integrationTest(c, + container, _ := s.integrationTest(c, nil, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) @@ -200,7 +204,7 @@ func (s *IntegrationSuite) TestCancel(c *C) { } func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { - container := s.integrationTest(c, + container, _ := s.integrationTest(c, [][]string{{ fmt.Sprintf("--job-name=%s", "zzzzz-dz642-queuedcontainer"), fmt.Sprintf("--nice=%d", 10000), @@ -219,24 +223,14 @@ func (s *IntegrationSuite) TestMissingFromSqueue(c *C) { func (s *IntegrationSuite) TestSbatchFail(c *C) { s.slurm = slurmFake{errBatch: errors.New("something terrible happened")} - container := s.integrationTest(c, + container, err := s.integrationTest(c, [][]string{{"--job-name=zzzzz-dz642-queuedcontainer", "--nice=10000", "--no-requeue", "--mem=11445", "--cpus-per-task=4", "--tmp=45777"}}, func(dispatcher *dispatch.Dispatcher, container arvados.Container) { dispatcher.UpdateState(container.UUID, dispatch.Running) dispatcher.UpdateState(container.UUID, dispatch.Complete) }) c.Check(container.State, Equals, arvados.ContainerStateComplete) - - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, IsNil) - - var ll arvados.LogList - err = arv.List("logs", arvadosclient.Dict{"filters": [][]string{ - {"object_uuid", "=", container.UUID}, - {"event_type", "=", "dispatch"}, - }}, &ll) - c.Assert(err, IsNil) - c.Assert(len(ll.Items), Equals, 1) + c.Check(err, ErrorMatches, `something terrible happened`) } type StubbedSuite struct { @@ -281,7 +275,7 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva dispatcher := dispatch.Dispatcher{ Arv: arv, PollPeriod: time.Second, - RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) { + RunContainer: func(disp *dispatch.Dispatcher, ctr arvados.Container, status <-chan arvados.Container) error { go func() { time.Sleep(time.Second) disp.UpdateState(ctr.UUID, dispatch.Running) @@ -289,6 +283,7 @@ func (s *StubbedSuite) testWithServerStub(c *C, apiStubResponses map[string]arva }() s.disp.runContainer(disp, ctr, status) cancel() + return nil }, } @@ -396,6 +391,9 @@ func (s *StubbedSuite) TestLoadLegacyConfig(c *C) { Client: APIHost: example.com AuthToken: abcdefg + KeepServiceURIs: + - https://example.com/keep1 + - https://example.com/keep2 SbatchArguments: ["--foo", "bar"] PollPeriod: 12s PrioritySpread: 42 @@ -406,22 +404,23 @@ BatchSize: 99 `) tmpfile, err := ioutil.TempFile("", "example") if err != nil { - log.Fatal(err) + c.Error(err) } defer os.Remove(tmpfile.Name()) // clean up if _, err := tmpfile.Write(content); err != nil { - log.Fatal(err) + c.Error(err) } if err := tmpfile.Close(); err != nil { - log.Fatal(err) + c.Error(err) } + os.Setenv("ARVADOS_KEEP_SERVICES", "") err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()}) c.Check(err, IsNil) - c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"}) + c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"}) c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg") c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) @@ -431,4 +430,8 @@ BatchSize: 99 c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345)) c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second)) c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99) + c.Check(s.disp.cluster.Containers.SLURM.SbatchEnvironmentVariables, DeepEquals, map[string]string{ + "ARVADOS_KEEP_SERVICES": "https://example.com/keep1 https://example.com/keep2", + }) + c.Check(os.Getenv("ARVADOS_KEEP_SERVICES"), Equals, "https://example.com/keep1 https://example.com/keep2") }