Merge branch '20978-instance-types'

[arvados.git] / lib / dispatchcloud / container / queue_test.go

diff --git a/lib/dispatchcloud/container/queue_test.go b/lib/dispatchcloud/container/queue_test.go
index 9d2f8309035c630ada692a83f068720568b34548..928c6dd8c87400403d8feaab6459b2ba0fff40f0 100644 (file)
--- a/lib/dispatchcloud/container/queue_test.go
+++ b/lib/dispatchcloud/container/queue_test.go
@@ -5,12 +5,14 @@
 package container
 
 import (
+       "errors"
+       "os"
        "sync"
        "testing"
        "time"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
        "github.com/sirupsen/logrus"
        check "gopkg.in/check.v1"
 )
@@ -22,11 +24,29 @@ func Test(t *testing.T) {
 
 var _ = check.Suite(&IntegrationSuite{})
 
+func logger() logrus.FieldLogger {
+       logger := logrus.StandardLogger()
+       if os.Getenv("ARVADOS_DEBUG") != "" {
+               logger.SetLevel(logrus.DebugLevel)
+       }
+       return logger
+}
+
 type IntegrationSuite struct{}
 
-func (*IntegrationSuite) TestControllerBackedQueue(c *check.C) {
+func (suite *IntegrationSuite) TearDownTest(c *check.C) {
+       err := arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil)
+       c.Check(err, check.IsNil)
+}
+
+func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
+       typeChooser := func(ctr *arvados.Container) ([]arvados.InstanceType, error) {
+               c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
+               return []arvados.InstanceType{{Name: "testType"}}, nil
+       }
+
        client := arvados.NewClientFromEnv()
-       cq := NewQueue(logrus.StandardLogger(), nil, testTypeChooser, client)
+       cq := NewQueue(logger(), nil, typeChooser, client)
 
        err := cq.Update()
        c.Check(err, check.IsNil)
@@ -42,19 +62,24 @@ func (*IntegrationSuite) TestControllerBackedQueue(c *check.C) {
        var wg sync.WaitGroup
        for uuid, ent := range ents {
                c.Check(ent.Container.UUID, check.Equals, uuid)
-               c.Check(ent.InstanceType.Name, check.Equals, "testType")
+               c.Check(ent.InstanceTypes, check.HasLen, 1)
+               c.Check(ent.InstanceTypes[0].Name, check.Equals, "testType")
                c.Check(ent.Container.State, check.Equals, arvados.ContainerStateQueued)
                c.Check(ent.Container.Priority > 0, check.Equals, true)
+               // Mounts should be deleted to avoid wasting memory
+               c.Check(ent.Container.Mounts, check.IsNil)
 
                ctr, ok := cq.Get(uuid)
                c.Check(ok, check.Equals, true)
                c.Check(ctr.UUID, check.Equals, uuid)
 
                wg.Add(1)
-               go func() {
+               go func(uuid string) {
                        defer wg.Done()
                        err := cq.Unlock(uuid)
                        c.Check(err, check.NotNil)
+                       c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued.*")
+
                        err = cq.Lock(uuid)
                        c.Check(err, check.IsNil)
                        ctr, ok := cq.Get(uuid)
@@ -62,6 +87,7 @@ func (*IntegrationSuite) TestControllerBackedQueue(c *check.C) {
                        c.Check(ctr.State, check.Equals, arvados.ContainerStateLocked)
                        err = cq.Lock(uuid)
                        c.Check(err, check.NotNil)
+
                        err = cq.Unlock(uuid)
                        c.Check(err, check.IsNil)
                        ctr, ok = cq.Get(uuid)
@@ -69,14 +95,73 @@ func (*IntegrationSuite) TestControllerBackedQueue(c *check.C) {
                        c.Check(ctr.State, check.Equals, arvados.ContainerStateQueued)
                        err = cq.Unlock(uuid)
                        c.Check(err, check.NotNil)
-               }()
+
+                       err = cq.Cancel(uuid)
+                       c.Check(err, check.IsNil)
+                       ctr, ok = cq.Get(uuid)
+                       c.Check(ok, check.Equals, true)
+                       c.Check(ctr.State, check.Equals, arvados.ContainerStateCancelled)
+                       err = cq.Lock(uuid)
+                       c.Check(err, check.NotNil)
+               }(uuid)
        }
        wg.Wait()
+}
 
-       err = cq.Cancel(arvadostest.CompletedContainerUUID)
-       c.Check(err, check.ErrorMatches, `.*State cannot change from Complete to Cancelled.*`)
+func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
+       errorTypeChooser := func(ctr *arvados.Container) ([]arvados.InstanceType, error) {
+               // Make sure the relevant container fields are
+               // actually populated.
+               c.Check(ctr.ContainerImage, check.Equals, "test")
+               c.Check(ctr.RuntimeConstraints.VCPUs, check.Equals, 4)
+               c.Check(ctr.RuntimeConstraints.RAM, check.Equals, int64(12000000000))
+               c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
+               c.Check(ctr.Mounts["/var/spool/cwl"].Capacity, check.Equals, int64(24000000000))
+               return nil, errors.New("no suitable instance type")
+       }
+
+       client := arvados.NewClientFromEnv()
+       cq := NewQueue(logger(), nil, errorTypeChooser, client)
+
+       ch := cq.Subscribe()
+       go func() {
+               defer cq.Unsubscribe(ch)
+               for range ch {
+                       // Container should never be added to
+                       // queue. Note that polling the queue this way
+                       // doesn't guarantee a bug (container being
+                       // incorrectly added to the queue) will cause
+                       // a test failure.
+                       _, ok := cq.Get(arvadostest.QueuedContainerUUID)
+                       if !c.Check(ok, check.Equals, false) {
+                               // Don't spam the log with more failures
+                               break
+                       }
+               }
+       }()
+
+       var ctr arvados.Container
+       err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
+       c.Check(err, check.IsNil)
+       c.Check(ctr.State, check.Equals, arvados.ContainerStateQueued)
+
+       go cq.Update()
+
+       // Wait for the cancel operation to take effect. Container
+       // will have state=Cancelled or just disappear from the queue.
+       suite.waitfor(c, time.Second, func() bool {
+               err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
+               return err == nil && ctr.State == arvados.ContainerStateCancelled
+       })
+       c.Check(ctr.RuntimeStatus["error"], check.Equals, `no suitable instance type`)
 }
 
-func testTypeChooser(ctr *arvados.Container) (arvados.InstanceType, error) {
-       return arvados.InstanceType{Name: "testType"}, nil
+func (suite *IntegrationSuite) waitfor(c *check.C, timeout time.Duration, fn func() bool) {
+       defer func() {
+               c.Check(fn(), check.Equals, true)
+       }()
+       deadline := time.Now().Add(timeout)
+       for !fn() && time.Now().Before(deadline) {
+               time.Sleep(timeout / 1000)
+       }
 }