lib/dispatchcloud/container/queue_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package container
   6
   7 import (
   8         "errors"
   9         "sync"
  10         "testing"
  11         "time"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/arvados"
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         "github.com/sirupsen/logrus"
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 // Gocheck boilerplate
  20 func Test(t *testing.T) {
  21         check.TestingT(t)
  22 }
  23
  24 var _ = check.Suite(&IntegrationSuite{})
  25
  26 type IntegrationSuite struct{}
  27
  28 func (suite *IntegrationSuite) TearDownTest(c *check.C) {
  29         err := arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil)
  30         c.Check(err, check.IsNil)
  31 }
  32
  33 func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
  34         typeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
  35                 return arvados.InstanceType{Name: "testType"}, nil
  36         }
  37
  38         client := arvados.NewClientFromEnv()
  39         cq := NewQueue(logrus.StandardLogger(), nil, typeChooser, client)
  40
  41         err := cq.Update()
  42         c.Check(err, check.IsNil)
  43
  44         ents, threshold := cq.Entries()
  45         c.Check(len(ents), check.Not(check.Equals), 0)
  46         c.Check(time.Since(threshold) < time.Minute, check.Equals, true)
  47         c.Check(time.Since(threshold) > 0, check.Equals, true)
  48
  49         _, ok := ents[arvadostest.QueuedContainerUUID]
  50         c.Check(ok, check.Equals, true)
  51
  52         var wg sync.WaitGroup
  53         for uuid, ent := range ents {
  54                 c.Check(ent.Container.UUID, check.Equals, uuid)
  55                 c.Check(ent.InstanceType.Name, check.Equals, "testType")
  56                 c.Check(ent.Container.State, check.Equals, arvados.ContainerStateQueued)
  57                 c.Check(ent.Container.Priority > 0, check.Equals, true)
  58
  59                 ctr, ok := cq.Get(uuid)
  60                 c.Check(ok, check.Equals, true)
  61                 c.Check(ctr.UUID, check.Equals, uuid)
  62
  63                 wg.Add(1)
  64                 go func() {
  65                         defer wg.Done()
  66                         err := cq.Unlock(uuid)
  67                         c.Check(err, check.NotNil)
  68                         err = cq.Lock(uuid)
  69                         c.Check(err, check.IsNil)
  70                         ctr, ok := cq.Get(uuid)
  71                         c.Check(ok, check.Equals, true)
  72                         c.Check(ctr.State, check.Equals, arvados.ContainerStateLocked)
  73                         err = cq.Lock(uuid)
  74                         c.Check(err, check.NotNil)
  75                         err = cq.Unlock(uuid)
  76                         c.Check(err, check.IsNil)
  77                         ctr, ok = cq.Get(uuid)
  78                         c.Check(ok, check.Equals, true)
  79                         c.Check(ctr.State, check.Equals, arvados.ContainerStateQueued)
  80                         err = cq.Unlock(uuid)
  81                         c.Check(err, check.NotNil)
  82                 }()
  83         }
  84         wg.Wait()
  85
  86         err = cq.Cancel(arvadostest.CompletedContainerUUID)
  87         c.Check(err, check.ErrorMatches, `.*State cannot change from Complete to Cancelled.*`)
  88 }
  89
  90 func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
  91         errorTypeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
  92                 return arvados.InstanceType{}, errors.New("no suitable instance type")
  93         }
  94
  95         client := arvados.NewClientFromEnv()
  96         cq := NewQueue(logrus.StandardLogger(), nil, errorTypeChooser, client)
  97
  98         var ctr arvados.Container
  99         err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
 100         c.Check(err, check.IsNil)
 101         c.Check(ctr.State, check.Equals, arvados.ContainerStateQueued)
 102
 103         cq.Update()
 104
 105         // Wait for the cancel operation to take effect. Container
 106         // will have state=Cancelled or just disappear from the queue.
 107         suite.waitfor(c, time.Second, func() bool {
 108                 err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
 109                 return err == nil && ctr.State == arvados.ContainerStateCancelled
 110         })
 111         c.Check(ctr.RuntimeStatus["error"], check.Equals, `no suitable instance type`)
 112 }
 113
 114 func (suite *IntegrationSuite) waitfor(c *check.C, timeout time.Duration, fn func() bool) {
 115         defer func() {
 116                 c.Check(fn(), check.Equals, true)
 117         }()
 118         deadline := time.Now().Add(timeout)
 119         for !fn() && time.Now().Before(deadline) {
 120                 time.Sleep(timeout / 1000)
 121         }
 122 }