lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "encoding/json"
   9         "io/ioutil"
  10         "math/rand"
  11         "net/http"
  12         "net/http/httptest"
  13         "os"
  14         "sync"
  15         "time"
  16
  17         "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
  18         "git.curoverse.com/arvados.git/sdk/go/arvados"
  19         "github.com/Sirupsen/logrus"
  20         "golang.org/x/crypto/ssh"
  21         check "gopkg.in/check.v1"
  22 )
  23
  24 var _ = check.Suite(&DispatcherSuite{})
  25
  26 type DispatcherSuite struct {
  27         cluster     *arvados.Cluster
  28         instanceSet *test.LameInstanceSet
  29         stubDriver  *test.StubDriver
  30         disp        *dispatcher
  31 }
  32
  33 func (s *DispatcherSuite) SetUpSuite(c *check.C) {
  34         if os.Getenv("ARVADOS_DEBUG") != "" {
  35                 logrus.StandardLogger().SetLevel(logrus.DebugLevel)
  36         }
  37 }
  38
  39 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  40         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  41         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  42         c.Assert(err, check.IsNil)
  43
  44         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  45         s.stubDriver = &test.StubDriver{
  46                 HostKey:          hostpriv,
  47                 AuthorizedKeys:   []ssh.PublicKey{dispatchpub},
  48                 ErrorRateDestroy: 0.1,
  49         }
  50
  51         s.cluster = &arvados.Cluster{
  52                 CloudVMs: arvados.CloudVMs{
  53                         Driver:          "test",
  54                         SyncInterval:    arvados.Duration(10 * time.Millisecond),
  55                         TimeoutIdle:     arvados.Duration(30 * time.Millisecond),
  56                         TimeoutBooting:  arvados.Duration(30 * time.Millisecond),
  57                         TimeoutProbe:    arvados.Duration(15 * time.Millisecond),
  58                         TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
  59                 },
  60                 Dispatch: arvados.Dispatch{
  61                         PrivateKey:         dispatchprivraw,
  62                         PollInterval:       arvados.Duration(5 * time.Millisecond),
  63                         ProbeInterval:      arvados.Duration(5 * time.Millisecond),
  64                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
  65                         MaxProbesPerSecond: 1000,
  66                 },
  67                 InstanceTypes: arvados.InstanceTypeMap{
  68                         test.InstanceType(1).Name:  test.InstanceType(1),
  69                         test.InstanceType(2).Name:  test.InstanceType(2),
  70                         test.InstanceType(3).Name:  test.InstanceType(3),
  71                         test.InstanceType(4).Name:  test.InstanceType(4),
  72                         test.InstanceType(6).Name:  test.InstanceType(6),
  73                         test.InstanceType(8).Name:  test.InstanceType(8),
  74                         test.InstanceType(16).Name: test.InstanceType(16),
  75                 },
  76                 NodeProfiles: map[string]arvados.NodeProfile{
  77                         "*": {
  78                                 Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
  79                                 DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
  80                         },
  81                 },
  82         }
  83         s.disp = &dispatcher{Cluster: s.cluster}
  84         // Test cases can modify s.cluster before calling
  85         // initialize(), and then modify private state before calling
  86         // go run().
  87 }
  88
  89 func (s *DispatcherSuite) TearDownTest(c *check.C) {
  90         s.disp.Close()
  91 }
  92
  93 // DispatchToStubDriver checks that the dispatcher wires everything
  94 // together effectively. It uses a real scheduler and worker pool with
  95 // a fake queue and cloud driver. The fake cloud driver injects
  96 // artificial errors in order to exercise a variety of code paths.
  97 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
  98         drivers["test"] = s.stubDriver
  99         s.disp.setupOnce.Do(s.disp.initialize)
 100         queue := &test.Queue{
 101                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 102                         return ChooseInstanceType(s.cluster, ctr)
 103                 },
 104         }
 105         for i := 0; i < 200; i++ {
 106                 queue.Containers = append(queue.Containers, arvados.Container{
 107                         UUID:     test.ContainerUUID(i + 1),
 108                         State:    arvados.ContainerStateQueued,
 109                         Priority: int64(i%20 + 1),
 110                         RuntimeConstraints: arvados.RuntimeConstraints{
 111                                 RAM:   int64(i%3+1) << 30,
 112                                 VCPUs: i%8 + 1,
 113                         },
 114                 })
 115         }
 116         s.disp.queue = queue
 117
 118         var mtx sync.Mutex
 119         done := make(chan struct{})
 120         waiting := map[string]struct{}{}
 121         for _, ctr := range queue.Containers {
 122                 waiting[ctr.UUID] = struct{}{}
 123         }
 124         executeContainer := func(ctr arvados.Container) int {
 125                 mtx.Lock()
 126                 defer mtx.Unlock()
 127                 if _, ok := waiting[ctr.UUID]; !ok {
 128                         c.Logf("container completed twice: %s -- perhaps completed after stub instance was killed?", ctr.UUID)
 129                         return 1
 130                 }
 131                 delete(waiting, ctr.UUID)
 132                 if len(waiting) == 0 {
 133                         close(done)
 134                 }
 135                 return int(rand.Uint32() & 0x3)
 136         }
 137         n := 0
 138         s.stubDriver.Queue = queue
 139         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 140                 n++
 141                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 142                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 143                 stubvm.ExecuteContainer = executeContainer
 144                 switch n % 7 {
 145                 case 0:
 146                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 147                 case 1:
 148                         stubvm.CrunchRunMissing = true
 149                 default:
 150                         stubvm.CrunchRunCrashRate = 0.1
 151                 }
 152         }
 153
 154         start := time.Now()
 155         go s.disp.run()
 156         err := s.disp.CheckHealth()
 157         c.Check(err, check.IsNil)
 158
 159         select {
 160         case <-done:
 161                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 162         case <-time.After(10 * time.Second):
 163                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
 164         }
 165
 166         deadline := time.Now().Add(time.Second)
 167         for range time.NewTicker(10 * time.Millisecond).C {
 168                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 169                 c.Check(err, check.IsNil)
 170                 queue.Update()
 171                 ents, _ := queue.Entries()
 172                 if len(ents) == 0 && len(insts) == 0 {
 173                         break
 174                 }
 175                 if time.Now().After(deadline) {
 176                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 177                 }
 178         }
 179 }
 180
 181 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 182         s.cluster.ManagementToken = "abcdefgh"
 183         drivers["test"] = s.stubDriver
 184         s.disp.setupOnce.Do(s.disp.initialize)
 185         s.disp.queue = &test.Queue{}
 186         go s.disp.run()
 187
 188         for _, token := range []string{"abc", ""} {
 189                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 190                 if token != "" {
 191                         req.Header.Set("Authorization", "Bearer "+token)
 192                 }
 193                 resp := httptest.NewRecorder()
 194                 s.disp.ServeHTTP(resp, req)
 195                 if token == "" {
 196                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 197                 } else {
 198                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 199                 }
 200         }
 201 }
 202
 203 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 204         s.cluster.ManagementToken = ""
 205         drivers["test"] = s.stubDriver
 206         s.disp.setupOnce.Do(s.disp.initialize)
 207         s.disp.queue = &test.Queue{}
 208         go s.disp.run()
 209
 210         for _, token := range []string{"abc", ""} {
 211                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 212                 if token != "" {
 213                         req.Header.Set("Authorization", "Bearer "+token)
 214                 }
 215                 resp := httptest.NewRecorder()
 216                 s.disp.ServeHTTP(resp, req)
 217                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 218         }
 219 }
 220
 221 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 222         s.cluster.ManagementToken = "abcdefgh"
 223         s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 224         drivers["test"] = s.stubDriver
 225         s.disp.setupOnce.Do(s.disp.initialize)
 226         s.disp.queue = &test.Queue{}
 227         go s.disp.run()
 228
 229         type instance struct {
 230                 Instance             string
 231                 WorkerState          string
 232                 Price                float64
 233                 LastContainerUUID    string
 234                 ArvadosInstanceType  string
 235                 ProviderInstanceType string
 236         }
 237         type instancesResponse struct {
 238                 Items []instance
 239         }
 240         getInstances := func() instancesResponse {
 241                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 242                 req.Header.Set("Authorization", "Bearer abcdefgh")
 243                 resp := httptest.NewRecorder()
 244                 s.disp.ServeHTTP(resp, req)
 245                 var sr instancesResponse
 246                 c.Check(resp.Code, check.Equals, http.StatusOK)
 247                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 248                 c.Check(err, check.IsNil)
 249                 return sr
 250         }
 251
 252         sr := getInstances()
 253         c.Check(len(sr.Items), check.Equals, 0)
 254
 255         ch := s.disp.pool.Subscribe()
 256         defer s.disp.pool.Unsubscribe(ch)
 257         err := s.disp.pool.Create(test.InstanceType(1))
 258         c.Check(err, check.IsNil)
 259         <-ch
 260
 261         sr = getInstances()
 262         c.Assert(len(sr.Items), check.Equals, 1)
 263         c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
 264         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 265         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 266         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 267         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 268         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 269 }