lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "encoding/json"
   9         "io/ioutil"
  10         "math/rand"
  11         "net/http"
  12         "net/http/httptest"
  13         "os"
  14         "sync"
  15         "time"
  16
  17         "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
  18         "git.curoverse.com/arvados.git/sdk/go/arvados"
  19         "golang.org/x/crypto/ssh"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 var _ = check.Suite(&DispatcherSuite{})
  24
  25 type DispatcherSuite struct {
  26         cluster    *arvados.Cluster
  27         stubDriver *test.StubDriver
  28         disp       *dispatcher
  29 }
  30
  31 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  32         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  33         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  34         c.Assert(err, check.IsNil)
  35
  36         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  37         s.stubDriver = &test.StubDriver{
  38                 HostKey:                   hostpriv,
  39                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
  40                 ErrorRateDestroy:          0.1,
  41                 MinTimeBetweenCreateCalls: time.Millisecond,
  42         }
  43
  44         s.cluster = &arvados.Cluster{
  45                 CloudVMs: arvados.CloudVMs{
  46                         Driver:           "test",
  47                         SyncInterval:     arvados.Duration(10 * time.Millisecond),
  48                         TimeoutIdle:      arvados.Duration(150 * time.Millisecond),
  49                         TimeoutBooting:   arvados.Duration(150 * time.Millisecond),
  50                         TimeoutProbe:     arvados.Duration(15 * time.Millisecond),
  51                         TimeoutShutdown:  arvados.Duration(5 * time.Millisecond),
  52                         DriverParameters: json.RawMessage(`{"test": true}`),
  53                 },
  54                 Dispatch: arvados.Dispatch{
  55                         PrivateKey:         dispatchprivraw,
  56                         PollInterval:       arvados.Duration(5 * time.Millisecond),
  57                         ProbeInterval:      arvados.Duration(5 * time.Millisecond),
  58                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
  59                         MaxProbesPerSecond: 1000,
  60                 },
  61                 InstanceTypes: arvados.InstanceTypeMap{
  62                         test.InstanceType(1).Name:  test.InstanceType(1),
  63                         test.InstanceType(2).Name:  test.InstanceType(2),
  64                         test.InstanceType(3).Name:  test.InstanceType(3),
  65                         test.InstanceType(4).Name:  test.InstanceType(4),
  66                         test.InstanceType(6).Name:  test.InstanceType(6),
  67                         test.InstanceType(8).Name:  test.InstanceType(8),
  68                         test.InstanceType(16).Name: test.InstanceType(16),
  69                 },
  70                 NodeProfiles: map[string]arvados.NodeProfile{
  71                         "*": {
  72                                 Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
  73                                 DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
  74                         },
  75                 },
  76         }
  77         s.disp = &dispatcher{Cluster: s.cluster}
  78         // Test cases can modify s.cluster before calling
  79         // initialize(), and then modify private state before calling
  80         // go run().
  81 }
  82
  83 func (s *DispatcherSuite) TearDownTest(c *check.C) {
  84         s.disp.Close()
  85 }
  86
  87 // DispatchToStubDriver checks that the dispatcher wires everything
  88 // together effectively. It uses a real scheduler and worker pool with
  89 // a fake queue and cloud driver. The fake cloud driver injects
  90 // artificial errors in order to exercise a variety of code paths.
  91 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
  92         drivers["test"] = s.stubDriver
  93         s.disp.setupOnce.Do(s.disp.initialize)
  94         queue := &test.Queue{
  95                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
  96                         return ChooseInstanceType(s.cluster, ctr)
  97                 },
  98         }
  99         for i := 0; i < 200; i++ {
 100                 queue.Containers = append(queue.Containers, arvados.Container{
 101                         UUID:     test.ContainerUUID(i + 1),
 102                         State:    arvados.ContainerStateQueued,
 103                         Priority: int64(i%20 + 1),
 104                         RuntimeConstraints: arvados.RuntimeConstraints{
 105                                 RAM:   int64(i%3+1) << 30,
 106                                 VCPUs: i%8 + 1,
 107                         },
 108                 })
 109         }
 110         s.disp.queue = queue
 111
 112         var mtx sync.Mutex
 113         done := make(chan struct{})
 114         waiting := map[string]struct{}{}
 115         for _, ctr := range queue.Containers {
 116                 waiting[ctr.UUID] = struct{}{}
 117         }
 118         executeContainer := func(ctr arvados.Container) int {
 119                 mtx.Lock()
 120                 defer mtx.Unlock()
 121                 if _, ok := waiting[ctr.UUID]; !ok {
 122                         c.Logf("container completed twice: %s -- perhaps completed after stub instance was killed?", ctr.UUID)
 123                         return 1
 124                 }
 125                 delete(waiting, ctr.UUID)
 126                 if len(waiting) == 0 {
 127                         close(done)
 128                 }
 129                 return int(rand.Uint32() & 0x3)
 130         }
 131         n := 0
 132         s.stubDriver.Queue = queue
 133         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 134                 n++
 135                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 136                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 137                 stubvm.ExecuteContainer = executeContainer
 138                 switch n % 7 {
 139                 case 0:
 140                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 141                 case 1:
 142                         stubvm.CrunchRunMissing = true
 143                 default:
 144                         stubvm.CrunchRunCrashRate = 0.1
 145                 }
 146         }
 147
 148         start := time.Now()
 149         go s.disp.run()
 150         err := s.disp.CheckHealth()
 151         c.Check(err, check.IsNil)
 152
 153         select {
 154         case <-done:
 155                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 156         case <-time.After(10 * time.Second):
 157                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
 158         }
 159
 160         deadline := time.Now().Add(time.Second)
 161         for range time.NewTicker(10 * time.Millisecond).C {
 162                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 163                 c.Check(err, check.IsNil)
 164                 queue.Update()
 165                 ents, _ := queue.Entries()
 166                 if len(ents) == 0 && len(insts) == 0 {
 167                         break
 168                 }
 169                 if time.Now().After(deadline) {
 170                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 171                 }
 172         }
 173 }
 174
 175 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 176         s.cluster.ManagementToken = "abcdefgh"
 177         drivers["test"] = s.stubDriver
 178         s.disp.setupOnce.Do(s.disp.initialize)
 179         s.disp.queue = &test.Queue{}
 180         go s.disp.run()
 181
 182         for _, token := range []string{"abc", ""} {
 183                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 184                 if token != "" {
 185                         req.Header.Set("Authorization", "Bearer "+token)
 186                 }
 187                 resp := httptest.NewRecorder()
 188                 s.disp.ServeHTTP(resp, req)
 189                 if token == "" {
 190                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 191                 } else {
 192                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 193                 }
 194         }
 195 }
 196
 197 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 198         s.cluster.ManagementToken = ""
 199         drivers["test"] = s.stubDriver
 200         s.disp.setupOnce.Do(s.disp.initialize)
 201         s.disp.queue = &test.Queue{}
 202         go s.disp.run()
 203
 204         for _, token := range []string{"abc", ""} {
 205                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 206                 if token != "" {
 207                         req.Header.Set("Authorization", "Bearer "+token)
 208                 }
 209                 resp := httptest.NewRecorder()
 210                 s.disp.ServeHTTP(resp, req)
 211                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 212         }
 213 }
 214
 215 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 216         s.cluster.ManagementToken = "abcdefgh"
 217         s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 218         drivers["test"] = s.stubDriver
 219         s.disp.setupOnce.Do(s.disp.initialize)
 220         s.disp.queue = &test.Queue{}
 221         go s.disp.run()
 222
 223         type instance struct {
 224                 Instance             string
 225                 WorkerState          string `json:"worker_state"`
 226                 Price                float64
 227                 LastContainerUUID    string `json:"last_container_uuid"`
 228                 ArvadosInstanceType  string `json:"arvados_instance_type"`
 229                 ProviderInstanceType string `json:"provider_instance_type"`
 230         }
 231         type instancesResponse struct {
 232                 Items []instance
 233         }
 234         getInstances := func() instancesResponse {
 235                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 236                 req.Header.Set("Authorization", "Bearer abcdefgh")
 237                 resp := httptest.NewRecorder()
 238                 s.disp.ServeHTTP(resp, req)
 239                 var sr instancesResponse
 240                 c.Check(resp.Code, check.Equals, http.StatusOK)
 241                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 242                 c.Check(err, check.IsNil)
 243                 return sr
 244         }
 245
 246         sr := getInstances()
 247         c.Check(len(sr.Items), check.Equals, 0)
 248
 249         ch := s.disp.pool.Subscribe()
 250         defer s.disp.pool.Unsubscribe(ch)
 251         ok := s.disp.pool.Create(test.InstanceType(1))
 252         c.Check(ok, check.Equals, true)
 253         <-ch
 254
 255         sr = getInstances()
 256         c.Assert(len(sr.Items), check.Equals, 1)
 257         c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
 258         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 259         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 260         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 261         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 262         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 263 }