lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "context"
   9         "encoding/json"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "net/http/httptest"
  14         "os"
  15         "sync"
  16         "time"
  17
  18         "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
  19         "git.curoverse.com/arvados.git/sdk/go/arvados"
  20         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  21         "golang.org/x/crypto/ssh"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 var _ = check.Suite(&DispatcherSuite{})
  26
  27 type DispatcherSuite struct {
  28         ctx        context.Context
  29         cancel     context.CancelFunc
  30         cluster    *arvados.Cluster
  31         stubDriver *test.StubDriver
  32         disp       *dispatcher
  33 }
  34
  35 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  36         s.ctx, s.cancel = context.WithCancel(context.Background())
  37         s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
  38         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  39         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  40         c.Assert(err, check.IsNil)
  41
  42         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  43         s.stubDriver = &test.StubDriver{
  44                 HostKey:                   hostpriv,
  45                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
  46                 ErrorRateDestroy:          0.1,
  47                 MinTimeBetweenCreateCalls: time.Millisecond,
  48         }
  49
  50         s.cluster = &arvados.Cluster{
  51                 CloudVMs: arvados.CloudVMs{
  52                         Driver:          "test",
  53                         SyncInterval:    arvados.Duration(10 * time.Millisecond),
  54                         TimeoutIdle:     arvados.Duration(150 * time.Millisecond),
  55                         TimeoutBooting:  arvados.Duration(150 * time.Millisecond),
  56                         TimeoutProbe:    arvados.Duration(15 * time.Millisecond),
  57                         TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
  58                 },
  59                 Dispatch: arvados.Dispatch{
  60                         PrivateKey:         string(dispatchprivraw),
  61                         PollInterval:       arvados.Duration(5 * time.Millisecond),
  62                         ProbeInterval:      arvados.Duration(5 * time.Millisecond),
  63                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
  64                         MaxProbesPerSecond: 1000,
  65                         TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
  66                         TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
  67                 },
  68                 InstanceTypes: arvados.InstanceTypeMap{
  69                         test.InstanceType(1).Name:  test.InstanceType(1),
  70                         test.InstanceType(2).Name:  test.InstanceType(2),
  71                         test.InstanceType(3).Name:  test.InstanceType(3),
  72                         test.InstanceType(4).Name:  test.InstanceType(4),
  73                         test.InstanceType(6).Name:  test.InstanceType(6),
  74                         test.InstanceType(8).Name:  test.InstanceType(8),
  75                         test.InstanceType(16).Name: test.InstanceType(16),
  76                 },
  77                 NodeProfiles: map[string]arvados.NodeProfile{
  78                         "*": {
  79                                 Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
  80                                 DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
  81                         },
  82                 },
  83         }
  84         s.disp = &dispatcher{
  85                 Cluster: s.cluster,
  86                 Context: s.ctx,
  87         }
  88         // Test cases can modify s.cluster before calling
  89         // initialize(), and then modify private state before calling
  90         // go run().
  91 }
  92
  93 func (s *DispatcherSuite) TearDownTest(c *check.C) {
  94         s.cancel()
  95         s.disp.Close()
  96 }
  97
  98 // DispatchToStubDriver checks that the dispatcher wires everything
  99 // together effectively. It uses a real scheduler and worker pool with
 100 // a fake queue and cloud driver. The fake cloud driver injects
 101 // artificial errors in order to exercise a variety of code paths.
 102 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 103         drivers["test"] = s.stubDriver
 104         s.disp.setupOnce.Do(s.disp.initialize)
 105         queue := &test.Queue{
 106                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 107                         return ChooseInstanceType(s.cluster, ctr)
 108                 },
 109         }
 110         for i := 0; i < 200; i++ {
 111                 queue.Containers = append(queue.Containers, arvados.Container{
 112                         UUID:     test.ContainerUUID(i + 1),
 113                         State:    arvados.ContainerStateQueued,
 114                         Priority: int64(i%20 + 1),
 115                         RuntimeConstraints: arvados.RuntimeConstraints{
 116                                 RAM:   int64(i%3+1) << 30,
 117                                 VCPUs: i%8 + 1,
 118                         },
 119                 })
 120         }
 121         s.disp.queue = queue
 122
 123         var mtx sync.Mutex
 124         done := make(chan struct{})
 125         waiting := map[string]struct{}{}
 126         for _, ctr := range queue.Containers {
 127                 waiting[ctr.UUID] = struct{}{}
 128         }
 129         finishContainer := func(ctr arvados.Container) {
 130                 mtx.Lock()
 131                 defer mtx.Unlock()
 132                 if _, ok := waiting[ctr.UUID]; !ok {
 133                         c.Errorf("container completed twice: %s", ctr.UUID)
 134                         return
 135                 }
 136                 delete(waiting, ctr.UUID)
 137                 if len(waiting) == 0 {
 138                         close(done)
 139                 }
 140         }
 141         executeContainer := func(ctr arvados.Container) int {
 142                 finishContainer(ctr)
 143                 return int(rand.Uint32() & 0x3)
 144         }
 145         n := 0
 146         s.stubDriver.Queue = queue
 147         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 148                 n++
 149                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 150                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 151                 stubvm.ExecuteContainer = executeContainer
 152                 stubvm.CrashRunningContainer = finishContainer
 153                 switch n % 7 {
 154                 case 0:
 155                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 156                 case 1:
 157                         stubvm.CrunchRunMissing = true
 158                 default:
 159                         stubvm.CrunchRunCrashRate = 0.1
 160                 }
 161         }
 162
 163         start := time.Now()
 164         go s.disp.run()
 165         err := s.disp.CheckHealth()
 166         c.Check(err, check.IsNil)
 167
 168         select {
 169         case <-done:
 170                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 171         case <-time.After(10 * time.Second):
 172                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
 173         }
 174
 175         deadline := time.Now().Add(5 * time.Second)
 176         for range time.NewTicker(10 * time.Millisecond).C {
 177                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 178                 c.Check(err, check.IsNil)
 179                 queue.Update()
 180                 ents, _ := queue.Entries()
 181                 if len(ents) == 0 && len(insts) == 0 {
 182                         break
 183                 }
 184                 if time.Now().After(deadline) {
 185                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 186                 }
 187         }
 188 }
 189
 190 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 191         s.cluster.ManagementToken = "abcdefgh"
 192         drivers["test"] = s.stubDriver
 193         s.disp.setupOnce.Do(s.disp.initialize)
 194         s.disp.queue = &test.Queue{}
 195         go s.disp.run()
 196
 197         for _, token := range []string{"abc", ""} {
 198                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 199                 if token != "" {
 200                         req.Header.Set("Authorization", "Bearer "+token)
 201                 }
 202                 resp := httptest.NewRecorder()
 203                 s.disp.ServeHTTP(resp, req)
 204                 if token == "" {
 205                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 206                 } else {
 207                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 208                 }
 209         }
 210 }
 211
 212 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 213         s.cluster.ManagementToken = ""
 214         drivers["test"] = s.stubDriver
 215         s.disp.setupOnce.Do(s.disp.initialize)
 216         s.disp.queue = &test.Queue{}
 217         go s.disp.run()
 218
 219         for _, token := range []string{"abc", ""} {
 220                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 221                 if token != "" {
 222                         req.Header.Set("Authorization", "Bearer "+token)
 223                 }
 224                 resp := httptest.NewRecorder()
 225                 s.disp.ServeHTTP(resp, req)
 226                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 227         }
 228 }
 229
 230 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 231         s.cluster.ManagementToken = "abcdefgh"
 232         s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 233         drivers["test"] = s.stubDriver
 234         s.disp.setupOnce.Do(s.disp.initialize)
 235         s.disp.queue = &test.Queue{}
 236         go s.disp.run()
 237
 238         type instance struct {
 239                 Instance             string
 240                 WorkerState          string `json:"worker_state"`
 241                 Price                float64
 242                 LastContainerUUID    string `json:"last_container_uuid"`
 243                 ArvadosInstanceType  string `json:"arvados_instance_type"`
 244                 ProviderInstanceType string `json:"provider_instance_type"`
 245         }
 246         type instancesResponse struct {
 247                 Items []instance
 248         }
 249         getInstances := func() instancesResponse {
 250                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 251                 req.Header.Set("Authorization", "Bearer abcdefgh")
 252                 resp := httptest.NewRecorder()
 253                 s.disp.ServeHTTP(resp, req)
 254                 var sr instancesResponse
 255                 c.Check(resp.Code, check.Equals, http.StatusOK)
 256                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 257                 c.Check(err, check.IsNil)
 258                 return sr
 259         }
 260
 261         sr := getInstances()
 262         c.Check(len(sr.Items), check.Equals, 0)
 263
 264         ch := s.disp.pool.Subscribe()
 265         defer s.disp.pool.Unsubscribe(ch)
 266         ok := s.disp.pool.Create(test.InstanceType(1))
 267         c.Check(ok, check.Equals, true)
 268         <-ch
 269
 270         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
 271                 sr = getInstances()
 272                 if len(sr.Items) > 0 {
 273                         break
 274                 }
 275                 time.Sleep(time.Millisecond)
 276         }
 277         c.Assert(len(sr.Items), check.Equals, 1)
 278         c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
 279         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 280         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 281         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 282         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 283         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 284 }