lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "context"
   9         "encoding/json"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "net/http/httptest"
  14         "os"
  15         "sync"
  16         "time"
  17
  18         "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
  19         "git.curoverse.com/arvados.git/sdk/go/arvados"
  20         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  21         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  22         "golang.org/x/crypto/ssh"
  23         check "gopkg.in/check.v1"
  24 )
  25
  26 var _ = check.Suite(&DispatcherSuite{})
  27
  28 type DispatcherSuite struct {
  29         ctx        context.Context
  30         cancel     context.CancelFunc
  31         cluster    *arvados.Cluster
  32         stubDriver *test.StubDriver
  33         disp       *dispatcher
  34 }
  35
  36 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  37         s.ctx, s.cancel = context.WithCancel(context.Background())
  38         s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
  39         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  40         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  41         c.Assert(err, check.IsNil)
  42
  43         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  44         s.stubDriver = &test.StubDriver{
  45                 HostKey:                   hostpriv,
  46                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
  47                 ErrorRateDestroy:          0.1,
  48                 MinTimeBetweenCreateCalls: time.Millisecond,
  49         }
  50
  51         s.cluster = &arvados.Cluster{
  52                 CloudVMs: arvados.CloudVMs{
  53                         Driver:               "test",
  54                         SyncInterval:         arvados.Duration(10 * time.Millisecond),
  55                         TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
  56                         TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
  57                         TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
  58                         TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
  59                         MaxCloudOpsPerSecond: 500,
  60                 },
  61                 Dispatch: arvados.Dispatch{
  62                         PrivateKey:         string(dispatchprivraw),
  63                         PollInterval:       arvados.Duration(5 * time.Millisecond),
  64                         ProbeInterval:      arvados.Duration(5 * time.Millisecond),
  65                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
  66                         MaxProbesPerSecond: 1000,
  67                         TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
  68                         TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
  69                 },
  70                 InstanceTypes: arvados.InstanceTypeMap{
  71                         test.InstanceType(1).Name:  test.InstanceType(1),
  72                         test.InstanceType(2).Name:  test.InstanceType(2),
  73                         test.InstanceType(3).Name:  test.InstanceType(3),
  74                         test.InstanceType(4).Name:  test.InstanceType(4),
  75                         test.InstanceType(6).Name:  test.InstanceType(6),
  76                         test.InstanceType(8).Name:  test.InstanceType(8),
  77                         test.InstanceType(16).Name: test.InstanceType(16),
  78                 },
  79                 NodeProfiles: map[string]arvados.NodeProfile{
  80                         "*": {
  81                                 Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
  82                                 DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
  83                         },
  84                 },
  85                 Services: arvados.Services{
  86                         DispatchCloud: arvados.Service{InternalURLs: map[arvados.URL]arvados.ServiceInstance{
  87                                 arvados.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}: {},
  88                         }},
  89                 },
  90         }
  91         s.disp = &dispatcher{
  92                 Cluster:   s.cluster,
  93                 Context:   s.ctx,
  94                 AuthToken: arvadostest.AdminToken,
  95         }
  96         // Test cases can modify s.cluster before calling
  97         // initialize(), and then modify private state before calling
  98         // go run().
  99 }
 100
 101 func (s *DispatcherSuite) TearDownTest(c *check.C) {
 102         s.cancel()
 103         s.disp.Close()
 104 }
 105
 106 // DispatchToStubDriver checks that the dispatcher wires everything
 107 // together effectively. It uses a real scheduler and worker pool with
 108 // a fake queue and cloud driver. The fake cloud driver injects
 109 // artificial errors in order to exercise a variety of code paths.
 110 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 111         drivers["test"] = s.stubDriver
 112         s.disp.setupOnce.Do(s.disp.initialize)
 113         queue := &test.Queue{
 114                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 115                         return ChooseInstanceType(s.cluster, ctr)
 116                 },
 117         }
 118         for i := 0; i < 200; i++ {
 119                 queue.Containers = append(queue.Containers, arvados.Container{
 120                         UUID:     test.ContainerUUID(i + 1),
 121                         State:    arvados.ContainerStateQueued,
 122                         Priority: int64(i%20 + 1),
 123                         RuntimeConstraints: arvados.RuntimeConstraints{
 124                                 RAM:   int64(i%3+1) << 30,
 125                                 VCPUs: i%8 + 1,
 126                         },
 127                 })
 128         }
 129         s.disp.queue = queue
 130
 131         var mtx sync.Mutex
 132         done := make(chan struct{})
 133         waiting := map[string]struct{}{}
 134         for _, ctr := range queue.Containers {
 135                 waiting[ctr.UUID] = struct{}{}
 136         }
 137         finishContainer := func(ctr arvados.Container) {
 138                 mtx.Lock()
 139                 defer mtx.Unlock()
 140                 if _, ok := waiting[ctr.UUID]; !ok {
 141                         c.Errorf("container completed twice: %s", ctr.UUID)
 142                         return
 143                 }
 144                 delete(waiting, ctr.UUID)
 145                 if len(waiting) == 0 {
 146                         close(done)
 147                 }
 148         }
 149         executeContainer := func(ctr arvados.Container) int {
 150                 finishContainer(ctr)
 151                 return int(rand.Uint32() & 0x3)
 152         }
 153         n := 0
 154         s.stubDriver.Queue = queue
 155         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 156                 n++
 157                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 158                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 159                 stubvm.ExecuteContainer = executeContainer
 160                 stubvm.CrashRunningContainer = finishContainer
 161                 switch n % 7 {
 162                 case 0:
 163                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 164                 case 1:
 165                         stubvm.CrunchRunMissing = true
 166                 case 2:
 167                         stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
 168                 default:
 169                         stubvm.CrunchRunCrashRate = 0.1
 170                 }
 171         }
 172
 173         start := time.Now()
 174         go s.disp.run()
 175         err := s.disp.CheckHealth()
 176         c.Check(err, check.IsNil)
 177
 178         select {
 179         case <-done:
 180                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 181         case <-time.After(10 * time.Second):
 182                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
 183         }
 184
 185         deadline := time.Now().Add(5 * time.Second)
 186         for range time.NewTicker(10 * time.Millisecond).C {
 187                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 188                 c.Check(err, check.IsNil)
 189                 queue.Update()
 190                 ents, _ := queue.Entries()
 191                 if len(ents) == 0 && len(insts) == 0 {
 192                         break
 193                 }
 194                 if time.Now().After(deadline) {
 195                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 196                 }
 197         }
 198 }
 199
 200 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 201         s.cluster.ManagementToken = "abcdefgh"
 202         drivers["test"] = s.stubDriver
 203         s.disp.setupOnce.Do(s.disp.initialize)
 204         s.disp.queue = &test.Queue{}
 205         go s.disp.run()
 206
 207         for _, token := range []string{"abc", ""} {
 208                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 209                 if token != "" {
 210                         req.Header.Set("Authorization", "Bearer "+token)
 211                 }
 212                 resp := httptest.NewRecorder()
 213                 s.disp.ServeHTTP(resp, req)
 214                 if token == "" {
 215                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 216                 } else {
 217                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 218                 }
 219         }
 220 }
 221
 222 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 223         s.cluster.ManagementToken = ""
 224         drivers["test"] = s.stubDriver
 225         s.disp.setupOnce.Do(s.disp.initialize)
 226         s.disp.queue = &test.Queue{}
 227         go s.disp.run()
 228
 229         for _, token := range []string{"abc", ""} {
 230                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 231                 if token != "" {
 232                         req.Header.Set("Authorization", "Bearer "+token)
 233                 }
 234                 resp := httptest.NewRecorder()
 235                 s.disp.ServeHTTP(resp, req)
 236                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 237         }
 238 }
 239
 240 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 241         s.cluster.ManagementToken = "abcdefgh"
 242         s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 243         drivers["test"] = s.stubDriver
 244         s.disp.setupOnce.Do(s.disp.initialize)
 245         s.disp.queue = &test.Queue{}
 246         go s.disp.run()
 247
 248         type instance struct {
 249                 Instance             string
 250                 WorkerState          string `json:"worker_state"`
 251                 Price                float64
 252                 LastContainerUUID    string `json:"last_container_uuid"`
 253                 ArvadosInstanceType  string `json:"arvados_instance_type"`
 254                 ProviderInstanceType string `json:"provider_instance_type"`
 255         }
 256         type instancesResponse struct {
 257                 Items []instance
 258         }
 259         getInstances := func() instancesResponse {
 260                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 261                 req.Header.Set("Authorization", "Bearer abcdefgh")
 262                 resp := httptest.NewRecorder()
 263                 s.disp.ServeHTTP(resp, req)
 264                 var sr instancesResponse
 265                 c.Check(resp.Code, check.Equals, http.StatusOK)
 266                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 267                 c.Check(err, check.IsNil)
 268                 return sr
 269         }
 270
 271         sr := getInstances()
 272         c.Check(len(sr.Items), check.Equals, 0)
 273
 274         ch := s.disp.pool.Subscribe()
 275         defer s.disp.pool.Unsubscribe(ch)
 276         ok := s.disp.pool.Create(test.InstanceType(1))
 277         c.Check(ok, check.Equals, true)
 278         <-ch
 279
 280         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
 281                 sr = getInstances()
 282                 if len(sr.Items) > 0 {
 283                         break
 284                 }
 285                 time.Sleep(time.Millisecond)
 286         }
 287         c.Assert(len(sr.Items), check.Equals, 1)
 288         c.Check(sr.Items[0].Instance, check.Matches, "stub.*")
 289         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 290         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 291         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 292         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 293         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 294 }