lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "context"
   9         "encoding/json"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "net/http/httptest"
  14         "os"
  15         "sync"
  16         "time"
  17
  18         "git.arvados.org/arvados.git/lib/dispatchcloud/test"
  19         "git.arvados.org/arvados.git/sdk/go/arvados"
  20         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  21         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  22         "github.com/prometheus/client_golang/prometheus"
  23         "golang.org/x/crypto/ssh"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&DispatcherSuite{})
  28
  29 type DispatcherSuite struct {
  30         ctx        context.Context
  31         cancel     context.CancelFunc
  32         cluster    *arvados.Cluster
  33         stubDriver *test.StubDriver
  34         disp       *dispatcher
  35 }
  36
  37 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  38         s.ctx, s.cancel = context.WithCancel(context.Background())
  39         s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
  40         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  41         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  42         c.Assert(err, check.IsNil)
  43
  44         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  45         s.stubDriver = &test.StubDriver{
  46                 HostKey:                   hostpriv,
  47                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
  48                 ErrorRateDestroy:          0.1,
  49                 MinTimeBetweenCreateCalls: time.Millisecond,
  50         }
  51
  52         s.cluster = &arvados.Cluster{
  53                 ManagementToken: "test-management-token",
  54                 Containers: arvados.ContainersConfig{
  55                         DispatchPrivateKey: string(dispatchprivraw),
  56                         StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
  57                         CloudVMs: arvados.CloudVMsConfig{
  58                                 Driver:               "test",
  59                                 SyncInterval:         arvados.Duration(10 * time.Millisecond),
  60                                 TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
  61                                 TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
  62                                 TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
  63                                 TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
  64                                 MaxCloudOpsPerSecond: 500,
  65                                 PollInterval:         arvados.Duration(5 * time.Millisecond),
  66                                 ProbeInterval:        arvados.Duration(5 * time.Millisecond),
  67                                 MaxProbesPerSecond:   1000,
  68                                 TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
  69                                 TimeoutStaleRunLock:  arvados.Duration(3 * time.Millisecond),
  70                                 TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
  71                                 ResourceTags:         map[string]string{"testtag": "test value"},
  72                                 TagKeyPrefix:         "test:",
  73                         },
  74                 },
  75                 InstanceTypes: arvados.InstanceTypeMap{
  76                         test.InstanceType(1).Name:  test.InstanceType(1),
  77                         test.InstanceType(2).Name:  test.InstanceType(2),
  78                         test.InstanceType(3).Name:  test.InstanceType(3),
  79                         test.InstanceType(4).Name:  test.InstanceType(4),
  80                         test.InstanceType(6).Name:  test.InstanceType(6),
  81                         test.InstanceType(8).Name:  test.InstanceType(8),
  82                         test.InstanceType(16).Name: test.InstanceType(16),
  83                 },
  84         }
  85         arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
  86         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
  87
  88         arvClient, err := arvados.NewClientFromConfig(s.cluster)
  89         c.Check(err, check.IsNil)
  90
  91         s.disp = &dispatcher{
  92                 Cluster:   s.cluster,
  93                 Context:   s.ctx,
  94                 ArvClient: arvClient,
  95                 AuthToken: arvadostest.AdminToken,
  96                 Registry:  prometheus.NewRegistry(),
  97         }
  98         // Test cases can modify s.cluster before calling
  99         // initialize(), and then modify private state before calling
 100         // go run().
 101 }
 102
 103 func (s *DispatcherSuite) TearDownTest(c *check.C) {
 104         s.cancel()
 105         s.disp.Close()
 106 }
 107
 108 // DispatchToStubDriver checks that the dispatcher wires everything
 109 // together effectively. It uses a real scheduler and worker pool with
 110 // a fake queue and cloud driver. The fake cloud driver injects
 111 // artificial errors in order to exercise a variety of code paths.
 112 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 113         Drivers["test"] = s.stubDriver
 114         s.disp.setupOnce.Do(s.disp.initialize)
 115         queue := &test.Queue{
 116                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 117                         return ChooseInstanceType(s.cluster, ctr)
 118                 },
 119                 Logger: ctxlog.TestLogger(c),
 120         }
 121         for i := 0; i < 200; i++ {
 122                 queue.Containers = append(queue.Containers, arvados.Container{
 123                         UUID:     test.ContainerUUID(i + 1),
 124                         State:    arvados.ContainerStateQueued,
 125                         Priority: int64(i%20 + 1),
 126                         RuntimeConstraints: arvados.RuntimeConstraints{
 127                                 RAM:   int64(i%3+1) << 30,
 128                                 VCPUs: i%8 + 1,
 129                         },
 130                 })
 131         }
 132         s.disp.queue = queue
 133
 134         var mtx sync.Mutex
 135         done := make(chan struct{})
 136         waiting := map[string]struct{}{}
 137         for _, ctr := range queue.Containers {
 138                 waiting[ctr.UUID] = struct{}{}
 139         }
 140         finishContainer := func(ctr arvados.Container) {
 141                 mtx.Lock()
 142                 defer mtx.Unlock()
 143                 if _, ok := waiting[ctr.UUID]; !ok {
 144                         c.Errorf("container completed twice: %s", ctr.UUID)
 145                         return
 146                 }
 147                 delete(waiting, ctr.UUID)
 148                 if len(waiting) == 0 {
 149                         close(done)
 150                 }
 151         }
 152         executeContainer := func(ctr arvados.Container) int {
 153                 finishContainer(ctr)
 154                 return int(rand.Uint32() & 0x3)
 155         }
 156         n := 0
 157         s.stubDriver.Queue = queue
 158         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 159                 n++
 160                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 161                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 162                 stubvm.ExecuteContainer = executeContainer
 163                 stubvm.CrashRunningContainer = finishContainer
 164                 switch n % 7 {
 165                 case 0:
 166                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 167                 case 1:
 168                         stubvm.CrunchRunMissing = true
 169                 case 2:
 170                         stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
 171                 default:
 172                         stubvm.CrunchRunCrashRate = 0.1
 173                         stubvm.ArvMountDeadlockRate = 0.1
 174                 }
 175         }
 176         s.stubDriver.Bugf = c.Errorf
 177
 178         start := time.Now()
 179         go s.disp.run()
 180         err := s.disp.CheckHealth()
 181         c.Check(err, check.IsNil)
 182
 183         select {
 184         case <-done:
 185                 c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 186         case <-time.After(10 * time.Second):
 187                 c.Fatalf("timed out; still waiting for %d containers: %q", len(waiting), waiting)
 188         }
 189
 190         deadline := time.Now().Add(5 * time.Second)
 191         for range time.NewTicker(10 * time.Millisecond).C {
 192                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 193                 c.Check(err, check.IsNil)
 194                 queue.Update()
 195                 ents, _ := queue.Entries()
 196                 if len(ents) == 0 && len(insts) == 0 {
 197                         break
 198                 }
 199                 if time.Now().After(deadline) {
 200                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 201                 }
 202         }
 203
 204         req := httptest.NewRequest("GET", "/metrics", nil)
 205         req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
 206         resp := httptest.NewRecorder()
 207         s.disp.ServeHTTP(resp, req)
 208         c.Check(resp.Code, check.Equals, http.StatusOK)
 209         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
 210         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
 211         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
 212         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
 213         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
 214         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} 0.*`)
 215         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
 216         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
 217         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
 218         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
 219         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
 220         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
 221         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
 222         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
 223         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
 224         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
 225         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
 226         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`)
 227         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`)
 228         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`)
 229         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`)
 230 }
 231
 232 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 233         s.cluster.ManagementToken = "abcdefgh"
 234         Drivers["test"] = s.stubDriver
 235         s.disp.setupOnce.Do(s.disp.initialize)
 236         s.disp.queue = &test.Queue{}
 237         go s.disp.run()
 238
 239         for _, token := range []string{"abc", ""} {
 240                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 241                 if token != "" {
 242                         req.Header.Set("Authorization", "Bearer "+token)
 243                 }
 244                 resp := httptest.NewRecorder()
 245                 s.disp.ServeHTTP(resp, req)
 246                 if token == "" {
 247                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 248                 } else {
 249                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 250                 }
 251         }
 252 }
 253
 254 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 255         s.cluster.ManagementToken = ""
 256         Drivers["test"] = s.stubDriver
 257         s.disp.setupOnce.Do(s.disp.initialize)
 258         s.disp.queue = &test.Queue{}
 259         go s.disp.run()
 260
 261         for _, token := range []string{"abc", ""} {
 262                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 263                 if token != "" {
 264                         req.Header.Set("Authorization", "Bearer "+token)
 265                 }
 266                 resp := httptest.NewRecorder()
 267                 s.disp.ServeHTTP(resp, req)
 268                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 269         }
 270 }
 271
 272 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 273         s.cluster.ManagementToken = "abcdefgh"
 274         s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 275         Drivers["test"] = s.stubDriver
 276         s.disp.setupOnce.Do(s.disp.initialize)
 277         s.disp.queue = &test.Queue{}
 278         go s.disp.run()
 279
 280         type instance struct {
 281                 Instance             string
 282                 WorkerState          string `json:"worker_state"`
 283                 Price                float64
 284                 LastContainerUUID    string `json:"last_container_uuid"`
 285                 ArvadosInstanceType  string `json:"arvados_instance_type"`
 286                 ProviderInstanceType string `json:"provider_instance_type"`
 287         }
 288         type instancesResponse struct {
 289                 Items []instance
 290         }
 291         getInstances := func() instancesResponse {
 292                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 293                 req.Header.Set("Authorization", "Bearer abcdefgh")
 294                 resp := httptest.NewRecorder()
 295                 s.disp.ServeHTTP(resp, req)
 296                 var sr instancesResponse
 297                 c.Check(resp.Code, check.Equals, http.StatusOK)
 298                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 299                 c.Check(err, check.IsNil)
 300                 return sr
 301         }
 302
 303         sr := getInstances()
 304         c.Check(len(sr.Items), check.Equals, 0)
 305
 306         ch := s.disp.pool.Subscribe()
 307         defer s.disp.pool.Unsubscribe(ch)
 308         ok := s.disp.pool.Create(test.InstanceType(1))
 309         c.Check(ok, check.Equals, true)
 310         <-ch
 311
 312         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
 313                 sr = getInstances()
 314                 if len(sr.Items) > 0 {
 315                         break
 316                 }
 317                 time.Sleep(time.Millisecond)
 318         }
 319         c.Assert(len(sr.Items), check.Equals, 1)
 320         c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
 321         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 322         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 323         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 324         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 325         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 326 }