lib/dispatchcloud/dispatcher_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package dispatchcloud
   6
   7 import (
   8         "context"
   9         "encoding/json"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "net/http/httptest"
  14         "os"
  15         "sync"
  16         "time"
  17
  18         "git.arvados.org/arvados.git/lib/config"
  19         "git.arvados.org/arvados.git/lib/dispatchcloud/test"
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  23         "github.com/prometheus/client_golang/prometheus"
  24         "golang.org/x/crypto/ssh"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&DispatcherSuite{})
  29
  30 type DispatcherSuite struct {
  31         ctx        context.Context
  32         cancel     context.CancelFunc
  33         cluster    *arvados.Cluster
  34         stubDriver *test.StubDriver
  35         disp       *dispatcher
  36 }
  37
  38 func (s *DispatcherSuite) SetUpTest(c *check.C) {
  39         s.ctx, s.cancel = context.WithCancel(context.Background())
  40         s.ctx = ctxlog.Context(s.ctx, ctxlog.TestLogger(c))
  41         dispatchpub, _ := test.LoadTestKey(c, "test/sshkey_dispatch")
  42         dispatchprivraw, err := ioutil.ReadFile("test/sshkey_dispatch")
  43         c.Assert(err, check.IsNil)
  44
  45         _, hostpriv := test.LoadTestKey(c, "test/sshkey_vm")
  46         s.stubDriver = &test.StubDriver{
  47                 HostKey:                   hostpriv,
  48                 AuthorizedKeys:            []ssh.PublicKey{dispatchpub},
  49                 ErrorRateDestroy:          0.1,
  50                 MinTimeBetweenCreateCalls: time.Millisecond,
  51         }
  52
  53         // We need the postgresql connection info from the integration
  54         // test config.
  55         cfg, err := config.NewLoader(nil, ctxlog.FromContext(s.ctx)).Load()
  56         c.Assert(err, check.IsNil)
  57         testcluster, err := cfg.GetCluster("")
  58         c.Assert(err, check.IsNil)
  59
  60         s.cluster = &arvados.Cluster{
  61                 ManagementToken: "test-management-token",
  62                 PostgreSQL:      testcluster.PostgreSQL,
  63                 Containers: arvados.ContainersConfig{
  64                         CrunchRunCommand:       "crunch-run",
  65                         CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
  66                         DispatchPrivateKey:     string(dispatchprivraw),
  67                         StaleLockTimeout:       arvados.Duration(5 * time.Millisecond),
  68                         RuntimeEngine:          "stub",
  69                         CloudVMs: arvados.CloudVMsConfig{
  70                                 Driver:               "test",
  71                                 SyncInterval:         arvados.Duration(10 * time.Millisecond),
  72                                 TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
  73                                 TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
  74                                 TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
  75                                 TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
  76                                 MaxCloudOpsPerSecond: 500,
  77                                 PollInterval:         arvados.Duration(5 * time.Millisecond),
  78                                 ProbeInterval:        arvados.Duration(5 * time.Millisecond),
  79                                 MaxProbesPerSecond:   1000,
  80                                 TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
  81                                 TimeoutStaleRunLock:  arvados.Duration(3 * time.Millisecond),
  82                                 TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
  83                                 ResourceTags:         map[string]string{"testtag": "test value"},
  84                                 TagKeyPrefix:         "test:",
  85                         },
  86                 },
  87                 InstanceTypes: arvados.InstanceTypeMap{
  88                         test.InstanceType(1).Name:  test.InstanceType(1),
  89                         test.InstanceType(2).Name:  test.InstanceType(2),
  90                         test.InstanceType(3).Name:  test.InstanceType(3),
  91                         test.InstanceType(4).Name:  test.InstanceType(4),
  92                         test.InstanceType(6).Name:  test.InstanceType(6),
  93                         test.InstanceType(8).Name:  test.InstanceType(8),
  94                         test.InstanceType(16).Name: test.InstanceType(16),
  95                 },
  96         }
  97         arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
  98         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
  99
 100         arvClient, err := arvados.NewClientFromConfig(s.cluster)
 101         c.Check(err, check.IsNil)
 102
 103         s.disp = &dispatcher{
 104                 Cluster:   s.cluster,
 105                 Context:   s.ctx,
 106                 ArvClient: arvClient,
 107                 AuthToken: arvadostest.AdminToken,
 108                 Registry:  prometheus.NewRegistry(),
 109         }
 110         // Test cases can modify s.cluster before calling
 111         // initialize(), and then modify private state before calling
 112         // go run().
 113 }
 114
 115 func (s *DispatcherSuite) TearDownTest(c *check.C) {
 116         s.cancel()
 117         s.disp.Close()
 118 }
 119
 120 // DispatchToStubDriver checks that the dispatcher wires everything
 121 // together effectively. It uses a real scheduler and worker pool with
 122 // a fake queue and cloud driver. The fake cloud driver injects
 123 // artificial errors in order to exercise a variety of code paths.
 124 func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
 125         Drivers["test"] = s.stubDriver
 126         s.disp.setupOnce.Do(s.disp.initialize)
 127         queue := &test.Queue{
 128                 ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
 129                         return ChooseInstanceType(s.cluster, ctr)
 130                 },
 131                 Logger: ctxlog.TestLogger(c),
 132         }
 133         for i := 0; i < 200; i++ {
 134                 queue.Containers = append(queue.Containers, arvados.Container{
 135                         UUID:     test.ContainerUUID(i + 1),
 136                         State:    arvados.ContainerStateQueued,
 137                         Priority: int64(i%20 + 1),
 138                         RuntimeConstraints: arvados.RuntimeConstraints{
 139                                 RAM:   int64(i%3+1) << 30,
 140                                 VCPUs: i%8 + 1,
 141                         },
 142                 })
 143         }
 144         s.disp.queue = queue
 145
 146         var mtx sync.Mutex
 147         done := make(chan struct{})
 148         waiting := map[string]struct{}{}
 149         for _, ctr := range queue.Containers {
 150                 waiting[ctr.UUID] = struct{}{}
 151         }
 152         finishContainer := func(ctr arvados.Container) {
 153                 mtx.Lock()
 154                 defer mtx.Unlock()
 155                 if _, ok := waiting[ctr.UUID]; !ok {
 156                         c.Errorf("container completed twice: %s", ctr.UUID)
 157                         return
 158                 }
 159                 delete(waiting, ctr.UUID)
 160                 if len(waiting) == 0 {
 161                         close(done)
 162                 }
 163         }
 164         executeContainer := func(ctr arvados.Container) int {
 165                 finishContainer(ctr)
 166                 return int(rand.Uint32() & 0x3)
 167         }
 168         n := 0
 169         s.stubDriver.Queue = queue
 170         s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
 171                 n++
 172                 stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
 173                 stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
 174                 stubvm.ExecuteContainer = executeContainer
 175                 stubvm.CrashRunningContainer = finishContainer
 176                 stubvm.ExtraCrunchRunArgs = "'--runtime-engine=stub' '--foo' '--extra='\\''args'\\'''"
 177                 switch n % 7 {
 178                 case 0:
 179                         stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
 180                 case 1:
 181                         stubvm.CrunchRunMissing = true
 182                 case 2:
 183                         stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond)
 184                 default:
 185                         stubvm.CrunchRunCrashRate = 0.1
 186                         stubvm.ArvMountDeadlockRate = 0.1
 187                 }
 188         }
 189         s.stubDriver.Bugf = c.Errorf
 190
 191         start := time.Now()
 192         go s.disp.run()
 193         err := s.disp.CheckHealth()
 194         c.Check(err, check.IsNil)
 195
 196         for len(waiting) > 0 {
 197                 waswaiting := len(waiting)
 198                 select {
 199                 case <-done:
 200                         // loop will end because len(waiting)==0
 201                 case <-time.After(3 * time.Second):
 202                         if len(waiting) >= waswaiting {
 203                                 c.Fatalf("timed out; no progress in 3s while waiting for %d containers: %q", len(waiting), waiting)
 204                         }
 205                 }
 206         }
 207         c.Logf("containers finished (%s), waiting for instances to shutdown and queue to clear", time.Since(start))
 208
 209         deadline := time.Now().Add(5 * time.Second)
 210         for range time.NewTicker(10 * time.Millisecond).C {
 211                 insts, err := s.stubDriver.InstanceSets()[0].Instances(nil)
 212                 c.Check(err, check.IsNil)
 213                 queue.Update()
 214                 ents, _ := queue.Entries()
 215                 if len(ents) == 0 && len(insts) == 0 {
 216                         break
 217                 }
 218                 if time.Now().After(deadline) {
 219                         c.Fatalf("timed out with %d containers (%v), %d instances (%+v)", len(ents), ents, len(insts), insts)
 220                 }
 221         }
 222
 223         req := httptest.NewRequest("GET", "/metrics", nil)
 224         req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
 225         resp := httptest.NewRecorder()
 226         s.disp.ServeHTTP(resp, req)
 227         c.Check(resp.Code, check.Equals, http.StatusOK)
 228         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Create"} [^0].*`)
 229         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="List"} [^0].*`)
 230         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="0",operation="Destroy"} [^0].*`)
 231         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="Create"} [^0].*`)
 232         c.Check(resp.Body.String(), check.Matches, `(?ms).*driver_operations{error="1",operation="List"} 0\n.*`)
 233         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="aborted"} 0.*`)
 234         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="disappeared"} [^0].*`)
 235         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="failure"} [^0].*`)
 236         c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`)
 237         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`)
 238         c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`)
 239         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`)
 240         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`)
 241         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`)
 242         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`)
 243         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`)
 244         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`)
 245         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`)
 246         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`)
 247         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`)
 248         c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`)
 249         c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="success"} [0-9]*`)
 250         c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="success"} [0-9e+.]*`)
 251         c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="fail"} [0-9]*`)
 252         c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="fail"} [0-9e+.]*`)
 253 }
 254
 255 func (s *DispatcherSuite) TestAPIPermissions(c *check.C) {
 256         s.cluster.ManagementToken = "abcdefgh"
 257         Drivers["test"] = s.stubDriver
 258         s.disp.setupOnce.Do(s.disp.initialize)
 259         s.disp.queue = &test.Queue{}
 260         go s.disp.run()
 261
 262         for _, token := range []string{"abc", ""} {
 263                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 264                 if token != "" {
 265                         req.Header.Set("Authorization", "Bearer "+token)
 266                 }
 267                 resp := httptest.NewRecorder()
 268                 s.disp.ServeHTTP(resp, req)
 269                 if token == "" {
 270                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 271                 } else {
 272                         c.Check(resp.Code, check.Equals, http.StatusForbidden)
 273                 }
 274         }
 275 }
 276
 277 func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 278         s.cluster.ManagementToken = ""
 279         Drivers["test"] = s.stubDriver
 280         s.disp.setupOnce.Do(s.disp.initialize)
 281         s.disp.queue = &test.Queue{}
 282         go s.disp.run()
 283
 284         for _, token := range []string{"abc", ""} {
 285                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 286                 if token != "" {
 287                         req.Header.Set("Authorization", "Bearer "+token)
 288                 }
 289                 resp := httptest.NewRecorder()
 290                 s.disp.ServeHTTP(resp, req)
 291                 c.Check(resp.Code, check.Equals, http.StatusForbidden)
 292         }
 293 }
 294
 295 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 296         s.cluster.ManagementToken = "abcdefgh"
 297         s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 298         Drivers["test"] = s.stubDriver
 299         s.disp.setupOnce.Do(s.disp.initialize)
 300         s.disp.queue = &test.Queue{}
 301         go s.disp.run()
 302
 303         type instance struct {
 304                 Instance             string
 305                 WorkerState          string `json:"worker_state"`
 306                 Price                float64
 307                 LastContainerUUID    string `json:"last_container_uuid"`
 308                 ArvadosInstanceType  string `json:"arvados_instance_type"`
 309                 ProviderInstanceType string `json:"provider_instance_type"`
 310         }
 311         type instancesResponse struct {
 312                 Items []instance
 313         }
 314         getInstances := func() instancesResponse {
 315                 req := httptest.NewRequest("GET", "/arvados/v1/dispatch/instances", nil)
 316                 req.Header.Set("Authorization", "Bearer abcdefgh")
 317                 resp := httptest.NewRecorder()
 318                 s.disp.ServeHTTP(resp, req)
 319                 var sr instancesResponse
 320                 c.Check(resp.Code, check.Equals, http.StatusOK)
 321                 err := json.Unmarshal(resp.Body.Bytes(), &sr)
 322                 c.Check(err, check.IsNil)
 323                 return sr
 324         }
 325
 326         sr := getInstances()
 327         c.Check(len(sr.Items), check.Equals, 0)
 328
 329         ch := s.disp.pool.Subscribe()
 330         defer s.disp.pool.Unsubscribe(ch)
 331         ok := s.disp.pool.Create(test.InstanceType(1))
 332         c.Check(ok, check.Equals, true)
 333         <-ch
 334
 335         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline); {
 336                 sr = getInstances()
 337                 if len(sr.Items) > 0 {
 338                         break
 339                 }
 340                 time.Sleep(time.Millisecond)
 341         }
 342         c.Assert(len(sr.Items), check.Equals, 1)
 343         c.Check(sr.Items[0].Instance, check.Matches, "inst.*")
 344         c.Check(sr.Items[0].WorkerState, check.Equals, "booting")
 345         c.Check(sr.Items[0].Price, check.Equals, 0.123)
 346         c.Check(sr.Items[0].LastContainerUUID, check.Equals, "")
 347         c.Check(sr.Items[0].ProviderInstanceType, check.Equals, test.InstanceType(1).ProviderType)
 348         c.Check(sr.Items[0].ArvadosInstanceType, check.Equals, test.InstanceType(1).Name)
 349 }