14807: Configurable rate limit for cloud provider API calls.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 44d5a0ae75b2b8b4400bdb090e0dfc9baaf90198..d1ee4b135e217e4cb2ee90f9d4539646ee5b0a09 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -49,12 +49,13 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 
        s.cluster = &arvados.Cluster{
                CloudVMs: arvados.CloudVMs{
-                       Driver:          "test",
-                       SyncInterval:    arvados.Duration(10 * time.Millisecond),
-                       TimeoutIdle:     arvados.Duration(150 * time.Millisecond),
-                       TimeoutBooting:  arvados.Duration(150 * time.Millisecond),
-                       TimeoutProbe:    arvados.Duration(15 * time.Millisecond),
-                       TimeoutShutdown: arvados.Duration(5 * time.Millisecond),
+                       Driver:               "test",
+                       SyncInterval:         arvados.Duration(10 * time.Millisecond),
+                       TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
+                       TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
+                       TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
+                       TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
+                       MaxCloudOpsPerSecond: 500,
                },
                Dispatch: arvados.Dispatch{
                        PrivateKey:         string(dispatchprivraw),

diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index 0343f85b91a7bc63d20034e111e04487608bef9c..eb1e48737c8b131cbb919ca71e8f6bbc377c553a 100644 (file)
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -6,12 +6,14 @@ package dispatchcloud
 
 import (
        "fmt"
+       "time"
 
        "git.curoverse.com/arvados.git/lib/cloud"
        "git.curoverse.com/arvados.git/lib/cloud/azure"
        "git.curoverse.com/arvados.git/lib/cloud/ec2"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "github.com/sirupsen/logrus"
+       "golang.org/x/crypto/ssh"
 )
 
 var drivers = map[string]cloud.Driver{
@@ -24,5 +26,33 @@ func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID, logger
        if !ok {
                return nil, fmt.Errorf("unsupported cloud driver %q", cluster.CloudVMs.Driver)
        }
-       return driver.InstanceSet(cluster.CloudVMs.DriverParameters, setID, logger)
+       is, err := driver.InstanceSet(cluster.CloudVMs.DriverParameters, setID, logger)
+       if maxops := cluster.CloudVMs.MaxCloudOpsPerSecond; maxops > 0 {
+               is = &rateLimitedInstanceSet{
+                       InstanceSet: is,
+                       ticker:      time.NewTicker(time.Second / time.Duration(maxops)),
+               }
+       }
+       return is, err
+}
+
+type rateLimitedInstanceSet struct {
+       cloud.InstanceSet
+       ticker *time.Ticker
+}
+
+func (is rateLimitedInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, tags cloud.InstanceTags, init cloud.InitCommand, pk ssh.PublicKey) (cloud.Instance, error) {
+       <-is.ticker.C
+       inst, err := is.InstanceSet.Create(it, image, tags, init, pk)
+       return &rateLimitedInstance{inst, is.ticker}, err
+}
+
+type rateLimitedInstance struct {
+       cloud.Instance
+       ticker *time.Ticker
+}
+
+func (inst *rateLimitedInstance) Destroy() error {
+       <-inst.ticker.C
+       return inst.Instance.Destroy()
 }

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 73addb739cd0e25b47212300d420fa4f2b8c8e7d..7c87ff0293052762641019f29d7ff442aa09e75d 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -154,6 +154,9 @@ type CloudVMs struct {
        // Time after shutdown to retry shutdown
        TimeoutShutdown Duration
 
+       // Maximum create/destroy-instance operations per second
+       MaxCloudOpsPerSecond int
+
        ImageID string
 
        Driver           string