"git.arvados.org/arvados.git/lib/cloud"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
}
type ec2InstanceSet struct {
- ec2config ec2InstanceSetConfig
- instanceSetID cloud.InstanceSetID
- logger logrus.FieldLogger
- client ec2Interface
- keysMtx sync.Mutex
- keys map[string]string
- throttleDelay atomic.Value
+ ec2config ec2InstanceSetConfig
+ instanceSetID cloud.InstanceSetID
+ logger logrus.FieldLogger
+ client ec2Interface
+ keysMtx sync.Mutex
+ keys map[string]string
+ throttleDelayCreate atomic.Value
+ throttleDelayInstances atomic.Value
}
func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
}
rsv, err := instanceSet.client.RunInstances(&rii)
-
- if request.IsErrorThrottle(err) {
- // Back off exponentially until a create call either
- // succeeds or returns a non-throttle error.
- d, _ := instanceSet.throttleDelay.Load().(time.Duration)
- d = d*3/2 + time.Second
- if d < throttleDelayMin {
- d = throttleDelayMin
- } else if d > throttleDelayMax {
- d = throttleDelayMax
- }
- instanceSet.throttleDelay.Store(d)
- return nil, rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
- } else if err != nil {
- instanceSet.throttleDelay.Store(time.Duration(0))
+ err = wrapError(err, &instanceSet.throttleDelayCreate)
+ if err != nil {
return nil, err
- } else {
- instanceSet.throttleDelay.Store(time.Duration(0))
}
return &ec2Instance{
dii := &ec2.DescribeInstancesInput{Filters: filters}
for {
dio, err := instanceSet.client.DescribeInstances(dii)
+ err = wrapError(err, &instanceSet.throttleDelayInstances)
if err != nil {
return nil, err
}
func (err rateLimitError) EarliestRetry() time.Time {
return err.earliestRetry
}
+
+var isCodeCapacity = map[string]bool{
+ "InsufficientInstanceCapacity": true,
+ "VcpuLimitExceeded": true,
+ "MaxSpotInstanceCountExceeded": true,
+}
+
+// isErrorCapacity returns whether the error is to be throttled based on its code.
+// Returns false if error is nil.
+func isErrorCapacity(err error) bool {
+ if aerr, ok := err.(awserr.Error); ok && aerr != nil {
+ if _, ok := isCodeCapacity[aerr.Code()]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+type ec2QuotaError struct {
+ error
+}
+
+func (er *ec2QuotaError) IsQuotaError() bool {
+ return true
+}
+
+func wrapError(err error, throttleValue *atomic.Value) error {
+ if request.IsErrorThrottle(err) {
+ // Back off exponentially until an upstream call
+ // either succeeds or returns a non-throttle error.
+ d, _ := throttleValue.Load().(time.Duration)
+ d = d*3/2 + time.Second
+ if d < throttleDelayMin {
+ d = throttleDelayMin
+ } else if d > throttleDelayMax {
+ d = throttleDelayMax
+ }
+ throttleValue.Store(d)
+ return rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
+ } else if isErrorCapacity(err) {
+ return &ec2QuotaError{err}
+ } else if err != nil {
+ throttleValue.Store(time.Duration(0))
+ return err
+ }
+ throttleValue.Store(time.Duration(0))
+ return nil
+}