package ec2
import (
+ "crypto/md5"
+ "crypto/rsa"
+ "crypto/sha1"
+ "crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
- "log"
- "strings"
+ "math/big"
+ "sync"
+ "sync/atomic"
+ "time"
- "git.curoverse.com/arvados.git/lib/cloud"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/lib/cloud"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+ "github.com/aws/aws-sdk-go/aws/ec2metadata"
+ "github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
)
-const ARVADOS_DISPATCH_ID = "arvados-dispatch-id"
-const TAG_PREFIX = "arvados-dispatch-tag-"
-
// Driver is the ec2 implementation of the cloud.Driver interface.
var Driver = cloud.DriverFunc(newEC2InstanceSet)
+const (
+ throttleDelayMin = time.Second
+ throttleDelayMax = time.Minute
+)
+
type ec2InstanceSetConfig struct {
- AccessKeyID string
- SecretAccessKey string
- Region string
- SecurityGroupId string
- SubnetId string
- AdminUsername string
- KeyPairName string
+ AccessKeyID string
+ SecretAccessKey string
+ Region string
+ SecurityGroupIDs arvados.StringSet
+ SubnetID string
+ AdminUsername string
+ EBSVolumeType string
}
-type EC2Interface interface {
+type ec2Interface interface {
+ DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error)
ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error)
RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error)
DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error)
}
type ec2InstanceSet struct {
- ec2config ec2InstanceSetConfig
- dispatcherID cloud.InstanceSetID
- logger logrus.FieldLogger
- client EC2Interface
- importedKey bool
+ ec2config ec2InstanceSetConfig
+ instanceSetID cloud.InstanceSetID
+ logger logrus.FieldLogger
+ client ec2Interface
+ keysMtx sync.Mutex
+ keys map[string]string
+ throttleDelayCreate atomic.Value
+ throttleDelayInstances atomic.Value
}
-func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
+func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
instanceSet := &ec2InstanceSet{
- dispatcherID: dispatcherID,
- logger: logger,
+ instanceSetID: instanceSetID,
+ logger: logger,
}
err = json.Unmarshal(config, &instanceSet.ec2config)
if err != nil {
return nil, err
}
- awsConfig := aws.NewConfig().
- WithCredentials(credentials.NewStaticCredentials(
- instanceSet.ec2config.AccessKeyID,
- instanceSet.ec2config.SecretAccessKey,
- "")).
- WithRegion(instanceSet.ec2config.Region)
+
+ sess, err := session.NewSession()
+ if err != nil {
+ return nil, err
+ }
+ // First try any static credentials, fall back to an IAM instance profile/role
+ creds := credentials.NewChainCredentials(
+ []credentials.Provider{
+ &credentials.StaticProvider{Value: credentials.Value{AccessKeyID: instanceSet.ec2config.AccessKeyID, SecretAccessKey: instanceSet.ec2config.SecretAccessKey}},
+ &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess)},
+ })
+
+ awsConfig := aws.NewConfig().WithCredentials(creds).WithRegion(instanceSet.ec2config.Region)
instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
+ instanceSet.keys = make(map[string]string)
+ if instanceSet.ec2config.EBSVolumeType == "" {
+ instanceSet.ec2config.EBSVolumeType = "gp2"
+ }
return instanceSet, nil
}
+func awsKeyFingerprint(pk ssh.PublicKey) (md5fp string, sha1fp string, err error) {
+ // AWS key fingerprints don't use the usual key fingerprint
+ // you get from ssh-keygen or ssh.FingerprintLegacyMD5()
+ // (you can get that from md5.Sum(pk.Marshal())
+ //
+ // AWS uses the md5 or sha1 of the PKIX DER encoding of the
+ // public key, so calculate those fingerprints here.
+ var rsaPub struct {
+ Name string
+ E *big.Int
+ N *big.Int
+ }
+ if err := ssh.Unmarshal(pk.Marshal(), &rsaPub); err != nil {
+ return "", "", fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err)
+ }
+ rsaPk := rsa.PublicKey{
+ E: int(rsaPub.E.Int64()),
+ N: rsaPub.N,
+ }
+ pkix, _ := x509.MarshalPKIXPublicKey(&rsaPk)
+ md5pkix := md5.Sum([]byte(pkix))
+ sha1pkix := sha1.Sum([]byte(pkix))
+ md5fp = ""
+ sha1fp = ""
+ for i := 0; i < len(md5pkix); i++ {
+ md5fp += fmt.Sprintf(":%02x", md5pkix[i])
+ }
+ for i := 0; i < len(sha1pkix); i++ {
+ sha1fp += fmt.Sprintf(":%02x", sha1pkix[i])
+ }
+ return md5fp[1:], sha1fp[1:], nil
+}
+
func (instanceSet *ec2InstanceSet) Create(
instanceType arvados.InstanceType,
imageID cloud.ImageID,
initCommand cloud.InitCommand,
publicKey ssh.PublicKey) (cloud.Instance, error) {
- if !instanceSet.importedKey {
- instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
- KeyName: &instanceSet.ec2config.KeyPairName,
- PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
- })
- instanceSet.importedKey = true
+ md5keyFingerprint, sha1keyFingerprint, err := awsKeyFingerprint(publicKey)
+ if err != nil {
+ return nil, fmt.Errorf("Could not make key fingerprint: %v", err)
}
+ instanceSet.keysMtx.Lock()
+ var keyname string
+ var ok bool
+ if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok {
+ keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{
+ Filters: []*ec2.Filter{{
+ Name: aws.String("fingerprint"),
+ Values: []*string{&md5keyFingerprint, &sha1keyFingerprint},
+ }},
+ })
+ if err != nil {
+ return nil, fmt.Errorf("Could not search for keypair: %v", err)
+ }
- ec2tags := []*ec2.Tag{
- &ec2.Tag{
- Key: aws.String(ARVADOS_DISPATCH_ID),
- Value: aws.String(string(instanceSet.dispatcherID)),
- },
- &ec2.Tag{
- Key: aws.String("arvados-class"),
- Value: aws.String("dynamic-compute"),
- },
+ if len(keyout.KeyPairs) > 0 {
+ keyname = *(keyout.KeyPairs[0].KeyName)
+ } else {
+ keyname = "arvados-dispatch-keypair-" + md5keyFingerprint
+ _, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+ KeyName: &keyname,
+ PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("Could not import keypair: %v", err)
+ }
+ }
+ instanceSet.keys[md5keyFingerprint] = keyname
}
+ instanceSet.keysMtx.Unlock()
+
+ ec2tags := []*ec2.Tag{}
for k, v := range newTags {
ec2tags = append(ec2tags, &ec2.Tag{
- Key: aws.String(TAG_PREFIX + k),
+ Key: aws.String(k),
Value: aws.String(v),
})
}
+ var groups []string
+ for sg := range instanceSet.ec2config.SecurityGroupIDs {
+ groups = append(groups, sg)
+ }
+
rii := ec2.RunInstancesInput{
ImageId: aws.String(string(imageID)),
InstanceType: &instanceType.ProviderType,
MaxCount: aws.Int64(1),
MinCount: aws.Int64(1),
- KeyName: &instanceSet.ec2config.KeyPairName,
+ KeyName: &keyname,
NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
- &ec2.InstanceNetworkInterfaceSpecification{
+ {
AssociatePublicIpAddress: aws.Bool(false),
DeleteOnTermination: aws.Bool(true),
DeviceIndex: aws.Int64(0),
- Groups: []*string{&instanceSet.ec2config.SecurityGroupId},
- SubnetId: &instanceSet.ec2config.SubnetId,
+ Groups: aws.StringSlice(groups),
+ SubnetId: &instanceSet.ec2config.SubnetID,
}},
DisableApiTermination: aws.Bool(false),
InstanceInitiatedShutdownBehavior: aws.String("terminate"),
- UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))),
TagSpecifications: []*ec2.TagSpecification{
- &ec2.TagSpecification{
+ {
ResourceType: aws.String("instance"),
Tags: ec2tags,
}},
+ UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))),
}
- if instanceType.AttachScratch {
- rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{&ec2.BlockDeviceMapping{
+ if instanceType.AddedScratch > 0 {
+ rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{
DeviceName: aws.String("/dev/xvdt"),
Ebs: &ec2.EbsBlockDevice{
DeleteOnTermination: aws.Bool(true),
- VolumeSize: aws.Int64((int64(instanceType.Scratch) / 1000000000) + 1),
- VolumeType: aws.String("gp2"),
+ VolumeSize: aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
+ VolumeType: &instanceSet.ec2config.EBSVolumeType,
}}}
}
}
rsv, err := instanceSet.client.RunInstances(&rii)
-
+ err = wrapError(err, &instanceSet.throttleDelayCreate)
if err != nil {
return nil, err
}
}, nil
}
-func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cloud.Instance, err error) {
- dii := &ec2.DescribeInstancesInput{
- Filters: []*ec2.Filter{&ec2.Filter{
- Name: aws.String("tag:" + ARVADOS_DISPATCH_ID),
- Values: []*string{aws.String(string(instanceSet.dispatcherID))},
- }}}
-
+func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances []cloud.Instance, err error) {
+ var filters []*ec2.Filter
+ for k, v := range tags {
+ filters = append(filters, &ec2.Filter{
+ Name: aws.String("tag:" + k),
+ Values: []*string{aws.String(v)},
+ })
+ }
+ dii := &ec2.DescribeInstancesInput{Filters: filters}
for {
dio, err := instanceSet.client.DescribeInstances(dii)
+ err = wrapError(err, &instanceSet.throttleDelayInstances)
if err != nil {
return nil, err
}
for _, rsv := range dio.Reservations {
for _, inst := range rsv.Instances {
- instances = append(instances, &ec2Instance{instanceSet, inst})
+ if *inst.State.Name != "shutting-down" && *inst.State.Name != "terminated" {
+ instances = append(instances, &ec2Instance{instanceSet, inst})
+ }
}
}
if dio.NextToken == nil {
}
}
-func (az *ec2InstanceSet) Stop() {
+func (instanceSet *ec2InstanceSet) Stop() {
}
type ec2Instance struct {
}
func (inst *ec2Instance) SetTags(newTags cloud.InstanceTags) error {
- ec2tags := []*ec2.Tag{
- &ec2.Tag{
- Key: aws.String(ARVADOS_DISPATCH_ID),
- Value: aws.String(string(inst.provider.dispatcherID)),
- },
- }
+ var ec2tags []*ec2.Tag
for k, v := range newTags {
ec2tags = append(ec2tags, &ec2.Tag{
- Key: aws.String(TAG_PREFIX + k),
+ Key: aws.String(k),
Value: aws.String(v),
})
}
tags := make(map[string]string)
for _, t := range inst.instance.Tags {
- if strings.HasPrefix(*t.Key, TAG_PREFIX) {
- tags[(*t.Key)[len(TAG_PREFIX):]] = *t.Value
- }
+ tags[*t.Key] = *t.Value
}
return tags
}
func (inst *ec2Instance) Destroy() error {
- log.Printf("terminating %v", *inst.instance.InstanceId)
_, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
InstanceIds: []*string{inst.instance.InstanceId},
})
func (inst *ec2Instance) Address() string {
if inst.instance.PrivateIpAddress != nil {
return *inst.instance.PrivateIpAddress
- } else {
- return ""
}
+ return ""
}
func (inst *ec2Instance) RemoteUser() string {
func (inst *ec2Instance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
return cloud.ErrNotImplemented
}
+
+type rateLimitError struct {
+ error
+ earliestRetry time.Time
+}
+
+func (err rateLimitError) EarliestRetry() time.Time {
+ return err.earliestRetry
+}
+
+var isCodeCapacity = map[string]bool{
+ "InsufficientInstanceCapacity": true,
+ "VcpuLimitExceeded": true,
+ "MaxSpotInstanceCountExceeded": true,
+}
+
+// isErrorCapacity returns whether the error is to be throttled based on its code.
+// Returns false if error is nil.
+func isErrorCapacity(err error) bool {
+ if aerr, ok := err.(awserr.Error); ok && aerr != nil {
+ if _, ok := isCodeCapacity[aerr.Code()]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+type ec2QuotaError struct {
+ error
+}
+
+func (er *ec2QuotaError) IsQuotaError() bool {
+ return true
+}
+
+func wrapError(err error, throttleValue *atomic.Value) error {
+ if request.IsErrorThrottle(err) {
+ // Back off exponentially until an upstream call
+ // either succeeds or returns a non-throttle error.
+ d, _ := throttleValue.Load().(time.Duration)
+ d = d*3/2 + time.Second
+ if d < throttleDelayMin {
+ d = throttleDelayMin
+ } else if d > throttleDelayMax {
+ d = throttleDelayMax
+ }
+ throttleValue.Store(d)
+ return rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
+ } else if isErrorCapacity(err) {
+ return &ec2QuotaError{err}
+ } else if err != nil {
+ throttleValue.Store(time.Duration(0))
+ return err
+ }
+ throttleValue.Store(time.Duration(0))
+ return nil
+}
projects / arvados.git / blobdiff