X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5e3f0c9b8c87c0ace05ea08ff1ab6434cb333f07..2e727c5d2d000faa6f1d9a566dc59568f1b276fe:/lib/cloud/ec2/ec2.go?ds=sidebyside diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go index 115c1a0c09..52b73f781c 100644 --- a/lib/cloud/ec2/ec2.go +++ b/lib/cloud/ec2/ec2.go @@ -5,64 +5,134 @@ package ec2 import ( + "crypto/md5" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" "encoding/base64" "encoding/json" - "log" - "strings" - - "git.curoverse.com/arvados.git/lib/cloud" - "git.curoverse.com/arvados.git/sdk/go/arvados" + "fmt" + "math/big" + "sync" + "sync/atomic" + "time" + + "git.arvados.org/arvados.git/lib/cloud" + "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) -const ARVADOS_DISPATCH_ID = "arvados-dispatch-id" -const TAG_PREFIX = "disispatch-" - // Driver is the ec2 implementation of the cloud.Driver interface. var Driver = cloud.DriverFunc(newEC2InstanceSet) +const ( + throttleDelayMin = time.Second + throttleDelayMax = time.Minute +) + type ec2InstanceSetConfig struct { - AccessKeyID string - SecretAccessKey string - Region string - SecurityGroupId string - SubnetId string - AdminUsername string - KeyPairName string + AccessKeyID string + SecretAccessKey string + Region string + SecurityGroupIDs arvados.StringSet + SubnetID string + AdminUsername string + EBSVolumeType string + IAMInstanceProfile string +} + +type ec2Interface interface { + DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) + ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) + RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) + DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) + CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) + TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) } type ec2InstanceSet struct { - ec2config ec2InstanceSetConfig - dispatcherID cloud.InstanceSetID - logger logrus.FieldLogger - client *ec2.EC2 - importedKey bool + ec2config ec2InstanceSetConfig + instanceSetID cloud.InstanceSetID + logger logrus.FieldLogger + client ec2Interface + keysMtx sync.Mutex + keys map[string]string + throttleDelayCreate atomic.Value + throttleDelayInstances atomic.Value } -func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) { +func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) { instanceSet := &ec2InstanceSet{ - dispatcherID: dispatcherID, - logger: logger, + instanceSetID: instanceSetID, + logger: logger, } err = json.Unmarshal(config, &instanceSet.ec2config) if err != nil { return nil, err } - awsConfig := aws.NewConfig(). - WithCredentials(credentials.NewStaticCredentials( - instanceSet.ec2config.AccessKeyID, - instanceSet.ec2config.SecretAccessKey, - "")). - WithRegion(instanceSet.ec2config.Region) + + sess, err := session.NewSession() + if err != nil { + return nil, err + } + // First try any static credentials, fall back to an IAM instance profile/role + creds := credentials.NewChainCredentials( + []credentials.Provider{ + &credentials.StaticProvider{Value: credentials.Value{AccessKeyID: instanceSet.ec2config.AccessKeyID, SecretAccessKey: instanceSet.ec2config.SecretAccessKey}}, + &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess)}, + }) + + awsConfig := aws.NewConfig().WithCredentials(creds).WithRegion(instanceSet.ec2config.Region) instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig))) + instanceSet.keys = make(map[string]string) + if instanceSet.ec2config.EBSVolumeType == "" { + instanceSet.ec2config.EBSVolumeType = "gp2" + } return instanceSet, nil } +func awsKeyFingerprint(pk ssh.PublicKey) (md5fp string, sha1fp string, err error) { + // AWS key fingerprints don't use the usual key fingerprint + // you get from ssh-keygen or ssh.FingerprintLegacyMD5() + // (you can get that from md5.Sum(pk.Marshal()) + // + // AWS uses the md5 or sha1 of the PKIX DER encoding of the + // public key, so calculate those fingerprints here. + var rsaPub struct { + Name string + E *big.Int + N *big.Int + } + if err := ssh.Unmarshal(pk.Marshal(), &rsaPub); err != nil { + return "", "", fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err) + } + rsaPk := rsa.PublicKey{ + E: int(rsaPub.E.Int64()), + N: rsaPub.N, + } + pkix, _ := x509.MarshalPKIXPublicKey(&rsaPk) + md5pkix := md5.Sum([]byte(pkix)) + sha1pkix := sha1.Sum([]byte(pkix)) + md5fp = "" + sha1fp = "" + for i := 0; i < len(md5pkix); i++ { + md5fp += fmt.Sprintf(":%02x", md5pkix[i]) + } + for i := 0; i < len(sha1pkix); i++ { + sha1fp += fmt.Sprintf(":%02x", sha1pkix[i]) + } + return md5fp[1:], sha1fp[1:], nil +} + func (instanceSet *ec2InstanceSet) Create( instanceType arvados.InstanceType, imageID cloud.ImageID, @@ -70,52 +140,105 @@ func (instanceSet *ec2InstanceSet) Create( initCommand cloud.InitCommand, publicKey ssh.PublicKey) (cloud.Instance, error) { - if !instanceSet.importedKey { - instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{ - KeyName: &instanceSet.ec2config.KeyPairName, - PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey), - }) - instanceSet.importedKey = true + md5keyFingerprint, sha1keyFingerprint, err := awsKeyFingerprint(publicKey) + if err != nil { + return nil, fmt.Errorf("Could not make key fingerprint: %v", err) } + instanceSet.keysMtx.Lock() + var keyname string + var ok bool + if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok { + keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{ + Filters: []*ec2.Filter{{ + Name: aws.String("fingerprint"), + Values: []*string{&md5keyFingerprint, &sha1keyFingerprint}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("Could not search for keypair: %v", err) + } - ec2tags := []*ec2.Tag{ - &ec2.Tag{ - Key: aws.String(ARVADOS_DISPATCH_ID), - Value: aws.String(string(instanceSet.dispatcherID)), - }, + if len(keyout.KeyPairs) > 0 { + keyname = *(keyout.KeyPairs[0].KeyName) + } else { + keyname = "arvados-dispatch-keypair-" + md5keyFingerprint + _, err := instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{ + KeyName: &keyname, + PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey), + }) + if err != nil { + return nil, fmt.Errorf("Could not import keypair: %v", err) + } + } + instanceSet.keys[md5keyFingerprint] = keyname } + instanceSet.keysMtx.Unlock() + + ec2tags := []*ec2.Tag{} for k, v := range newTags { ec2tags = append(ec2tags, &ec2.Tag{ - Key: aws.String(TAG_PREFIX + k), + Key: aws.String(k), Value: aws.String(v), }) } - rsv, err := instanceSet.client.RunInstances(&ec2.RunInstancesInput{ + var groups []string + for sg := range instanceSet.ec2config.SecurityGroupIDs { + groups = append(groups, sg) + } + + rii := ec2.RunInstancesInput{ ImageId: aws.String(string(imageID)), InstanceType: &instanceType.ProviderType, MaxCount: aws.Int64(1), MinCount: aws.Int64(1), - KeyName: &instanceSet.ec2config.KeyPairName, + KeyName: &keyname, NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ - &ec2.InstanceNetworkInterfaceSpecification{ + { AssociatePublicIpAddress: aws.Bool(false), DeleteOnTermination: aws.Bool(true), DeviceIndex: aws.Int64(0), - Groups: []*string{&instanceSet.ec2config.SecurityGroupId}, - SubnetId: &instanceSet.ec2config.SubnetId, + Groups: aws.StringSlice(groups), + SubnetId: &instanceSet.ec2config.SubnetID, }}, DisableApiTermination: aws.Bool(false), InstanceInitiatedShutdownBehavior: aws.String("terminate"), - UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))), TagSpecifications: []*ec2.TagSpecification{ - &ec2.TagSpecification{ + { ResourceType: aws.String("instance"), Tags: ec2tags, }}, - }) + UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))), + } + + if instanceType.AddedScratch > 0 { + rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvdt"), + Ebs: &ec2.EbsBlockDevice{ + DeleteOnTermination: aws.Bool(true), + VolumeSize: aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30), + VolumeType: &instanceSet.ec2config.EBSVolumeType, + }}} + } + + if instanceType.Preemptible { + rii.InstanceMarketOptions = &ec2.InstanceMarketOptionsRequest{ + MarketType: aws.String("spot"), + SpotOptions: &ec2.SpotMarketOptions{ + InstanceInterruptionBehavior: aws.String("terminate"), + MaxPrice: aws.String(fmt.Sprintf("%v", instanceType.Price)), + }} + } + + if instanceSet.ec2config.IAMInstanceProfile != "" { + rii.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{ + Name: aws.String(instanceSet.ec2config.IAMInstanceProfile), + } + } + rsv, err := instanceSet.client.RunInstances(&rii) + err = wrapError(err, &instanceSet.throttleDelayCreate) if err != nil { return nil, err } @@ -126,22 +249,27 @@ func (instanceSet *ec2InstanceSet) Create( }, nil } -func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cloud.Instance, err error) { - dii := &ec2.DescribeInstancesInput{ - Filters: []*ec2.Filter{&ec2.Filter{ - Name: aws.String("tag:" + ARVADOS_DISPATCH_ID), - Values: []*string{aws.String(string(instanceSet.dispatcherID))}, - }}} - +func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances []cloud.Instance, err error) { + var filters []*ec2.Filter + for k, v := range tags { + filters = append(filters, &ec2.Filter{ + Name: aws.String("tag:" + k), + Values: []*string{aws.String(v)}, + }) + } + dii := &ec2.DescribeInstancesInput{Filters: filters} for { dio, err := instanceSet.client.DescribeInstances(dii) + err = wrapError(err, &instanceSet.throttleDelayInstances) if err != nil { return nil, err } for _, rsv := range dio.Reservations { for _, inst := range rsv.Instances { - instances = append(instances, &ec2Instance{instanceSet, inst}) + if *inst.State.Name != "shutting-down" && *inst.State.Name != "terminated" { + instances = append(instances, &ec2Instance{instanceSet, inst}) + } } } if dio.NextToken == nil { @@ -151,7 +279,7 @@ func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cl } } -func (az *ec2InstanceSet) Stop() { +func (instanceSet *ec2InstanceSet) Stop() { } type ec2Instance struct { @@ -172,15 +300,10 @@ func (inst *ec2Instance) ProviderType() string { } func (inst *ec2Instance) SetTags(newTags cloud.InstanceTags) error { - ec2tags := []*ec2.Tag{ - &ec2.Tag{ - Key: aws.String(ARVADOS_DISPATCH_ID), - Value: aws.String(string(inst.provider.dispatcherID)), - }, - } + var ec2tags []*ec2.Tag for k, v := range newTags { ec2tags = append(ec2tags, &ec2.Tag{ - Key: aws.String(TAG_PREFIX + k), + Key: aws.String(k), Value: aws.String(v), }) } @@ -197,16 +320,13 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags { tags := make(map[string]string) for _, t := range inst.instance.Tags { - if strings.HasPrefix(*t.Key, TAG_PREFIX) { - tags[(*t.Key)[len(TAG_PREFIX):]] = *t.Value - } + tags[*t.Key] = *t.Value } return tags } func (inst *ec2Instance) Destroy() error { - log.Printf("terminating %v", *inst.instance.InstanceId) _, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{ InstanceIds: []*string{inst.instance.InstanceId}, }) @@ -216,9 +336,8 @@ func (inst *ec2Instance) Destroy() error { func (inst *ec2Instance) Address() string { if inst.instance.PrivateIpAddress != nil { return *inst.instance.PrivateIpAddress - } else { - return "" } + return "" } func (inst *ec2Instance) RemoteUser() string { @@ -228,3 +347,60 @@ func (inst *ec2Instance) RemoteUser() string { func (inst *ec2Instance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error { return cloud.ErrNotImplemented } + +type rateLimitError struct { + error + earliestRetry time.Time +} + +func (err rateLimitError) EarliestRetry() time.Time { + return err.earliestRetry +} + +var isCodeCapacity = map[string]bool{ + "InsufficientInstanceCapacity": true, + "VcpuLimitExceeded": true, + "MaxSpotInstanceCountExceeded": true, +} + +// isErrorCapacity returns whether the error is to be throttled based on its code. +// Returns false if error is nil. +func isErrorCapacity(err error) bool { + if aerr, ok := err.(awserr.Error); ok && aerr != nil { + if _, ok := isCodeCapacity[aerr.Code()]; ok { + return true + } + } + return false +} + +type ec2QuotaError struct { + error +} + +func (er *ec2QuotaError) IsQuotaError() bool { + return true +} + +func wrapError(err error, throttleValue *atomic.Value) error { + if request.IsErrorThrottle(err) { + // Back off exponentially until an upstream call + // either succeeds or returns a non-throttle error. + d, _ := throttleValue.Load().(time.Duration) + d = d*3/2 + time.Second + if d < throttleDelayMin { + d = throttleDelayMin + } else if d > throttleDelayMax { + d = throttleDelayMax + } + throttleValue.Store(d) + return rateLimitError{error: err, earliestRetry: time.Now().Add(d)} + } else if isErrorCapacity(err) { + return &ec2QuotaError{err} + } else if err != nil { + throttleValue.Store(time.Duration(0)) + return err + } + throttleValue.Store(time.Duration(0)) + return nil +}