X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dc6c729a9e738d876642f2153e1fedd05878245a..be28c5f528a93ee32eef4c1dc2d0872cb718b29f:/lib/cloud/ec2/ec2.go

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index d970993676..c329c1f88a 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -13,11 +13,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"math/big"
-	"strings"
 	"sync"
 
-	"git.curoverse.com/arvados.git/lib/cloud"
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/lib/cloud"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -26,9 +25,6 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-const arvadosDispatchID = "arvados-dispatch-id"
-const tagPrefix = "arvados-dispatch-tag-"
-
 // Driver is the ec2 implementation of the cloud.Driver interface.
 var Driver = cloud.DriverFunc(newEC2InstanceSet)
 
@@ -36,10 +32,10 @@ type ec2InstanceSetConfig struct {
 	AccessKeyID      string
 	SecretAccessKey  string
 	Region           string
-	SecurityGroupIDs []string
+	SecurityGroupIDs arvados.StringSet
 	SubnetID         string
 	AdminUsername    string
-	EbsVolumeType    string
+	EBSVolumeType    string
 }
 
 type ec2Interface interface {
@@ -52,18 +48,18 @@ type ec2Interface interface {
 }
 
 type ec2InstanceSet struct {
-	ec2config    ec2InstanceSetConfig
-	dispatcherID cloud.InstanceSetID
-	logger       logrus.FieldLogger
-	client       ec2Interface
-	keysMtx      sync.Mutex
-	keys         map[string]string
+	ec2config     ec2InstanceSetConfig
+	instanceSetID cloud.InstanceSetID
+	logger        logrus.FieldLogger
+	client        ec2Interface
+	keysMtx       sync.Mutex
+	keys          map[string]string
 }
 
-func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
+func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
 	instanceSet := &ec2InstanceSet{
-		dispatcherID: dispatcherID,
-		logger:       logger,
+		instanceSetID: instanceSetID,
+		logger:        logger,
 	}
 	err = json.Unmarshal(config, &instanceSet.ec2config)
 	if err != nil {
@@ -77,8 +73,8 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
 		WithRegion(instanceSet.ec2config.Region)
 	instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
 	instanceSet.keys = make(map[string]string)
-	if instanceSet.ec2config.EbsVolumeType == "" {
-		instanceSet.ec2config.EbsVolumeType = "gp2"
+	if instanceSet.ec2config.EBSVolumeType == "" {
+		instanceSet.ec2config.EBSVolumeType = "gp2"
 	}
 	return instanceSet, nil
 }
@@ -132,7 +128,7 @@ func (instanceSet *ec2InstanceSet) Create(
 	var ok bool
 	if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok {
 		keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{
-			Filters: []*ec2.Filter{&ec2.Filter{
+			Filters: []*ec2.Filter{{
 				Name:   aws.String("fingerprint"),
 				Values: []*string{&md5keyFingerprint, &sha1keyFingerprint},
 			}},
@@ -157,23 +153,19 @@ func (instanceSet *ec2InstanceSet) Create(
 	}
 	instanceSet.keysMtx.Unlock()
 
-	ec2tags := []*ec2.Tag{
-		&ec2.Tag{
-			Key:   aws.String(arvadosDispatchID),
-			Value: aws.String(string(instanceSet.dispatcherID)),
-		},
-		&ec2.Tag{
-			Key:   aws.String("arvados-class"),
-			Value: aws.String("dynamic-compute"),
-		},
-	}
+	ec2tags := []*ec2.Tag{}
 	for k, v := range newTags {
 		ec2tags = append(ec2tags, &ec2.Tag{
-			Key:   aws.String(tagPrefix + k),
+			Key:   aws.String(k),
 			Value: aws.String(v),
 		})
 	}
 
+	var groups []string
+	for sg := range instanceSet.ec2config.SecurityGroupIDs {
+		groups = append(groups, sg)
+	}
+
 	rii := ec2.RunInstancesInput{
 		ImageId:      aws.String(string(imageID)),
 		InstanceType: &instanceType.ProviderType,
@@ -182,30 +174,30 @@ func (instanceSet *ec2InstanceSet) Create(
 		KeyName:      &keyname,
 
 		NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
-			&ec2.InstanceNetworkInterfaceSpecification{
+			{
 				AssociatePublicIpAddress: aws.Bool(false),
 				DeleteOnTermination:      aws.Bool(true),
 				DeviceIndex:              aws.Int64(0),
-				Groups:                   aws.StringSlice(instanceSet.ec2config.SecurityGroupIDs),
+				Groups:                   aws.StringSlice(groups),
 				SubnetId:                 &instanceSet.ec2config.SubnetID,
 			}},
 		DisableApiTermination:             aws.Bool(false),
 		InstanceInitiatedShutdownBehavior: aws.String("terminate"),
-		UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))),
 		TagSpecifications: []*ec2.TagSpecification{
-			&ec2.TagSpecification{
+			{
 				ResourceType: aws.String("instance"),
 				Tags:         ec2tags,
 			}},
+		UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))),
 	}
 
 	if instanceType.AddedScratch > 0 {
-		rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{&ec2.BlockDeviceMapping{
+		rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{
 			DeviceName: aws.String("/dev/xvdt"),
 			Ebs: &ec2.EbsBlockDevice{
 				DeleteOnTermination: aws.Bool(true),
 				VolumeSize:          aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
-				VolumeType:          &instanceSet.ec2config.EbsVolumeType,
+				VolumeType:          &instanceSet.ec2config.EBSVolumeType,
 			}}}
 	}
 
@@ -230,13 +222,15 @@ func (instanceSet *ec2InstanceSet) Create(
 	}, nil
 }
 
-func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cloud.Instance, err error) {
-	dii := &ec2.DescribeInstancesInput{
-		Filters: []*ec2.Filter{&ec2.Filter{
-			Name:   aws.String("tag:" + arvadosDispatchID),
-			Values: []*string{aws.String(string(instanceSet.dispatcherID))},
-		}}}
-
+func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances []cloud.Instance, err error) {
+	var filters []*ec2.Filter
+	for k, v := range tags {
+		filters = append(filters, &ec2.Filter{
+			Name:   aws.String("tag:" + k),
+			Values: []*string{aws.String(v)},
+		})
+	}
+	dii := &ec2.DescribeInstancesInput{Filters: filters}
 	for {
 		dio, err := instanceSet.client.DescribeInstances(dii)
 		if err != nil {
@@ -245,7 +239,7 @@ func (instanceSet *ec2InstanceSet) Instances(cloud.InstanceTags) (instances []cl
 
 		for _, rsv := range dio.Reservations {
 			for _, inst := range rsv.Instances {
-				if *inst.State.Name != "terminated" {
+				if *inst.State.Name != "shutting-down" && *inst.State.Name != "terminated" {
 					instances = append(instances, &ec2Instance{instanceSet, inst})
 				}
 			}
@@ -278,15 +272,10 @@ func (inst *ec2Instance) ProviderType() string {
 }
 
 func (inst *ec2Instance) SetTags(newTags cloud.InstanceTags) error {
-	ec2tags := []*ec2.Tag{
-		&ec2.Tag{
-			Key:   aws.String(arvadosDispatchID),
-			Value: aws.String(string(inst.provider.dispatcherID)),
-		},
-	}
+	var ec2tags []*ec2.Tag
 	for k, v := range newTags {
 		ec2tags = append(ec2tags, &ec2.Tag{
-			Key:   aws.String(tagPrefix + k),
+			Key:   aws.String(k),
 			Value: aws.String(v),
 		})
 	}
@@ -303,9 +292,7 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags {
 	tags := make(map[string]string)
 
 	for _, t := range inst.instance.Tags {
-		if strings.HasPrefix(*t.Key, tagPrefix) {
-			tags[(*t.Key)[len(tagPrefix):]] = *t.Value
-		}
+		tags[*t.Key] = *t.Value
 	}
 
 	return tags