14291: Add EbsVolumeType, also test fixes

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index caaca1f31e51677c3881dbf82ea9197ff53660c2..095d32eaae39de11bc031619d121bbe16562c2ed 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -933,6 +933,7 @@ gostuff=(
     lib/crunchstat
     lib/cloud
     lib/cloud/azure
+    lib/cloud/ec2
     lib/dispatchcloud
     lib/dispatchcloud/container
     lib/dispatchcloud/scheduler

diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go
index 61649c39800d7ad8e31252313285e0e85fe7b16c..bd82a424f4032fe6995bf436adf1ccf82ef89a91 100644 (file)
--- a/lib/cloud/azure/azure_test.go
+++ b/lib/cloud/azure/azure_test.go
@@ -43,6 +43,7 @@ import (
        "time"
 
        "git.curoverse.com/arvados.git/lib/cloud"
+       "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/config"
        "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
@@ -67,8 +68,6 @@ var _ = check.Suite(&AzureInstanceSetSuite{})
 
 type VirtualMachinesClientStub struct{}
 
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
-
 func (*VirtualMachinesClientStub) createOrUpdate(ctx context.Context,
        resourceGroupName string,
        VMName string,
@@ -157,7 +156,7 @@ func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
                c.Fatal("Error making provider", err)
        }
 
-       pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+       pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
        c.Assert(err, check.IsNil)
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny"],

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 35153b3ff8fd015f43ee45d29a97bd9b38cd1473..464d42a36f2ae565448aa8178abc424249a1b166 100644 (file)
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -8,7 +8,6 @@ import (
        "encoding/base64"
        "encoding/json"
        "fmt"
-       "log"
        "strings"
        "sync"
 
@@ -35,6 +34,7 @@ type ec2InstanceSetConfig struct {
        SecurityGroupIDs []string
        SubnetID         string
        AdminUsername    string
+       EbsVolumeType    string
 }
 
 type ec2Interface interface {
@@ -71,6 +71,9 @@ func newEC2InstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID,
                WithRegion(instanceSet.ec2config.Region)
        instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
        instanceSet.keys = make(map[string]string)
+       if instanceSet.ec2config.EbsVolumeType == "" {
+               instanceSet.ec2config.EbsVolumeType = "gp2"
+       }
        return instanceSet, nil
 }
 
@@ -145,8 +148,8 @@ func (instanceSet *ec2InstanceSet) Create(
                        DeviceName: aws.String("/dev/xvdt"),
                        Ebs: &ec2.EbsBlockDevice{
                                DeleteOnTermination: aws.Bool(true),
-                               VolumeSize:          aws.Int64((int64(instanceType.AddedScratch) / 1000000000) + 1),
-                               VolumeType:          aws.String("gp2"),
+                               VolumeSize:          aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
+                               VolumeType:          &instanceSet.ec2config.EbsVolumeType,
                        }}}
        }
 
@@ -251,7 +254,6 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags {
 }
 
 func (inst *ec2Instance) Destroy() error {
-       log.Printf("terminating %v", *inst.instance.InstanceId)
        _, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
                InstanceIds: []*string{inst.instance.InstanceId},
        })

diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index ba65758cca05438bae0e3b94a640f4e7551e6798..3138cba3d44200161d6688d979199e42f2130fde 100644 (file)
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -25,16 +25,15 @@ package ec2
 import (
        "encoding/json"
        "flag"
-       "log"
        "testing"
 
        "git.curoverse.com/arvados.git/lib/cloud"
+       "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/config"
        "github.com/aws/aws-sdk-go/aws"
        "github.com/aws/aws-sdk-go/service/ec2"
        "github.com/sirupsen/logrus"
-       "golang.org/x/crypto/ssh"
        check "gopkg.in/check.v1"
 )
 
@@ -131,15 +130,13 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error)
        return &ap, cloud.ImageID("blob"), cluster, nil
 }
 
-var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
-
 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
        ap, img, cluster, err := GetInstanceSet()
        if err != nil {
                c.Fatal("Error making provider", err)
        }
 
-       pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+       pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
        c.Assert(err, check.IsNil)
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny"],
@@ -161,7 +158,7 @@ func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
                c.Fatal("Error making provider", err)
        }
 
-       pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+       pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
        c.Assert(err, check.IsNil)
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
@@ -183,7 +180,7 @@ func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
                c.Fatal("Error making provider", err)
        }
 
-       pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
+       pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
        c.Assert(err, check.IsNil)
 
        inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
@@ -227,7 +224,7 @@ func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 
        for _, i := range l {
                tg := i.Tags()
-               log.Printf("%v %v %v", i.String(), i.Address(), tg)
+               c.Logf("%v %v %v", i.String(), i.Address(), tg)
        }
 }