lib/cloud/ec2/ec2_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud:
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud/ec2 -live-ec2-cfg ec2config.yml -check.f=TestCreate
   9 //
  10 // Tests should be run individually and in the order they are listed in the file:
  11 //
  12 // Example azconfig.yml:
  13 //
  14 // ImageIDForTestSuite: ami-xxxxxxxxxxxxxxxxx
  15 // DriverParameters:
  16 //       AccessKeyID: XXXXXXXXXXXXXX
  17 //       SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
  18 //       Region: us-east-1
  19 //       SecurityGroupIDs: [sg-xxxxxxxx]
  20 //       SubnetID: subnet-xxxxxxxx
  21 //       AdminUsername: crunch
  22
  23 package ec2
  24
  25 import (
  26         "encoding/json"
  27         "flag"
  28         "log"
  29         "testing"
  30
  31         "git.curoverse.com/arvados.git/lib/cloud"
  32         "git.curoverse.com/arvados.git/sdk/go/arvados"
  33         "git.curoverse.com/arvados.git/sdk/go/config"
  34         "github.com/aws/aws-sdk-go/aws"
  35         "github.com/aws/aws-sdk-go/service/ec2"
  36         "github.com/sirupsen/logrus"
  37         "golang.org/x/crypto/ssh"
  38         check "gopkg.in/check.v1"
  39 )
  40
  41 var live = flag.String("live-ec2-cfg", "", "Test with real EC2 API, provide config file")
  42
  43 // Gocheck boilerplate
  44 func Test(t *testing.T) {
  45         check.TestingT(t)
  46 }
  47
  48 type EC2InstanceSetSuite struct{}
  49
  50 var _ = check.Suite(&EC2InstanceSetSuite{})
  51
  52 type testConfig struct {
  53         ImageIDForTestSuite string
  54         DriverParameters    json.RawMessage
  55 }
  56
  57 type ec2stub struct {
  58 }
  59
  60 func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) {
  61         return nil, nil
  62 }
  63
  64 func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
  65         return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{
  66                 InstanceId: aws.String("i-123"),
  67                 Tags:       input.TagSpecifications[0].Tags,
  68         }}}, nil
  69 }
  70
  71 func (e *ec2stub) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
  72         return &ec2.DescribeInstancesOutput{}, nil
  73 }
  74
  75 func (e *ec2stub) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
  76         return nil, nil
  77 }
  78
  79 func (e *ec2stub) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) {
  80         return nil, nil
  81 }
  82
  83 func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
  84         cluster := arvados.Cluster{
  85                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
  86                         "tiny": arvados.InstanceType{
  87                                 Name:         "tiny",
  88                                 ProviderType: "t2.micro",
  89                                 VCPUs:        1,
  90                                 RAM:          4000000000,
  91                                 Scratch:      10000000000,
  92                                 Price:        .02,
  93                                 Preemptible:  false,
  94                         },
  95                         "tiny-with-extra-scratch": arvados.InstanceType{
  96                                 Name:         "tiny",
  97                                 ProviderType: "t2.micro",
  98                                 VCPUs:        1,
  99                                 RAM:          4000000000,
 100                                 Price:        .02,
 101                                 Preemptible:  false,
 102                                 AddedScratch: 20000000000,
 103                         },
 104                         "tiny-preemptible": arvados.InstanceType{
 105                                 Name:         "tiny",
 106                                 ProviderType: "t2.micro",
 107                                 VCPUs:        1,
 108                                 RAM:          4000000000,
 109                                 Scratch:      10000000000,
 110                                 Price:        .02,
 111                                 Preemptible:  true,
 112                         },
 113                 })}
 114         if *live != "" {
 115                 var exampleCfg testConfig
 116                 err := config.LoadFile(&exampleCfg, *live)
 117                 if err != nil {
 118                         return nil, cloud.ImageID(""), cluster, err
 119                 }
 120
 121                 ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", logrus.StandardLogger())
 122                 return ap, cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, err
 123         }
 124         ap := ec2InstanceSet{
 125                 ec2config:    ec2InstanceSetConfig{},
 126                 dispatcherID: "test123",
 127                 logger:       logrus.StandardLogger(),
 128                 client:       &ec2stub{},
 129                 keys:         make(map[string]string),
 130         }
 131         return &ap, cloud.ImageID("blob"), cluster, nil
 132 }
 133
 134 var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
 135
 136 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 137         ap, img, cluster, err := GetInstanceSet()
 138         if err != nil {
 139                 c.Fatal("Error making provider", err)
 140         }
 141
 142         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 143         c.Assert(err, check.IsNil)
 144
 145         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 146                 img, map[string]string{
 147                         "TestTagName": "test tag value",
 148                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 149
 150         c.Assert(err, check.IsNil)
 151
 152         tags := inst.Tags()
 153         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 154         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 155
 156 }
 157
 158 func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 159         ap, img, cluster, err := GetInstanceSet()
 160         if err != nil {
 161                 c.Fatal("Error making provider", err)
 162         }
 163
 164         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 165         c.Assert(err, check.IsNil)
 166
 167         inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
 168                 img, map[string]string{
 169                         "TestTagName": "test tag value",
 170                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 171
 172         c.Assert(err, check.IsNil)
 173
 174         tags := inst.Tags()
 175         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 176         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 177
 178 }
 179
 180 func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
 181         ap, img, cluster, err := GetInstanceSet()
 182         if err != nil {
 183                 c.Fatal("Error making provider", err)
 184         }
 185
 186         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 187         c.Assert(err, check.IsNil)
 188
 189         inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
 190                 img, map[string]string{
 191                         "TestTagName": "test tag value",
 192                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 193
 194         c.Assert(err, check.IsNil)
 195
 196         tags := inst.Tags()
 197         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 198         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 199
 200 }
 201
 202 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
 203         ap, _, _, err := GetInstanceSet()
 204         if err != nil {
 205                 c.Fatal("Error making provider", err)
 206         }
 207
 208         l, err := ap.Instances(nil)
 209         c.Assert(err, check.IsNil)
 210
 211         for _, i := range l {
 212                 tg := i.Tags()
 213                 tg["TestTag2"] = "123 test tag 2"
 214                 c.Check(i.SetTags(tg), check.IsNil)
 215         }
 216 }
 217
 218 func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 219         ap, _, _, err := GetInstanceSet()
 220         if err != nil {
 221                 c.Fatal("Error making provider: ", err)
 222         }
 223
 224         l, err := ap.Instances(nil)
 225
 226         c.Assert(err, check.IsNil)
 227
 228         for _, i := range l {
 229                 tg := i.Tags()
 230                 log.Printf("%v %v %v", i.String(), i.Address(), tg)
 231         }
 232 }
 233
 234 func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
 235         ap, _, _, err := GetInstanceSet()
 236         if err != nil {
 237                 c.Fatal("Error making provider", err)
 238         }
 239
 240         l, err := ap.Instances(nil)
 241         c.Assert(err, check.IsNil)
 242
 243         for _, i := range l {
 244                 c.Check(i.Destroy(), check.IsNil)
 245         }
 246 }