lib/cloud/ec2/ec2_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud:
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud/ec2 -live-ec2-cfg ec2config.yml -check.f=TestCreate
   9 //
  10 // Tests should be run individually and in the order they are listed in the file:
  11 //
  12 // Example azconfig.yml:
  13 //
  14 // ImageIDForTestSuite: ami-xxxxxxxxxxxxxxxxx
  15 // DriverParameters:
  16 //       AccessKeyID: XXXXXXXXXXXXXX
  17 //       SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
  18 //       Region: us-east-1
  19 //       SecurityGroupId: sg-xxxxxxxx
  20 //       SubnetId: subnet-xxxxxxxx
  21 //       AdminUsername: crunch
  22 //       KeyPairName: arvados-dispatcher-keypair
  23
  24 package ec2
  25
  26 import (
  27         "encoding/json"
  28         "flag"
  29         "log"
  30         "testing"
  31
  32         "git.curoverse.com/arvados.git/lib/cloud"
  33         "git.curoverse.com/arvados.git/sdk/go/arvados"
  34         "git.curoverse.com/arvados.git/sdk/go/config"
  35         "github.com/aws/aws-sdk-go/aws"
  36         "github.com/aws/aws-sdk-go/service/ec2"
  37         "github.com/sirupsen/logrus"
  38         "golang.org/x/crypto/ssh"
  39         check "gopkg.in/check.v1"
  40 )
  41
  42 var live = flag.String("live-ec2-cfg", "", "Test with real EC2 API, provide config file")
  43
  44 // Gocheck boilerplate
  45 func Test(t *testing.T) {
  46         check.TestingT(t)
  47 }
  48
  49 type EC2InstanceSetSuite struct{}
  50
  51 var _ = check.Suite(&EC2InstanceSetSuite{})
  52
  53 type testConfig struct {
  54         ImageIDForTestSuite string
  55         DriverParameters    json.RawMessage
  56 }
  57
  58 type ec2stub struct {
  59 }
  60
  61 func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) {
  62         return nil, nil
  63 }
  64
  65 func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
  66         return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{
  67                 InstanceId: aws.String("i-123"),
  68                 Tags:       input.TagSpecifications[0].Tags,
  69         }}}, nil
  70 }
  71
  72 func (e *ec2stub) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
  73         return &ec2.DescribeInstancesOutput{}, nil
  74 }
  75
  76 func (e *ec2stub) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
  77         return nil, nil
  78 }
  79
  80 func (e *ec2stub) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) {
  81         return nil, nil
  82 }
  83
  84 func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
  85         cluster := arvados.Cluster{
  86                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
  87                         "tiny": arvados.InstanceType{
  88                                 Name:         "tiny",
  89                                 ProviderType: "t2.micro",
  90                                 VCPUs:        1,
  91                                 RAM:          4000000000,
  92                                 Scratch:      10000000000,
  93                                 Price:        .02,
  94                                 Preemptible:  false,
  95                         },
  96                         "tiny-with-extra-scratch": arvados.InstanceType{
  97                                 Name:         "tiny",
  98                                 ProviderType: "t2.micro",
  99                                 VCPUs:        1,
 100                                 RAM:          4000000000,
 101                                 Scratch:      10000000000,
 102                                 ExtraScratch: 20000000000,
 103                                 Price:        .02,
 104                                 Preemptible:  false,
 105                         },
 106                         "tiny-preemptible": arvados.InstanceType{
 107                                 Name:         "tiny",
 108                                 ProviderType: "t2.micro",
 109                                 VCPUs:        1,
 110                                 RAM:          4000000000,
 111                                 Scratch:      10000000000,
 112                                 Price:        .02,
 113                                 Preemptible:  true,
 114                         },
 115                 })}
 116         if *live != "" {
 117                 var exampleCfg testConfig
 118                 err := config.LoadFile(&exampleCfg, *live)
 119                 if err != nil {
 120                         return nil, cloud.ImageID(""), cluster, err
 121                 }
 122
 123                 ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", logrus.StandardLogger())
 124                 return ap, cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, err
 125         }
 126         ap := ec2InstanceSet{
 127                 ec2config:    ec2InstanceSetConfig{},
 128                 dispatcherID: "test123",
 129                 logger:       logrus.StandardLogger(),
 130                 client:       &ec2stub{},
 131         }
 132         return &ap, cloud.ImageID("blob"), cluster, nil
 133 }
 134
 135 var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
 136
 137 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 138         ap, img, cluster, err := GetInstanceSet()
 139         if err != nil {
 140                 c.Fatal("Error making provider", err)
 141         }
 142
 143         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 144         c.Assert(err, check.IsNil)
 145
 146         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 147                 img, map[string]string{
 148                         "TestTagName": "test tag value",
 149                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 150
 151         c.Assert(err, check.IsNil)
 152
 153         tags := inst.Tags()
 154         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 155         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 156
 157 }
 158
 159 func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 160         ap, img, cluster, err := GetInstanceSet()
 161         if err != nil {
 162                 c.Fatal("Error making provider", err)
 163         }
 164
 165         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 166         c.Assert(err, check.IsNil)
 167
 168         inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
 169                 img, map[string]string{
 170                         "TestTagName": "test tag value",
 171                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 172
 173         c.Assert(err, check.IsNil)
 174
 175         tags := inst.Tags()
 176         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 177         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 178
 179 }
 180
 181 func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
 182         ap, img, cluster, err := GetInstanceSet()
 183         if err != nil {
 184                 c.Fatal("Error making provider", err)
 185         }
 186
 187         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 188         c.Assert(err, check.IsNil)
 189
 190         inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
 191                 img, map[string]string{
 192                         "TestTagName": "test tag value",
 193                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 194
 195         c.Assert(err, check.IsNil)
 196
 197         tags := inst.Tags()
 198         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 199         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 200
 201 }
 202
 203 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
 204         ap, _, _, err := GetInstanceSet()
 205         if err != nil {
 206                 c.Fatal("Error making provider", err)
 207         }
 208
 209         l, err := ap.Instances(nil)
 210         c.Assert(err, check.IsNil)
 211
 212         for _, i := range l {
 213                 tg := i.Tags()
 214                 tg["TestTag2"] = "123 test tag 2"
 215                 c.Check(i.SetTags(tg), check.IsNil)
 216         }
 217 }
 218
 219 func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 220         ap, _, _, err := GetInstanceSet()
 221         if err != nil {
 222                 c.Fatal("Error making provider: ", err)
 223         }
 224
 225         l, err := ap.Instances(nil)
 226
 227         c.Assert(err, check.IsNil)
 228
 229         for _, i := range l {
 230                 tg := i.Tags()
 231                 log.Printf("%v %v %v", i.String(), i.Address(), tg)
 232         }
 233 }
 234
 235 func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
 236         ap, _, _, err := GetInstanceSet()
 237         if err != nil {
 238                 c.Fatal("Error making provider", err)
 239         }
 240
 241         l, err := ap.Instances(nil)
 242         c.Assert(err, check.IsNil)
 243
 244         for _, i := range l {
 245                 c.Check(i.Destroy(), check.IsNil)
 246         }
 247 }