lib/cloud/ec2/ec2_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud:
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud/ec2 -live-ec2-cfg ec2config.yml -check.f=TestCreate
   9 //
  10 // Tests should be run individually and in the order they are listed in the file:
  11 //
  12 // Example azconfig.yml:
  13 //
  14 // ImageIDForTestSuite: ami-xxxxxxxxxxxxxxxxx
  15 // DriverParameters:
  16 //       AccessKeyID: XXXXXXXXXXXXXX
  17 //       SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
  18 //       Region: us-east-1
  19 //       SecurityGroupIDs: [sg-xxxxxxxx]
  20 //       SubnetID: subnet-xxxxxxxx
  21 //       AdminUsername: crunch
  22
  23 package ec2
  24
  25 import (
  26         "encoding/json"
  27         "flag"
  28         "testing"
  29
  30         "git.curoverse.com/arvados.git/lib/cloud"
  31         "git.curoverse.com/arvados.git/lib/dispatchcloud/test"
  32         "git.curoverse.com/arvados.git/sdk/go/arvados"
  33         "git.curoverse.com/arvados.git/sdk/go/config"
  34         "github.com/aws/aws-sdk-go/aws"
  35         "github.com/aws/aws-sdk-go/service/ec2"
  36         "github.com/sirupsen/logrus"
  37         check "gopkg.in/check.v1"
  38 )
  39
  40 var live = flag.String("live-ec2-cfg", "", "Test with real EC2 API, provide config file")
  41
  42 // Gocheck boilerplate
  43 func Test(t *testing.T) {
  44         check.TestingT(t)
  45 }
  46
  47 type EC2InstanceSetSuite struct{}
  48
  49 var _ = check.Suite(&EC2InstanceSetSuite{})
  50
  51 type testConfig struct {
  52         ImageIDForTestSuite string
  53         DriverParameters    json.RawMessage
  54 }
  55
  56 type ec2stub struct {
  57 }
  58
  59 func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) {
  60         return nil, nil
  61 }
  62
  63 func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
  64         return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{
  65                 InstanceId: aws.String("i-123"),
  66                 Tags:       input.TagSpecifications[0].Tags,
  67         }}}, nil
  68 }
  69
  70 func (e *ec2stub) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
  71         return &ec2.DescribeInstancesOutput{}, nil
  72 }
  73
  74 func (e *ec2stub) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
  75         return nil, nil
  76 }
  77
  78 func (e *ec2stub) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) {
  79         return nil, nil
  80 }
  81
  82 func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
  83         cluster := arvados.Cluster{
  84                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
  85                         "tiny": arvados.InstanceType{
  86                                 Name:         "tiny",
  87                                 ProviderType: "t2.micro",
  88                                 VCPUs:        1,
  89                                 RAM:          4000000000,
  90                                 Scratch:      10000000000,
  91                                 Price:        .02,
  92                                 Preemptible:  false,
  93                         },
  94                         "tiny-with-extra-scratch": arvados.InstanceType{
  95                                 Name:         "tiny",
  96                                 ProviderType: "t2.micro",
  97                                 VCPUs:        1,
  98                                 RAM:          4000000000,
  99                                 Price:        .02,
 100                                 Preemptible:  false,
 101                                 AddedScratch: 20000000000,
 102                         },
 103                         "tiny-preemptible": arvados.InstanceType{
 104                                 Name:         "tiny",
 105                                 ProviderType: "t2.micro",
 106                                 VCPUs:        1,
 107                                 RAM:          4000000000,
 108                                 Scratch:      10000000000,
 109                                 Price:        .02,
 110                                 Preemptible:  true,
 111                         },
 112                 })}
 113         if *live != "" {
 114                 var exampleCfg testConfig
 115                 err := config.LoadFile(&exampleCfg, *live)
 116                 if err != nil {
 117                         return nil, cloud.ImageID(""), cluster, err
 118                 }
 119
 120                 ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", logrus.StandardLogger())
 121                 return ap, cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, err
 122         }
 123         ap := ec2InstanceSet{
 124                 ec2config:    ec2InstanceSetConfig{},
 125                 dispatcherID: "test123",
 126                 logger:       logrus.StandardLogger(),
 127                 client:       &ec2stub{},
 128                 keys:         make(map[string]string),
 129         }
 130         return &ap, cloud.ImageID("blob"), cluster, nil
 131 }
 132
 133 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 134         ap, img, cluster, err := GetInstanceSet()
 135         if err != nil {
 136                 c.Fatal("Error making provider", err)
 137         }
 138
 139         pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 140         c.Assert(err, check.IsNil)
 141
 142         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 143                 img, map[string]string{
 144                         "TestTagName": "test tag value",
 145                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 146
 147         c.Assert(err, check.IsNil)
 148
 149         tags := inst.Tags()
 150         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 151         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 152
 153 }
 154
 155 func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 156         ap, img, cluster, err := GetInstanceSet()
 157         if err != nil {
 158                 c.Fatal("Error making provider", err)
 159         }
 160
 161         pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 162         c.Assert(err, check.IsNil)
 163
 164         inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
 165                 img, map[string]string{
 166                         "TestTagName": "test tag value",
 167                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 168
 169         c.Assert(err, check.IsNil)
 170
 171         tags := inst.Tags()
 172         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 173         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 174
 175 }
 176
 177 func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
 178         ap, img, cluster, err := GetInstanceSet()
 179         if err != nil {
 180                 c.Fatal("Error making provider", err)
 181         }
 182
 183         pk, _ := test.LoadTestKey(c, "../../dispatchcloud/test/sshkey_dispatch")
 184         c.Assert(err, check.IsNil)
 185
 186         inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
 187                 img, map[string]string{
 188                         "TestTagName": "test tag value",
 189                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 190
 191         c.Assert(err, check.IsNil)
 192
 193         tags := inst.Tags()
 194         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 195         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 196
 197 }
 198
 199 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
 200         ap, _, _, err := GetInstanceSet()
 201         if err != nil {
 202                 c.Fatal("Error making provider", err)
 203         }
 204
 205         l, err := ap.Instances(nil)
 206         c.Assert(err, check.IsNil)
 207
 208         for _, i := range l {
 209                 tg := i.Tags()
 210                 tg["TestTag2"] = "123 test tag 2"
 211                 c.Check(i.SetTags(tg), check.IsNil)
 212         }
 213 }
 214
 215 func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 216         ap, _, _, err := GetInstanceSet()
 217         if err != nil {
 218                 c.Fatal("Error making provider: ", err)
 219         }
 220
 221         l, err := ap.Instances(nil)
 222
 223         c.Assert(err, check.IsNil)
 224
 225         for _, i := range l {
 226                 tg := i.Tags()
 227                 c.Logf("%v %v %v", i.String(), i.Address(), tg)
 228         }
 229 }
 230
 231 func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
 232         ap, _, _, err := GetInstanceSet()
 233         if err != nil {
 234                 c.Fatal("Error making provider", err)
 235         }
 236
 237         l, err := ap.Instances(nil)
 238         c.Assert(err, check.IsNil)
 239
 240         for _, i := range l {
 241                 c.Check(i.Destroy(), check.IsNil)
 242         }
 243 }