lib/cloud/ec2/ec2_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud:
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud/ec2 -live-ec2-cfg ec2config.yml -check.f=TestCreate
   9 //
  10 // Tests should be run individually and in the order they are listed in the file:
  11 //
  12 // Example azconfig.yml:
  13 //
  14 // ImageIDForTestSuite: ami-xxxxxxxxxxxxxxxxx
  15 // DriverParameters:
  16 //       AccessKeyID: XXXXXXXXXXXXXX
  17 //       SecretAccessKey: xxxxxxxxxxxxxxxxxxxx
  18 //       Region: us-east-1
  19 //       SecurityGroupId: sg-xxxxxxxx
  20 //       SubnetId: subnet-xxxxxxxx
  21 //       AdminUsername: crunch
  22 //       KeyPairName: arvados-dispatcher-keypair
  23
  24 package ec2
  25
  26 import (
  27         "encoding/json"
  28         "flag"
  29         "log"
  30         "testing"
  31
  32         "git.curoverse.com/arvados.git/lib/cloud"
  33         "git.curoverse.com/arvados.git/sdk/go/arvados"
  34         "git.curoverse.com/arvados.git/sdk/go/config"
  35         "github.com/sirupsen/logrus"
  36         "golang.org/x/crypto/ssh"
  37         check "gopkg.in/check.v1"
  38 )
  39
  40 var live = flag.String("live-ec2-cfg", "", "Test with real EC2 API, provide config file")
  41
  42 // Gocheck boilerplate
  43 func Test(t *testing.T) {
  44         check.TestingT(t)
  45 }
  46
  47 type EC2InstanceSetSuite struct{}
  48
  49 var _ = check.Suite(&EC2InstanceSetSuite{})
  50
  51 type testConfig struct {
  52         ImageIDForTestSuite string
  53         DriverParameters    json.RawMessage
  54 }
  55
  56 func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
  57         cluster := arvados.Cluster{
  58                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
  59                         "tiny": arvados.InstanceType{
  60                                 Name:         "tiny",
  61                                 ProviderType: "m1.small",
  62                                 VCPUs:        1,
  63                                 RAM:          4000000000,
  64                                 Scratch:      10000000000,
  65                                 Price:        .02,
  66                                 Preemptible:  false,
  67                         },
  68                         "tiny-with-extra-scratch": arvados.InstanceType{
  69                                 Name:         "tiny",
  70                                 ProviderType: "m1.small",
  71                                 VCPUs:        1,
  72                                 RAM:          4000000000,
  73                                 Scratch:      10000000000,
  74                                 ExtraScratch: 20000000000,
  75                                 Price:        .02,
  76                                 Preemptible:  false,
  77                         },
  78                         "tiny-preemptible": arvados.InstanceType{
  79                                 Name:         "tiny",
  80                                 ProviderType: "m1.small",
  81                                 VCPUs:        1,
  82                                 RAM:          4000000000,
  83                                 Scratch:      10000000000,
  84                                 Price:        .02,
  85                                 Preemptible:  true,
  86                         },
  87                 })}
  88         if *live != "" {
  89                 var exampleCfg testConfig
  90                 err := config.LoadFile(&exampleCfg, *live)
  91                 if err != nil {
  92                         return nil, cloud.ImageID(""), cluster, err
  93                 }
  94
  95                 ap, err := newEC2InstanceSet(exampleCfg.DriverParameters, "test123", logrus.StandardLogger())
  96                 return ap, cloud.ImageID(exampleCfg.ImageIDForTestSuite), cluster, err
  97         }
  98         ap := ec2InstanceSet{
  99                 ec2config:    ec2InstanceSetConfig{},
 100                 dispatcherID: "test123",
 101                 logger:       logrus.StandardLogger(),
 102         }
 103         return &ap, cloud.ImageID("blob"), cluster, nil
 104 }
 105
 106 var testKey = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
 107
 108 func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 109         ap, img, cluster, err := GetInstanceSet()
 110         if err != nil {
 111                 c.Fatal("Error making provider", err)
 112         }
 113
 114         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 115         c.Assert(err, check.IsNil)
 116
 117         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 118                 img, map[string]string{
 119                         "TestTagName": "test tag value",
 120                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 121
 122         c.Assert(err, check.IsNil)
 123
 124         tags := inst.Tags()
 125         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 126         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 127
 128 }
 129
 130 func (*EC2InstanceSetSuite) TestCreateWithExtraScratch(c *check.C) {
 131         ap, img, cluster, err := GetInstanceSet()
 132         if err != nil {
 133                 c.Fatal("Error making provider", err)
 134         }
 135
 136         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 137         c.Assert(err, check.IsNil)
 138
 139         inst, err := ap.Create(cluster.InstanceTypes["tiny-with-extra-scratch"],
 140                 img, map[string]string{
 141                         "TestTagName": "test tag value",
 142                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 143
 144         c.Assert(err, check.IsNil)
 145
 146         tags := inst.Tags()
 147         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 148         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 149
 150 }
 151
 152 func (*EC2InstanceSetSuite) TestCreatePreemptible(c *check.C) {
 153         ap, img, cluster, err := GetInstanceSet()
 154         if err != nil {
 155                 c.Fatal("Error making provider", err)
 156         }
 157
 158         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 159         c.Assert(err, check.IsNil)
 160
 161         inst, err := ap.Create(cluster.InstanceTypes["tiny-preemptible"],
 162                 img, map[string]string{
 163                         "TestTagName": "test tag value",
 164                 }, "umask 0600; echo -n test-file-data >/var/run/test-file", pk)
 165
 166         c.Assert(err, check.IsNil)
 167
 168         tags := inst.Tags()
 169         c.Check(tags["TestTagName"], check.Equals, "test tag value")
 170         c.Logf("inst.String()=%v Address()=%v Tags()=%v", inst.String(), inst.Address(), tags)
 171
 172 }
 173
 174 func (*EC2InstanceSetSuite) TestListInstances(c *check.C) {
 175         ap, _, _, err := GetInstanceSet()
 176         if err != nil {
 177                 c.Fatal("Error making provider: ", err)
 178         }
 179
 180         l, err := ap.Instances(nil)
 181
 182         c.Assert(err, check.IsNil)
 183
 184         for _, i := range l {
 185                 tg := i.Tags()
 186                 log.Printf("%v %v %v", i.String(), i.Address(), tg)
 187         }
 188 }
 189
 190 func (*EC2InstanceSetSuite) TestDestroyInstances(c *check.C) {
 191         ap, _, _, err := GetInstanceSet()
 192         if err != nil {
 193                 c.Fatal("Error making provider", err)
 194         }
 195
 196         l, err := ap.Instances(nil)
 197         c.Assert(err, check.IsNil)
 198
 199         for _, i := range l {
 200                 c.Check(i.Destroy(), check.IsNil)
 201         }
 202 }