lib/cloud/azure/azure_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud -live-azure-cfg azconfig.yml -check.f=TestListInstances
   9 //
  10 // Example azconfig.yml:
  11 //
  12 // subscription_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  13 // key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  14 // region: centralus
  15 // cloud_environment: AzurePublicCloud
  16 // secret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
  17 // tenant_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  18 // resource_group: zzzzz
  19 // network: zzzzz
  20 // subnet: zzzzz-subnet-private
  21 // storage_account: example
  22 // blob_container: vhds
  23 // image: "https://example.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX.vhd"
  24 // delete_dangling_resources_after: 20
  25 // authorized_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example"
  26
  27 package azure
  28
  29 import (
  30         "context"
  31         "errors"
  32         "flag"
  33         "io/ioutil"
  34         "log"
  35         "net"
  36         "net/http"
  37         "os"
  38         "testing"
  39         "time"
  40
  41         "git.curoverse.com/arvados.git/lib/cloud"
  42         "git.curoverse.com/arvados.git/sdk/go/arvados"
  43         "git.curoverse.com/arvados.git/sdk/go/config"
  44         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  45         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  46         "github.com/Azure/azure-sdk-for-go/storage"
  47         "github.com/Azure/go-autorest/autorest"
  48         "github.com/Azure/go-autorest/autorest/azure"
  49         "github.com/Azure/go-autorest/autorest/to"
  50         "github.com/jmcvetta/randutil"
  51         "github.com/sirupsen/logrus"
  52         "golang.org/x/crypto/ssh"
  53         check "gopkg.in/check.v1"
  54 )
  55
  56 // Gocheck boilerplate
  57 func Test(t *testing.T) {
  58         check.TestingT(t)
  59 }
  60
  61 type AzureInstanceSetSuite struct{}
  62
  63 var _ = check.Suite(&AzureInstanceSetSuite{})
  64
  65 type VirtualMachinesClientStub struct{}
  66
  67 var testKey []byte = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
  68
  69 func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
  70         resourceGroupName string,
  71         VMName string,
  72         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  73         parameters.ID = &VMName
  74         parameters.Name = &VMName
  75         return parameters, nil
  76 }
  77
  78 func (*VirtualMachinesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  79         return nil, nil
  80 }
  81
  82 func (*VirtualMachinesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  83         return compute.VirtualMachineListResultIterator{}, nil
  84 }
  85
  86 type InterfacesClientStub struct{}
  87
  88 func (*InterfacesClientStub) CreateOrUpdate(ctx context.Context,
  89         resourceGroupName string,
  90         nicName string,
  91         parameters network.Interface) (result network.Interface, err error) {
  92         parameters.ID = to.StringPtr(nicName)
  93         (*parameters.IPConfigurations)[0].PrivateIPAddress = to.StringPtr("192.168.5.5")
  94         return parameters, nil
  95 }
  96
  97 func (*InterfacesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  98         return nil, nil
  99 }
 100
 101 func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
 102         return network.InterfaceListResultIterator{}, nil
 103 }
 104
 105 var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
 106
 107 func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) {
 108         cluster := arvados.Cluster{
 109                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
 110                         "tiny": arvados.InstanceType{
 111                                 Name:         "tiny",
 112                                 ProviderType: "Standard_D1_v2",
 113                                 VCPUs:        1,
 114                                 RAM:          4000000000,
 115                                 Scratch:      10000000000,
 116                                 Price:        .02,
 117                                 Preemptible:  false,
 118                         },
 119                 })}
 120         if *live != "" {
 121                 cfg := make(map[string]interface{})
 122                 err := config.LoadFile(&cfg, *live)
 123                 if err != nil {
 124                         return nil, cloud.ImageID(""), cluster, err
 125                 }
 126                 ap, err := NewAzureInstanceSet(cfg, "test123", logrus.StandardLogger())
 127                 return ap, cloud.ImageID(cfg["image"].(string)), cluster, err
 128         } else {
 129                 ap := AzureInstanceSet{
 130                         azconfig: AzureInstanceSetConfig{
 131                                 BlobContainer: "vhds",
 132                         },
 133                         dispatcherID: "test123",
 134                         namePrefix:   "compute-test123-",
 135                         logger:       logrus.StandardLogger(),
 136                         deleteNIC:    make(chan string),
 137                         deleteBlob:   make(chan storage.Blob),
 138                 }
 139                 ap.ctx, ap.stopFunc = context.WithCancel(context.Background())
 140                 ap.vmClient = &VirtualMachinesClientStub{}
 141                 ap.netClient = &InterfacesClientStub{}
 142                 return &ap, cloud.ImageID("blob"), cluster, nil
 143         }
 144 }
 145
 146 func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
 147         ap, img, cluster, err := GetInstanceSet()
 148         if err != nil {
 149                 c.Fatal("Error making provider", err)
 150         }
 151
 152         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 153         c.Assert(err, check.IsNil)
 154
 155         nodetoken, err := randutil.String(40, "abcdefghijklmnopqrstuvwxyz0123456789")
 156         c.Assert(err, check.IsNil)
 157
 158         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 159                 img, map[string]string{
 160                         "node-token": nodetoken},
 161                 pk)
 162
 163         c.Assert(err, check.IsNil)
 164
 165         tg := inst.Tags()
 166         log.Printf("Result %v %v %v", inst.String(), inst.Address(), tg)
 167
 168 }
 169
 170 func (*AzureInstanceSetSuite) TestListInstances(c *check.C) {
 171         ap, _, _, err := GetInstanceSet()
 172         if err != nil {
 173                 c.Fatal("Error making provider", err)
 174         }
 175
 176         l, err := ap.Instances(nil)
 177
 178         c.Assert(err, check.IsNil)
 179
 180         for _, i := range l {
 181                 tg := i.Tags()
 182                 log.Printf("%v %v %v", i.String(), i.Address(), tg)
 183         }
 184 }
 185
 186 func (*AzureInstanceSetSuite) TestManageNics(c *check.C) {
 187         ap, _, _, err := GetInstanceSet()
 188         if err != nil {
 189                 c.Fatal("Error making provider", err)
 190         }
 191
 192         ap.(*AzureInstanceSet).ManageNics()
 193         ap.Stop()
 194 }
 195
 196 func (*AzureInstanceSetSuite) TestManageBlobs(c *check.C) {
 197         ap, _, _, err := GetInstanceSet()
 198         if err != nil {
 199                 c.Fatal("Error making provider", err)
 200         }
 201
 202         ap.(*AzureInstanceSet).ManageBlobs()
 203         ap.Stop()
 204 }
 205
 206 func (*AzureInstanceSetSuite) TestDestroyInstances(c *check.C) {
 207         ap, _, _, err := GetInstanceSet()
 208         if err != nil {
 209                 c.Fatal("Error making provider", err)
 210         }
 211
 212         l, err := ap.Instances(nil)
 213         c.Assert(err, check.IsNil)
 214
 215         for _, i := range l {
 216                 c.Check(i.Destroy(), check.IsNil)
 217         }
 218 }
 219
 220 func (*AzureInstanceSetSuite) TestDeleteFake(c *check.C) {
 221         ap, _, _, err := GetInstanceSet()
 222         if err != nil {
 223                 c.Fatal("Error making provider", err)
 224         }
 225
 226         _, err = ap.(*AzureInstanceSet).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
 227
 228         de, ok := err.(autorest.DetailedError)
 229         if ok {
 230                 rq := de.Original.(*azure.RequestError)
 231
 232                 log.Printf("%v %q %q", rq.Response.StatusCode, rq.ServiceError.Code, rq.ServiceError.Message)
 233         }
 234 }
 235
 236 func (*AzureInstanceSetSuite) TestWrapError(c *check.C) {
 237         retryError := autorest.DetailedError{
 238                 Original: &azure.RequestError{
 239                         DetailedError: autorest.DetailedError{
 240                                 Response: &http.Response{
 241                                         StatusCode: 429,
 242                                         Header:     map[string][]string{"Retry-After": []string{"123"}},
 243                                 },
 244                         },
 245                         ServiceError: &azure.ServiceError{},
 246                 },
 247         }
 248         wrapped := WrapAzureError(retryError)
 249         _, ok := wrapped.(cloud.RateLimitError)
 250         c.Check(ok, check.Equals, true)
 251
 252         quotaError := autorest.DetailedError{
 253                 Original: &azure.RequestError{
 254                         DetailedError: autorest.DetailedError{
 255                                 Response: &http.Response{
 256                                         StatusCode: 503,
 257                                 },
 258                         },
 259                         ServiceError: &azure.ServiceError{
 260                                 Message: "No more quota",
 261                         },
 262                 },
 263         }
 264         wrapped = WrapAzureError(quotaError)
 265         _, ok = wrapped.(cloud.QuotaError)
 266         c.Check(ok, check.Equals, true)
 267 }
 268
 269 func (*AzureInstanceSetSuite) TestSetTags(c *check.C) {
 270         ap, _, _, err := GetInstanceSet()
 271         if err != nil {
 272                 c.Fatal("Error making provider", err)
 273         }
 274         l, err := ap.Instances(nil)
 275         c.Assert(err, check.IsNil)
 276
 277         if len(l) > 0 {
 278                 err = l[0].SetTags(map[string]string{"foo": "bar"})
 279                 if err != nil {
 280                         c.Fatal("Error setting tags", err)
 281                 }
 282         }
 283         l, err = ap.Instances(nil)
 284         c.Assert(err, check.IsNil)
 285
 286         if len(l) > 0 {
 287                 tg := l[0].Tags()
 288                 log.Printf("tags are %v", tg)
 289         }
 290 }
 291
 292 func (*AzureInstanceSetSuite) TestSSH(c *check.C) {
 293         ap, _, _, err := GetInstanceSet()
 294         if err != nil {
 295                 c.Fatal("Error making provider", err)
 296         }
 297         l, err := ap.Instances(nil)
 298         c.Assert(err, check.IsNil)
 299
 300         if len(l) > 0 {
 301
 302                 sshclient, err := SetupSSHClient(c, l[0])
 303                 c.Assert(err, check.IsNil)
 304
 305                 sess, err := sshclient.NewSession()
 306                 c.Assert(err, check.IsNil)
 307
 308                 out, err := sess.Output("cat /home/crunch/node-token")
 309                 c.Assert(err, check.IsNil)
 310
 311                 log.Printf("%v", string(out))
 312
 313                 sshclient.Conn.Close()
 314         }
 315 }
 316
 317 func SetupSSHClient(c *check.C, inst cloud.Instance) (*ssh.Client, error) {
 318         addr := inst.Address() + ":2222"
 319         if addr == "" {
 320                 return nil, errors.New("instance has no address")
 321         }
 322
 323         f, err := os.Open("azconfig_sshkey")
 324         c.Assert(err, check.IsNil)
 325
 326         keybytes, err := ioutil.ReadAll(f)
 327         c.Assert(err, check.IsNil)
 328
 329         priv, err := ssh.ParsePrivateKey(keybytes)
 330         c.Assert(err, check.IsNil)
 331
 332         var receivedKey ssh.PublicKey
 333         client, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{
 334                 User: "crunch",
 335                 Auth: []ssh.AuthMethod{
 336                         ssh.PublicKeys(priv),
 337                 },
 338                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
 339                         receivedKey = key
 340                         return nil
 341                 },
 342                 Timeout: time.Minute,
 343         })
 344
 345         if err != nil {
 346                 return nil, err
 347         } else if receivedKey == nil {
 348                 return nil, errors.New("BUG: key was never provided to HostKeyCallback")
 349         }
 350
 351         err = inst.VerifyHostKey(receivedKey, client)
 352         c.Assert(err, check.IsNil)
 353
 354         return client, nil
 355 }