lib/cloud/azure_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4 //
   5 //
   6 // How to manually run individual tests against the real cloud
   7 //
   8 // $ go test -v git.curoverse.com/arvados.git/lib/cloud -live-azure-cfg azconfig.yml -check.f=TestListInstances
   9 //
  10 // Example azconfig.yml:
  11 //
  12 // subscription_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  13 // key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  14 // region: centralus
  15 // cloud_environment: AzurePublicCloud
  16 // secret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
  17 // tenant_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
  18 // resource_group: zzzzz
  19 // network: zzzzz
  20 // subnet: zzzzz-subnet-private
  21 // storage_account: example
  22 // blob_container: vhds
  23 // image: "https://example.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX.vhd"
  24 // delete_dangling_resources_after: 20
  25 // authorized_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example"
  26
  27 package cloud
  28
  29 import (
  30         "context"
  31         "errors"
  32         "flag"
  33         "io/ioutil"
  34         "log"
  35         "net"
  36         "net/http"
  37         "os"
  38         "time"
  39
  40         "git.curoverse.com/arvados.git/sdk/go/arvados"
  41         "git.curoverse.com/arvados.git/sdk/go/config"
  42         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  43         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  44         "github.com/Azure/azure-sdk-for-go/storage"
  45         "github.com/Azure/go-autorest/autorest"
  46         "github.com/Azure/go-autorest/autorest/azure"
  47         "github.com/Azure/go-autorest/autorest/to"
  48         "github.com/jmcvetta/randutil"
  49         "github.com/sirupsen/logrus"
  50         "golang.org/x/crypto/ssh"
  51         check "gopkg.in/check.v1"
  52 )
  53
  54 type AzureInstanceSetSuite struct{}
  55
  56 var _ = check.Suite(&AzureInstanceSetSuite{})
  57
  58 type VirtualMachinesClientStub struct{}
  59
  60 var testKey []byte = []byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLQS1ExT2+WjA0d/hntEAyAtgeN1W2ik2QX8c2zO6HjlPHWXL92r07W0WMuDib40Pcevpi1BXeBWXA9ZB5KKMJB+ukaAu22KklnQuUmNvk6ZXnPKSkGxuCYvPQb08WhHf3p1VxiKfP3iauedBDM4x9/bkJohlBBQiFXzNUcQ+a6rKiMzmJN2gbL8ncyUzc+XQ5q4JndTwTGtOlzDiGOc9O4z5Dd76wtAVJneOuuNpwfFRVHThpJM6VThpCZOnl8APaceWXKeuwOuCae3COZMz++xQfxOfZ9Z8aIwo+TlQhsRaNfZ4Vjrop6ej8dtfZtgUFKfbXEOYaHrGrWGotFDTD example@example`)
  61
  62 func (*VirtualMachinesClientStub) CreateOrUpdate(ctx context.Context,
  63         resourceGroupName string,
  64         VMName string,
  65         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  66         parameters.ID = &VMName
  67         parameters.Name = &VMName
  68         return parameters, nil
  69 }
  70
  71 func (*VirtualMachinesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  72         return nil, nil
  73 }
  74
  75 func (*VirtualMachinesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  76         return compute.VirtualMachineListResultIterator{}, nil
  77 }
  78
  79 type InterfacesClientStub struct{}
  80
  81 func (*InterfacesClientStub) CreateOrUpdate(ctx context.Context,
  82         resourceGroupName string,
  83         nicName string,
  84         parameters network.Interface) (result network.Interface, err error) {
  85         parameters.ID = to.StringPtr(nicName)
  86         (*parameters.IPConfigurations)[0].PrivateIPAddress = to.StringPtr("192.168.5.5")
  87         return parameters, nil
  88 }
  89
  90 func (*InterfacesClientStub) Delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  91         return nil, nil
  92 }
  93
  94 func (*InterfacesClientStub) ListComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
  95         return network.InterfaceListResultIterator{}, nil
  96 }
  97
  98 var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide config file")
  99
 100 func GetInstanceSet() (InstanceSet, ImageID, arvados.Cluster, error) {
 101         cluster := arvados.Cluster{
 102                 InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{
 103                         "tiny": arvados.InstanceType{
 104                                 Name:         "tiny",
 105                                 ProviderType: "Standard_D1_v2",
 106                                 VCPUs:        1,
 107                                 RAM:          4000000000,
 108                                 Scratch:      10000000000,
 109                                 Price:        .02,
 110                                 Preemptible:  false,
 111                         },
 112                 })}
 113         if *live != "" {
 114                 cfg := make(map[string]interface{})
 115                 err := config.LoadFile(&cfg, *live)
 116                 if err != nil {
 117                         return nil, ImageID(""), cluster, err
 118                 }
 119                 ap, err := NewAzureInstanceSet(cfg, "test123", logrus.StandardLogger())
 120                 return ap, ImageID(cfg["image"].(string)), cluster, err
 121         } else {
 122                 ap := AzureInstanceSet{
 123                         azconfig: AzureInstanceSetConfig{
 124                                 BlobContainer: "vhds",
 125                         },
 126                         dispatcherID: "test123",
 127                         namePrefix:   "compute-test123-",
 128                         logger:       logrus.StandardLogger(),
 129                         deleteNIC:    make(chan string),
 130                         deleteBlob:   make(chan storage.Blob),
 131                 }
 132                 ap.ctx, ap.stopFunc = context.WithCancel(context.Background())
 133                 ap.vmClient = &VirtualMachinesClientStub{}
 134                 ap.netClient = &InterfacesClientStub{}
 135                 return &ap, ImageID("blob"), cluster, nil
 136         }
 137 }
 138
 139 func (*AzureInstanceSetSuite) TestCreate(c *check.C) {
 140         ap, img, cluster, err := GetInstanceSet()
 141         if err != nil {
 142                 c.Fatal("Error making provider", err)
 143         }
 144
 145         pk, _, _, _, err := ssh.ParseAuthorizedKey(testKey)
 146         c.Assert(err, check.IsNil)
 147
 148         nodetoken, err := randutil.String(40, "abcdefghijklmnopqrstuvwxyz0123456789")
 149         c.Assert(err, check.IsNil)
 150
 151         inst, err := ap.Create(cluster.InstanceTypes["tiny"],
 152                 img, map[string]string{
 153                         "node-token": nodetoken},
 154                 pk)
 155
 156         c.Assert(err, check.IsNil)
 157
 158         tg := inst.Tags()
 159         log.Printf("Result %v %v %v", inst.String(), inst.Address(), tg)
 160
 161 }
 162
 163 func (*AzureInstanceSetSuite) TestListInstances(c *check.C) {
 164         ap, _, _, err := GetInstanceSet()
 165         if err != nil {
 166                 c.Fatal("Error making provider", err)
 167         }
 168
 169         l, err := ap.Instances(nil)
 170
 171         c.Assert(err, check.IsNil)
 172
 173         for _, i := range l {
 174                 tg := i.Tags()
 175                 log.Printf("%v %v %v", i.String(), i.Address(), tg)
 176         }
 177 }
 178
 179 func (*AzureInstanceSetSuite) TestManageNics(c *check.C) {
 180         ap, _, _, err := GetInstanceSet()
 181         if err != nil {
 182                 c.Fatal("Error making provider", err)
 183         }
 184
 185         ap.(*AzureInstanceSet).ManageNics()
 186         ap.Stop()
 187 }
 188
 189 func (*AzureInstanceSetSuite) TestManageBlobs(c *check.C) {
 190         ap, _, _, err := GetInstanceSet()
 191         if err != nil {
 192                 c.Fatal("Error making provider", err)
 193         }
 194
 195         ap.(*AzureInstanceSet).ManageBlobs()
 196         ap.Stop()
 197 }
 198
 199 func (*AzureInstanceSetSuite) TestDestroyInstances(c *check.C) {
 200         ap, _, _, err := GetInstanceSet()
 201         if err != nil {
 202                 c.Fatal("Error making provider", err)
 203         }
 204
 205         l, err := ap.Instances(nil)
 206         c.Assert(err, check.IsNil)
 207
 208         for _, i := range l {
 209                 c.Check(i.Destroy(), check.IsNil)
 210         }
 211 }
 212
 213 func (*AzureInstanceSetSuite) TestDeleteFake(c *check.C) {
 214         ap, _, _, err := GetInstanceSet()
 215         if err != nil {
 216                 c.Fatal("Error making provider", err)
 217         }
 218
 219         _, err = ap.(*AzureInstanceSet).netClient.Delete(context.Background(), "fakefakefake", "fakefakefake")
 220
 221         de, ok := err.(autorest.DetailedError)
 222         if ok {
 223                 rq := de.Original.(*azure.RequestError)
 224
 225                 log.Printf("%v %q %q", rq.Response.StatusCode, rq.ServiceError.Code, rq.ServiceError.Message)
 226         }
 227 }
 228
 229 func (*AzureInstanceSetSuite) TestWrapError(c *check.C) {
 230         retryError := autorest.DetailedError{
 231                 Original: &azure.RequestError{
 232                         DetailedError: autorest.DetailedError{
 233                                 Response: &http.Response{
 234                                         StatusCode: 429,
 235                                         Header:     map[string][]string{"Retry-After": []string{"123"}},
 236                                 },
 237                         },
 238                         ServiceError: &azure.ServiceError{},
 239                 },
 240         }
 241         wrapped := WrapAzureError(retryError)
 242         _, ok := wrapped.(RateLimitError)
 243         c.Check(ok, check.Equals, true)
 244
 245         quotaError := autorest.DetailedError{
 246                 Original: &azure.RequestError{
 247                         DetailedError: autorest.DetailedError{
 248                                 Response: &http.Response{
 249                                         StatusCode: 503,
 250                                 },
 251                         },
 252                         ServiceError: &azure.ServiceError{
 253                                 Message: "No more quota",
 254                         },
 255                 },
 256         }
 257         wrapped = WrapAzureError(quotaError)
 258         _, ok = wrapped.(QuotaError)
 259         c.Check(ok, check.Equals, true)
 260 }
 261
 262 func (*AzureInstanceSetSuite) TestSetTags(c *check.C) {
 263         ap, _, _, err := GetInstanceSet()
 264         if err != nil {
 265                 c.Fatal("Error making provider", err)
 266         }
 267         l, err := ap.Instances(nil)
 268         c.Assert(err, check.IsNil)
 269
 270         if len(l) > 0 {
 271                 err = l[0].SetTags(map[string]string{"foo": "bar"})
 272                 if err != nil {
 273                         c.Fatal("Error setting tags", err)
 274                 }
 275         }
 276         l, err = ap.Instances(nil)
 277         c.Assert(err, check.IsNil)
 278
 279         if len(l) > 0 {
 280                 tg := l[0].Tags()
 281                 log.Printf("tags are %v", tg)
 282         }
 283 }
 284
 285 func (*AzureInstanceSetSuite) TestSSH(c *check.C) {
 286         ap, _, _, err := GetInstanceSet()
 287         if err != nil {
 288                 c.Fatal("Error making provider", err)
 289         }
 290         l, err := ap.Instances(nil)
 291         c.Assert(err, check.IsNil)
 292
 293         if len(l) > 0 {
 294
 295                 sshclient, err := SetupSSHClient(c, l[0])
 296                 c.Assert(err, check.IsNil)
 297
 298                 sess, err := sshclient.NewSession()
 299                 c.Assert(err, check.IsNil)
 300
 301                 out, err := sess.Output("cat /home/crunch/node-token")
 302                 c.Assert(err, check.IsNil)
 303
 304                 log.Printf("%v", string(out))
 305
 306                 sshclient.Conn.Close()
 307         }
 308 }
 309
 310 func SetupSSHClient(c *check.C, inst Instance) (*ssh.Client, error) {
 311         addr := inst.Address() + ":2222"
 312         if addr == "" {
 313                 return nil, errors.New("instance has no address")
 314         }
 315
 316         f, err := os.Open("azconfig_sshkey")
 317         c.Assert(err, check.IsNil)
 318
 319         keybytes, err := ioutil.ReadAll(f)
 320         c.Assert(err, check.IsNil)
 321
 322         priv, err := ssh.ParsePrivateKey(keybytes)
 323         c.Assert(err, check.IsNil)
 324
 325         var receivedKey ssh.PublicKey
 326         client, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{
 327                 User: "crunch",
 328                 Auth: []ssh.AuthMethod{
 329                         ssh.PublicKeys(priv),
 330                 },
 331                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
 332                         receivedKey = key
 333                         return nil
 334                 },
 335                 Timeout: time.Minute,
 336         })
 337
 338         if err != nil {
 339                 return nil, err
 340         } else if receivedKey == nil {
 341                 return nil, errors.New("BUG: key was never provided to HostKeyCallback")
 342         }
 343
 344         err = inst.VerifyHostKey(receivedKey, client)
 345         c.Assert(err, check.IsNil)
 346
 347         return client, nil
 348 }