lib/cloud/azure/azure.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package azure
   6
   7 import (
   8         "context"
   9         "encoding/base64"
  10         "encoding/json"
  11         "fmt"
  12         "net/http"
  13         "regexp"
  14         "strconv"
  15         "strings"
  16         "sync"
  17         "time"
  18
  19         "git.curoverse.com/arvados.git/lib/cloud"
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  22         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  23         storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
  24         "github.com/Azure/azure-sdk-for-go/storage"
  25         "github.com/Azure/go-autorest/autorest"
  26         "github.com/Azure/go-autorest/autorest/azure"
  27         "github.com/Azure/go-autorest/autorest/azure/auth"
  28         "github.com/Azure/go-autorest/autorest/to"
  29         "github.com/jmcvetta/randutil"
  30         "github.com/sirupsen/logrus"
  31         "golang.org/x/crypto/ssh"
  32 )
  33
  34 // Driver is the azure implementation of the cloud.Driver interface.
  35 var Driver = cloud.DriverFunc(newAzureInstanceSet)
  36
  37 type azureInstanceSetConfig struct {
  38         SubscriptionID               string
  39         ClientID                     string
  40         ClientSecret                 string
  41         TenantID                     string
  42         CloudEnvironment             string
  43         ResourceGroup                string
  44         Location                     string
  45         Network                      string
  46         Subnet                       string
  47         StorageAccount               string
  48         BlobContainer                string
  49         DeleteDanglingResourcesAfter arvados.Duration
  50         AdminUsername                string
  51 }
  52
  53 type containerWrapper interface {
  54         GetBlobReference(name string) *storage.Blob
  55         ListBlobs(params storage.ListBlobsParameters) (storage.BlobListResponse, error)
  56 }
  57
  58 type virtualMachinesClientWrapper interface {
  59         createOrUpdate(ctx context.Context,
  60                 resourceGroupName string,
  61                 VMName string,
  62                 parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
  63         delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
  64         listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error)
  65 }
  66
  67 type virtualMachinesClientImpl struct {
  68         inner compute.VirtualMachinesClient
  69 }
  70
  71 func (cl *virtualMachinesClientImpl) createOrUpdate(ctx context.Context,
  72         resourceGroupName string,
  73         VMName string,
  74         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  75
  76         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
  77         if err != nil {
  78                 return compute.VirtualMachine{}, wrapAzureError(err)
  79         }
  80         future.WaitForCompletionRef(ctx, cl.inner.Client)
  81         r, err := future.Result(cl.inner)
  82         return r, wrapAzureError(err)
  83 }
  84
  85 func (cl *virtualMachinesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  86         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
  87         if err != nil {
  88                 return nil, wrapAzureError(err)
  89         }
  90         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
  91         return future.Response(), wrapAzureError(err)
  92 }
  93
  94 func (cl *virtualMachinesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  95         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
  96         return r, wrapAzureError(err)
  97 }
  98
  99 type interfacesClientWrapper interface {
 100         createOrUpdate(ctx context.Context,
 101                 resourceGroupName string,
 102                 networkInterfaceName string,
 103                 parameters network.Interface) (result network.Interface, err error)
 104         delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
 105         listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error)
 106 }
 107
 108 type interfacesClientImpl struct {
 109         inner network.InterfacesClient
 110 }
 111
 112 func (cl *interfacesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
 113         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
 114         if err != nil {
 115                 return nil, wrapAzureError(err)
 116         }
 117         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
 118         return future.Response(), wrapAzureError(err)
 119 }
 120
 121 func (cl *interfacesClientImpl) createOrUpdate(ctx context.Context,
 122         resourceGroupName string,
 123         networkInterfaceName string,
 124         parameters network.Interface) (result network.Interface, err error) {
 125
 126         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
 127         if err != nil {
 128                 return network.Interface{}, wrapAzureError(err)
 129         }
 130         future.WaitForCompletionRef(ctx, cl.inner.Client)
 131         r, err := future.Result(cl.inner)
 132         return r, wrapAzureError(err)
 133 }
 134
 135 func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
 136         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
 137         return r, wrapAzureError(err)
 138 }
 139
 140 var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`)
 141
 142 type azureRateLimitError struct {
 143         azure.RequestError
 144         firstRetry time.Time
 145 }
 146
 147 func (ar *azureRateLimitError) EarliestRetry() time.Time {
 148         return ar.firstRetry
 149 }
 150
 151 type azureQuotaError struct {
 152         azure.RequestError
 153 }
 154
 155 func (ar *azureQuotaError) IsQuotaError() bool {
 156         return true
 157 }
 158
 159 func wrapAzureError(err error) error {
 160         de, ok := err.(autorest.DetailedError)
 161         if !ok {
 162                 return err
 163         }
 164         rq, ok := de.Original.(*azure.RequestError)
 165         if !ok {
 166                 return err
 167         }
 168         if rq.Response == nil {
 169                 return err
 170         }
 171         if rq.Response.StatusCode == 429 || len(rq.Response.Header["Retry-After"]) >= 1 {
 172                 // API throttling
 173                 ra := rq.Response.Header["Retry-After"][0]
 174                 earliestRetry, parseErr := http.ParseTime(ra)
 175                 if parseErr != nil {
 176                         // Could not parse as a timestamp, must be number of seconds
 177                         dur, parseErr := strconv.ParseInt(ra, 10, 64)
 178                         if parseErr == nil {
 179                                 earliestRetry = time.Now().Add(time.Duration(dur) * time.Second)
 180                         } else {
 181                                 // Couldn't make sense of retry-after,
 182                                 // so set retry to 20 seconds
 183                                 earliestRetry = time.Now().Add(20 * time.Second)
 184                         }
 185                 }
 186                 return &azureRateLimitError{*rq, earliestRetry}
 187         }
 188         if rq.ServiceError == nil {
 189                 return err
 190         }
 191         if quotaRe.FindString(rq.ServiceError.Code) != "" || quotaRe.FindString(rq.ServiceError.Message) != "" {
 192                 return &azureQuotaError{*rq}
 193         }
 194         return err
 195 }
 196
 197 type azureInstanceSet struct {
 198         azconfig     azureInstanceSetConfig
 199         vmClient     virtualMachinesClientWrapper
 200         netClient    interfacesClientWrapper
 201         blobcont     containerWrapper
 202         azureEnv     azure.Environment
 203         interfaces   map[string]network.Interface
 204         dispatcherID string
 205         namePrefix   string
 206         ctx          context.Context
 207         stopFunc     context.CancelFunc
 208         stopWg       sync.WaitGroup
 209         deleteNIC    chan string
 210         deleteBlob   chan storage.Blob
 211         logger       logrus.FieldLogger
 212 }
 213
 214 func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
 215         azcfg := azureInstanceSetConfig{}
 216         err = json.Unmarshal(config, &azcfg)
 217         if err != nil {
 218                 return nil, err
 219         }
 220
 221         az := azureInstanceSet{logger: logger}
 222         az.ctx, az.stopFunc = context.WithCancel(context.Background())
 223         err = az.setup(azcfg, string(dispatcherID))
 224         if err != nil {
 225                 az.stopFunc()
 226                 return nil, err
 227         }
 228         return &az, nil
 229 }
 230
 231 func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID string) (err error) {
 232         az.azconfig = azcfg
 233         vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
 234         netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
 235         storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
 236
 237         az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment)
 238         if err != nil {
 239                 return err
 240         }
 241
 242         authorizer, err := auth.ClientCredentialsConfig{
 243                 ClientID:     az.azconfig.ClientID,
 244                 ClientSecret: az.azconfig.ClientSecret,
 245                 TenantID:     az.azconfig.TenantID,
 246                 Resource:     az.azureEnv.ResourceManagerEndpoint,
 247                 AADEndpoint:  az.azureEnv.ActiveDirectoryEndpoint,
 248         }.Authorizer()
 249         if err != nil {
 250                 return err
 251         }
 252
 253         vmClient.Authorizer = authorizer
 254         netClient.Authorizer = authorizer
 255         storageAcctClient.Authorizer = authorizer
 256
 257         az.vmClient = &virtualMachinesClientImpl{vmClient}
 258         az.netClient = &interfacesClientImpl{netClient}
 259
 260         result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
 261         if err != nil {
 262                 az.logger.WithError(err).Warn("Couldn't get account keys")
 263                 return err
 264         }
 265
 266         key1 := *(*result.Keys)[0].Value
 267         client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv)
 268         if err != nil {
 269                 az.logger.WithError(err).Warn("Couldn't make client")
 270                 return err
 271         }
 272
 273         blobsvc := client.GetBlobService()
 274         az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer)
 275
 276         az.dispatcherID = dispatcherID
 277         az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
 278
 279         go func() {
 280                 az.stopWg.Add(1)
 281                 defer az.stopWg.Done()
 282
 283                 tk := time.NewTicker(5 * time.Minute)
 284                 for {
 285                         select {
 286                         case <-az.ctx.Done():
 287                                 tk.Stop()
 288                                 return
 289                         case <-tk.C:
 290                                 az.manageBlobs()
 291                         }
 292                 }
 293         }()
 294
 295         az.deleteNIC = make(chan string)
 296         az.deleteBlob = make(chan storage.Blob)
 297
 298         for i := 0; i < 4; i++ {
 299                 go func() {
 300                         for {
 301                                 nicname, ok := <-az.deleteNIC
 302                                 if !ok {
 303                                         return
 304                                 }
 305                                 _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, nicname)
 306                                 if delerr != nil {
 307                                         az.logger.WithError(delerr).Warnf("Error deleting %v", nicname)
 308                                 } else {
 309                                         az.logger.Printf("Deleted NIC %v", nicname)
 310                                 }
 311                         }
 312                 }()
 313                 go func() {
 314                         for {
 315                                 blob, ok := <-az.deleteBlob
 316                                 if !ok {
 317                                         return
 318                                 }
 319                                 err := blob.Delete(nil)
 320                                 if err != nil {
 321                                         az.logger.WithError(err).Warnf("Error deleting %v", blob.Name)
 322                                 } else {
 323                                         az.logger.Printf("Deleted blob %v", blob.Name)
 324                                 }
 325                         }
 326                 }()
 327         }
 328
 329         return nil
 330 }
 331
 332 func (az *azureInstanceSet) Create(
 333         instanceType arvados.InstanceType,
 334         imageID cloud.ImageID,
 335         newTags cloud.InstanceTags,
 336         initCommand cloud.InitCommand,
 337         publicKey ssh.PublicKey) (cloud.Instance, error) {
 338
 339         az.stopWg.Add(1)
 340         defer az.stopWg.Done()
 341
 342         if instanceType.AddedScratch > 0 {
 343                 return nil, fmt.Errorf("cannot create instance type %q: driver does not implement non-zero AddedScratch (%d)", instanceType.Name, instanceType.AddedScratch)
 344         }
 345
 346         name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
 347         if err != nil {
 348                 return nil, err
 349         }
 350
 351         name = az.namePrefix + name
 352
 353         tags := map[string]*string{}
 354         for k, v := range newTags {
 355                 tags[k] = to.StringPtr(v)
 356         }
 357         tags["created-at"] = to.StringPtr(time.Now().Format(time.RFC3339Nano))
 358
 359         nicParameters := network.Interface{
 360                 Location: &az.azconfig.Location,
 361                 Tags:     tags,
 362                 InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
 363                         IPConfigurations: &[]network.InterfaceIPConfiguration{
 364                                 network.InterfaceIPConfiguration{
 365                                         Name: to.StringPtr("ip1"),
 366                                         InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
 367                                                 Subnet: &network.Subnet{
 368                                                         ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+
 369                                                                 "/Microsoft.Network/virtualnetworks/%s/subnets/%s",
 370                                                                 az.azconfig.SubscriptionID,
 371                                                                 az.azconfig.ResourceGroup,
 372                                                                 az.azconfig.Network,
 373                                                                 az.azconfig.Subnet)),
 374                                                 },
 375                                                 PrivateIPAllocationMethod: network.Dynamic,
 376                                         },
 377                                 },
 378                         },
 379                 },
 380         }
 381         nic, err := az.netClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
 382         if err != nil {
 383                 return nil, wrapAzureError(err)
 384         }
 385
 386         blobname := fmt.Sprintf("%s-os.vhd", name)
 387         instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s",
 388                 az.azconfig.StorageAccount,
 389                 az.azureEnv.StorageEndpointSuffix,
 390                 az.azconfig.BlobContainer,
 391                 blobname)
 392
 393         customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))
 394
 395         vmParameters := compute.VirtualMachine{
 396                 Location: &az.azconfig.Location,
 397                 Tags:     tags,
 398                 VirtualMachineProperties: &compute.VirtualMachineProperties{
 399                         HardwareProfile: &compute.HardwareProfile{
 400                                 VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
 401                         },
 402                         StorageProfile: &compute.StorageProfile{
 403                                 OsDisk: &compute.OSDisk{
 404                                         OsType:       compute.Linux,
 405                                         Name:         to.StringPtr(name + "-os"),
 406                                         CreateOption: compute.FromImage,
 407                                         Image: &compute.VirtualHardDisk{
 408                                                 URI: to.StringPtr(string(imageID)),
 409                                         },
 410                                         Vhd: &compute.VirtualHardDisk{
 411                                                 URI: &instanceVhd,
 412                                         },
 413                                 },
 414                         },
 415                         NetworkProfile: &compute.NetworkProfile{
 416                                 NetworkInterfaces: &[]compute.NetworkInterfaceReference{
 417                                         compute.NetworkInterfaceReference{
 418                                                 ID: nic.ID,
 419                                                 NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
 420                                                         Primary: to.BoolPtr(true),
 421                                                 },
 422                                         },
 423                                 },
 424                         },
 425                         OsProfile: &compute.OSProfile{
 426                                 ComputerName:  &name,
 427                                 AdminUsername: to.StringPtr(az.azconfig.AdminUsername),
 428                                 LinuxConfiguration: &compute.LinuxConfiguration{
 429                                         DisablePasswordAuthentication: to.BoolPtr(true),
 430                                         SSH: &compute.SSHConfiguration{
 431                                                 PublicKeys: &[]compute.SSHPublicKey{
 432                                                         {
 433                                                                 Path:    to.StringPtr("/home/" + az.azconfig.AdminUsername + "/.ssh/authorized_keys"),
 434                                                                 KeyData: to.StringPtr(string(ssh.MarshalAuthorizedKey(publicKey))),
 435                                                         },
 436                                                 },
 437                                         },
 438                                 },
 439                                 CustomData: &customData,
 440                         },
 441                 },
 442         }
 443
 444         vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters)
 445         if err != nil {
 446                 _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil)
 447                 if delerr != nil {
 448                         az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create")
 449                 }
 450
 451                 _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name)
 452                 if delerr != nil {
 453                         az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create")
 454                 }
 455
 456                 return nil, wrapAzureError(err)
 457         }
 458
 459         return &azureInstance{
 460                 provider: az,
 461                 nic:      nic,
 462                 vm:       vm,
 463         }, nil
 464 }
 465
 466 func (az *azureInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
 467         az.stopWg.Add(1)
 468         defer az.stopWg.Done()
 469
 470         interfaces, err := az.manageNics()
 471         if err != nil {
 472                 return nil, err
 473         }
 474
 475         result, err := az.vmClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 476         if err != nil {
 477                 return nil, wrapAzureError(err)
 478         }
 479
 480         var instances []cloud.Instance
 481         for ; result.NotDone(); err = result.Next() {
 482                 if err != nil {
 483                         return nil, wrapAzureError(err)
 484                 }
 485                 instances = append(instances, &azureInstance{
 486                         provider: az,
 487                         vm:       result.Value(),
 488                         nic:      interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID],
 489                 })
 490         }
 491         return instances, nil
 492 }
 493
 494 // ManageNics returns a list of Azure network interface resources.
 495 // Also performs garbage collection of NICs which have "namePrefix",
 496 // are not associated with a virtual machine and have a "created-at"
 497 // time more than DeleteDanglingResourcesAfter (to prevent racing and
 498 // deleting newly created NICs) in the past are deleted.
 499 func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) {
 500         az.stopWg.Add(1)
 501         defer az.stopWg.Done()
 502
 503         result, err := az.netClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 504         if err != nil {
 505                 return nil, wrapAzureError(err)
 506         }
 507
 508         interfaces := make(map[string]network.Interface)
 509
 510         timestamp := time.Now()
 511         for ; result.NotDone(); err = result.Next() {
 512                 if err != nil {
 513                         az.logger.WithError(err).Warnf("Error listing nics")
 514                         return interfaces, nil
 515                 }
 516                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 517                         if result.Value().VirtualMachine != nil {
 518                                 interfaces[*result.Value().ID] = result.Value()
 519                         } else {
 520                                 if result.Value().Tags["created-at"] != nil {
 521                                         createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
 522                                         if err == nil {
 523                                                 if timestamp.Sub(createdAt) > az.azconfig.DeleteDanglingResourcesAfter.Duration() {
 524                                                         az.logger.Printf("Will delete %v because it is older than %s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
 525                                                         az.deleteNIC <- *result.Value().Name
 526                                                 }
 527                                         }
 528                                 }
 529                         }
 530                 }
 531         }
 532         return interfaces, nil
 533 }
 534
 535 // ManageBlobs garbage collects blobs (VM disk images) in the
 536 // configured storage account container.  It will delete blobs which
 537 // have "namePrefix", are "available" (which means they are not
 538 // leased to a VM) and haven't been modified for
 539 // DeleteDanglingResourcesAfter seconds.
 540 func (az *azureInstanceSet) manageBlobs() {
 541
 542         page := storage.ListBlobsParameters{Prefix: az.namePrefix}
 543         timestamp := time.Now()
 544
 545         for {
 546                 response, err := az.blobcont.ListBlobs(page)
 547                 if err != nil {
 548                         az.logger.WithError(err).Warn("Error listing blobs")
 549                         return
 550                 }
 551                 for _, b := range response.Blobs {
 552                         age := timestamp.Sub(time.Time(b.Properties.LastModified))
 553                         if b.Properties.BlobType == storage.BlobTypePage &&
 554                                 b.Properties.LeaseState == "available" &&
 555                                 b.Properties.LeaseStatus == "unlocked" &&
 556                                 age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
 557
 558                                 az.logger.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
 559                                 az.deleteBlob <- b
 560                         }
 561                 }
 562                 if response.NextMarker != "" {
 563                         page.Marker = response.NextMarker
 564                 } else {
 565                         break
 566                 }
 567         }
 568 }
 569
 570 func (az *azureInstanceSet) Stop() {
 571         az.stopFunc()
 572         az.stopWg.Wait()
 573         close(az.deleteNIC)
 574         close(az.deleteBlob)
 575 }
 576
 577 type azureInstance struct {
 578         provider *azureInstanceSet
 579         nic      network.Interface
 580         vm       compute.VirtualMachine
 581 }
 582
 583 func (ai *azureInstance) ID() cloud.InstanceID {
 584         return cloud.InstanceID(*ai.vm.ID)
 585 }
 586
 587 func (ai *azureInstance) String() string {
 588         return *ai.vm.Name
 589 }
 590
 591 func (ai *azureInstance) ProviderType() string {
 592         return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
 593 }
 594
 595 func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error {
 596         ai.provider.stopWg.Add(1)
 597         defer ai.provider.stopWg.Done()
 598
 599         tags := map[string]*string{}
 600         for k, v := range ai.vm.Tags {
 601                 tags[k] = v
 602         }
 603         for k, v := range newTags {
 604                 tags[k] = to.StringPtr(v)
 605         }
 606
 607         vmParameters := compute.VirtualMachine{
 608                 Location: &ai.provider.azconfig.Location,
 609                 Tags:     tags,
 610         }
 611         vm, err := ai.provider.vmClient.createOrUpdate(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
 612         if err != nil {
 613                 return wrapAzureError(err)
 614         }
 615         ai.vm = vm
 616
 617         return nil
 618 }
 619
 620 func (ai *azureInstance) Tags() cloud.InstanceTags {
 621         tags := cloud.InstanceTags{}
 622         for k, v := range ai.vm.Tags {
 623                 tags[k] = *v
 624         }
 625         return tags
 626 }
 627
 628 func (ai *azureInstance) Destroy() error {
 629         ai.provider.stopWg.Add(1)
 630         defer ai.provider.stopWg.Done()
 631
 632         _, err := ai.provider.vmClient.delete(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
 633         return wrapAzureError(err)
 634 }
 635
 636 func (ai *azureInstance) Address() string {
 637         if iprops := ai.nic.InterfacePropertiesFormat; iprops == nil {
 638                 return ""
 639         } else if ipconfs := iprops.IPConfigurations; ipconfs == nil || len(*ipconfs) == 0 {
 640                 return ""
 641         } else if ipconfprops := (*ipconfs)[0].InterfaceIPConfigurationPropertiesFormat; ipconfprops == nil {
 642                 return ""
 643         } else if addr := ipconfprops.PrivateIPAddress; addr == nil {
 644                 return ""
 645         } else {
 646                 return *addr
 647         }
 648 }
 649
 650 func (ai *azureInstance) RemoteUser() string {
 651         return ai.provider.azconfig.AdminUsername
 652 }
 653
 654 func (ai *azureInstance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
 655         return cloud.ErrNotImplemented
 656 }