lib/cloud/azure/azure.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package azure
   6
   7 import (
   8         "context"
   9         "encoding/base64"
  10         "encoding/json"
  11         "fmt"
  12         "net/http"
  13         "regexp"
  14         "strconv"
  15         "strings"
  16         "sync"
  17         "time"
  18
  19         "git.curoverse.com/arvados.git/lib/cloud"
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  22         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  23         storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
  24         "github.com/Azure/azure-sdk-for-go/storage"
  25         "github.com/Azure/go-autorest/autorest"
  26         "github.com/Azure/go-autorest/autorest/azure"
  27         "github.com/Azure/go-autorest/autorest/azure/auth"
  28         "github.com/Azure/go-autorest/autorest/to"
  29         "github.com/jmcvetta/randutil"
  30         "github.com/sirupsen/logrus"
  31         "golang.org/x/crypto/ssh"
  32 )
  33
  34 // Driver is the azure implementation of the cloud.Driver interface.
  35 var Driver = cloud.DriverFunc(newAzureInstanceSet)
  36
  37 type azureInstanceSetConfig struct {
  38         SubscriptionID               string
  39         ClientID                     string
  40         ClientSecret                 string
  41         TenantID                     string
  42         CloudEnvironment             string
  43         ResourceGroup                string
  44         Location                     string
  45         Network                      string
  46         Subnet                       string
  47         StorageAccount               string
  48         BlobContainer                string
  49         DeleteDanglingResourcesAfter arvados.Duration
  50         AdminUsername                string
  51 }
  52
  53 const tagKeyInstanceSecret = "InstanceSecret"
  54
  55 type containerWrapper interface {
  56         GetBlobReference(name string) *storage.Blob
  57         ListBlobs(params storage.ListBlobsParameters) (storage.BlobListResponse, error)
  58 }
  59
  60 type virtualMachinesClientWrapper interface {
  61         createOrUpdate(ctx context.Context,
  62                 resourceGroupName string,
  63                 VMName string,
  64                 parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
  65         delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
  66         listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error)
  67 }
  68
  69 type virtualMachinesClientImpl struct {
  70         inner compute.VirtualMachinesClient
  71 }
  72
  73 func (cl *virtualMachinesClientImpl) createOrUpdate(ctx context.Context,
  74         resourceGroupName string,
  75         VMName string,
  76         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  77
  78         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
  79         if err != nil {
  80                 return compute.VirtualMachine{}, wrapAzureError(err)
  81         }
  82         future.WaitForCompletionRef(ctx, cl.inner.Client)
  83         r, err := future.Result(cl.inner)
  84         return r, wrapAzureError(err)
  85 }
  86
  87 func (cl *virtualMachinesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  88         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
  89         if err != nil {
  90                 return nil, wrapAzureError(err)
  91         }
  92         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
  93         return future.Response(), wrapAzureError(err)
  94 }
  95
  96 func (cl *virtualMachinesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  97         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
  98         return r, wrapAzureError(err)
  99 }
 100
 101 type interfacesClientWrapper interface {
 102         createOrUpdate(ctx context.Context,
 103                 resourceGroupName string,
 104                 networkInterfaceName string,
 105                 parameters network.Interface) (result network.Interface, err error)
 106         delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
 107         listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error)
 108 }
 109
 110 type interfacesClientImpl struct {
 111         inner network.InterfacesClient
 112 }
 113
 114 func (cl *interfacesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
 115         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
 116         if err != nil {
 117                 return nil, wrapAzureError(err)
 118         }
 119         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
 120         return future.Response(), wrapAzureError(err)
 121 }
 122
 123 func (cl *interfacesClientImpl) createOrUpdate(ctx context.Context,
 124         resourceGroupName string,
 125         networkInterfaceName string,
 126         parameters network.Interface) (result network.Interface, err error) {
 127
 128         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
 129         if err != nil {
 130                 return network.Interface{}, wrapAzureError(err)
 131         }
 132         future.WaitForCompletionRef(ctx, cl.inner.Client)
 133         r, err := future.Result(cl.inner)
 134         return r, wrapAzureError(err)
 135 }
 136
 137 func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
 138         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
 139         return r, wrapAzureError(err)
 140 }
 141
 142 var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`)
 143
 144 type azureRateLimitError struct {
 145         azure.RequestError
 146         firstRetry time.Time
 147 }
 148
 149 func (ar *azureRateLimitError) EarliestRetry() time.Time {
 150         return ar.firstRetry
 151 }
 152
 153 type azureQuotaError struct {
 154         azure.RequestError
 155 }
 156
 157 func (ar *azureQuotaError) IsQuotaError() bool {
 158         return true
 159 }
 160
 161 func wrapAzureError(err error) error {
 162         de, ok := err.(autorest.DetailedError)
 163         if !ok {
 164                 return err
 165         }
 166         rq, ok := de.Original.(*azure.RequestError)
 167         if !ok {
 168                 return err
 169         }
 170         if rq.Response == nil {
 171                 return err
 172         }
 173         if rq.Response.StatusCode == 429 || len(rq.Response.Header["Retry-After"]) >= 1 {
 174                 // API throttling
 175                 ra := rq.Response.Header["Retry-After"][0]
 176                 earliestRetry, parseErr := http.ParseTime(ra)
 177                 if parseErr != nil {
 178                         // Could not parse as a timestamp, must be number of seconds
 179                         dur, parseErr := strconv.ParseInt(ra, 10, 64)
 180                         if parseErr == nil {
 181                                 earliestRetry = time.Now().Add(time.Duration(dur) * time.Second)
 182                         } else {
 183                                 // Couldn't make sense of retry-after,
 184                                 // so set retry to 20 seconds
 185                                 earliestRetry = time.Now().Add(20 * time.Second)
 186                         }
 187                 }
 188                 return &azureRateLimitError{*rq, earliestRetry}
 189         }
 190         if rq.ServiceError == nil {
 191                 return err
 192         }
 193         if quotaRe.FindString(rq.ServiceError.Code) != "" || quotaRe.FindString(rq.ServiceError.Message) != "" {
 194                 return &azureQuotaError{*rq}
 195         }
 196         return err
 197 }
 198
 199 type azureInstanceSet struct {
 200         azconfig     azureInstanceSetConfig
 201         vmClient     virtualMachinesClientWrapper
 202         netClient    interfacesClientWrapper
 203         blobcont     containerWrapper
 204         azureEnv     azure.Environment
 205         interfaces   map[string]network.Interface
 206         dispatcherID string
 207         namePrefix   string
 208         ctx          context.Context
 209         stopFunc     context.CancelFunc
 210         stopWg       sync.WaitGroup
 211         deleteNIC    chan string
 212         deleteBlob   chan storage.Blob
 213         logger       logrus.FieldLogger
 214 }
 215
 216 func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
 217         azcfg := azureInstanceSetConfig{}
 218         err = json.Unmarshal(config, &azcfg)
 219         if err != nil {
 220                 return nil, err
 221         }
 222
 223         az := azureInstanceSet{logger: logger}
 224         az.ctx, az.stopFunc = context.WithCancel(context.Background())
 225         err = az.setup(azcfg, string(dispatcherID))
 226         if err != nil {
 227                 az.stopFunc()
 228                 return nil, err
 229         }
 230         return &az, nil
 231 }
 232
 233 func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID string) (err error) {
 234         az.azconfig = azcfg
 235         vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
 236         netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
 237         storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
 238
 239         az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment)
 240         if err != nil {
 241                 return err
 242         }
 243
 244         authorizer, err := auth.ClientCredentialsConfig{
 245                 ClientID:     az.azconfig.ClientID,
 246                 ClientSecret: az.azconfig.ClientSecret,
 247                 TenantID:     az.azconfig.TenantID,
 248                 Resource:     az.azureEnv.ResourceManagerEndpoint,
 249                 AADEndpoint:  az.azureEnv.ActiveDirectoryEndpoint,
 250         }.Authorizer()
 251         if err != nil {
 252                 return err
 253         }
 254
 255         vmClient.Authorizer = authorizer
 256         netClient.Authorizer = authorizer
 257         storageAcctClient.Authorizer = authorizer
 258
 259         az.vmClient = &virtualMachinesClientImpl{vmClient}
 260         az.netClient = &interfacesClientImpl{netClient}
 261
 262         result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
 263         if err != nil {
 264                 az.logger.WithError(err).Warn("Couldn't get account keys")
 265                 return err
 266         }
 267
 268         key1 := *(*result.Keys)[0].Value
 269         client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv)
 270         if err != nil {
 271                 az.logger.WithError(err).Warn("Couldn't make client")
 272                 return err
 273         }
 274
 275         blobsvc := client.GetBlobService()
 276         az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer)
 277
 278         az.dispatcherID = dispatcherID
 279         az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
 280
 281         go func() {
 282                 az.stopWg.Add(1)
 283                 defer az.stopWg.Done()
 284
 285                 tk := time.NewTicker(5 * time.Minute)
 286                 for {
 287                         select {
 288                         case <-az.ctx.Done():
 289                                 tk.Stop()
 290                                 return
 291                         case <-tk.C:
 292                                 az.manageBlobs()
 293                         }
 294                 }
 295         }()
 296
 297         az.deleteNIC = make(chan string)
 298         az.deleteBlob = make(chan storage.Blob)
 299
 300         for i := 0; i < 4; i++ {
 301                 go func() {
 302                         for {
 303                                 nicname, ok := <-az.deleteNIC
 304                                 if !ok {
 305                                         return
 306                                 }
 307                                 _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, nicname)
 308                                 if delerr != nil {
 309                                         az.logger.WithError(delerr).Warnf("Error deleting %v", nicname)
 310                                 } else {
 311                                         az.logger.Printf("Deleted NIC %v", nicname)
 312                                 }
 313                         }
 314                 }()
 315                 go func() {
 316                         for {
 317                                 blob, ok := <-az.deleteBlob
 318                                 if !ok {
 319                                         return
 320                                 }
 321                                 err := blob.Delete(nil)
 322                                 if err != nil {
 323                                         az.logger.WithError(err).Warnf("Error deleting %v", blob.Name)
 324                                 } else {
 325                                         az.logger.Printf("Deleted blob %v", blob.Name)
 326                                 }
 327                         }
 328                 }()
 329         }
 330
 331         return nil
 332 }
 333
 334 func (az *azureInstanceSet) Create(
 335         instanceType arvados.InstanceType,
 336         imageID cloud.ImageID,
 337         newTags cloud.InstanceTags,
 338         initCommand cloud.InitCommand,
 339         publicKey ssh.PublicKey) (cloud.Instance, error) {
 340
 341         az.stopWg.Add(1)
 342         defer az.stopWg.Done()
 343
 344         name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
 345         if err != nil {
 346                 return nil, err
 347         }
 348
 349         name = az.namePrefix + name
 350
 351         timestamp := time.Now().Format(time.RFC3339Nano)
 352
 353         tags := make(map[string]*string)
 354         tags["created-at"] = &timestamp
 355         for k, v := range newTags {
 356                 newstr := v
 357                 tags["dispatch-"+k] = &newstr
 358         }
 359
 360         nicParameters := network.Interface{
 361                 Location: &az.azconfig.Location,
 362                 Tags:     tags,
 363                 InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
 364                         IPConfigurations: &[]network.InterfaceIPConfiguration{
 365                                 network.InterfaceIPConfiguration{
 366                                         Name: to.StringPtr("ip1"),
 367                                         InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
 368                                                 Subnet: &network.Subnet{
 369                                                         ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+
 370                                                                 "/Microsoft.Network/virtualnetworks/%s/subnets/%s",
 371                                                                 az.azconfig.SubscriptionID,
 372                                                                 az.azconfig.ResourceGroup,
 373                                                                 az.azconfig.Network,
 374                                                                 az.azconfig.Subnet)),
 375                                                 },
 376                                                 PrivateIPAllocationMethod: network.Dynamic,
 377                                         },
 378                                 },
 379                         },
 380                 },
 381         }
 382         nic, err := az.netClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
 383         if err != nil {
 384                 return nil, wrapAzureError(err)
 385         }
 386
 387         blobname := fmt.Sprintf("%s-os.vhd", name)
 388         instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s",
 389                 az.azconfig.StorageAccount,
 390                 az.azureEnv.StorageEndpointSuffix,
 391                 az.azconfig.BlobContainer,
 392                 blobname)
 393
 394         customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))
 395
 396         vmParameters := compute.VirtualMachine{
 397                 Location: &az.azconfig.Location,
 398                 Tags:     tags,
 399                 VirtualMachineProperties: &compute.VirtualMachineProperties{
 400                         HardwareProfile: &compute.HardwareProfile{
 401                                 VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
 402                         },
 403                         StorageProfile: &compute.StorageProfile{
 404                                 OsDisk: &compute.OSDisk{
 405                                         OsType:       compute.Linux,
 406                                         Name:         to.StringPtr(name + "-os"),
 407                                         CreateOption: compute.FromImage,
 408                                         Image: &compute.VirtualHardDisk{
 409                                                 URI: to.StringPtr(string(imageID)),
 410                                         },
 411                                         Vhd: &compute.VirtualHardDisk{
 412                                                 URI: &instanceVhd,
 413                                         },
 414                                 },
 415                         },
 416                         NetworkProfile: &compute.NetworkProfile{
 417                                 NetworkInterfaces: &[]compute.NetworkInterfaceReference{
 418                                         compute.NetworkInterfaceReference{
 419                                                 ID: nic.ID,
 420                                                 NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
 421                                                         Primary: to.BoolPtr(true),
 422                                                 },
 423                                         },
 424                                 },
 425                         },
 426                         OsProfile: &compute.OSProfile{
 427                                 ComputerName:  &name,
 428                                 AdminUsername: to.StringPtr(az.azconfig.AdminUsername),
 429                                 LinuxConfiguration: &compute.LinuxConfiguration{
 430                                         DisablePasswordAuthentication: to.BoolPtr(true),
 431                                         SSH: &compute.SSHConfiguration{
 432                                                 PublicKeys: &[]compute.SSHPublicKey{
 433                                                         {
 434                                                                 Path:    to.StringPtr("/home/" + az.azconfig.AdminUsername + "/.ssh/authorized_keys"),
 435                                                                 KeyData: to.StringPtr(string(ssh.MarshalAuthorizedKey(publicKey))),
 436                                                         },
 437                                                 },
 438                                         },
 439                                 },
 440                                 CustomData: &customData,
 441                         },
 442                 },
 443         }
 444
 445         vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters)
 446         if err != nil {
 447                 _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil)
 448                 if delerr != nil {
 449                         az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create")
 450                 }
 451
 452                 _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name)
 453                 if delerr != nil {
 454                         az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create")
 455                 }
 456
 457                 return nil, wrapAzureError(err)
 458         }
 459
 460         return &azureInstance{
 461                 provider: az,
 462                 nic:      nic,
 463                 vm:       vm,
 464         }, nil
 465 }
 466
 467 func (az *azureInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
 468         az.stopWg.Add(1)
 469         defer az.stopWg.Done()
 470
 471         interfaces, err := az.manageNics()
 472         if err != nil {
 473                 return nil, err
 474         }
 475
 476         result, err := az.vmClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 477         if err != nil {
 478                 return nil, wrapAzureError(err)
 479         }
 480
 481         instances := make([]cloud.Instance, 0)
 482
 483         for ; result.NotDone(); err = result.Next() {
 484                 if err != nil {
 485                         return nil, wrapAzureError(err)
 486                 }
 487                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 488                         instances = append(instances, &azureInstance{
 489                                 provider: az,
 490                                 vm:       result.Value(),
 491                                 nic:      interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
 492                 }
 493         }
 494         return instances, nil
 495 }
 496
 497 // ManageNics returns a list of Azure network interface resources.
 498 // Also performs garbage collection of NICs which have "namePrefix", are
 499 // not associated with a virtual machine and have a "create-at" time
 500 // more than DeleteDanglingResourcesAfter (to prevent racing and
 501 // deleting newly created NICs) in the past are deleted.
 502 func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) {
 503         az.stopWg.Add(1)
 504         defer az.stopWg.Done()
 505
 506         result, err := az.netClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 507         if err != nil {
 508                 return nil, wrapAzureError(err)
 509         }
 510
 511         interfaces := make(map[string]network.Interface)
 512
 513         timestamp := time.Now()
 514         for ; result.NotDone(); err = result.Next() {
 515                 if err != nil {
 516                         az.logger.WithError(err).Warnf("Error listing nics")
 517                         return interfaces, nil
 518                 }
 519                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 520                         if result.Value().VirtualMachine != nil {
 521                                 interfaces[*result.Value().ID] = result.Value()
 522                         } else {
 523                                 if result.Value().Tags["created-at"] != nil {
 524                                         createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
 525                                         if err == nil {
 526                                                 if timestamp.Sub(createdAt) > az.azconfig.DeleteDanglingResourcesAfter.Duration() {
 527                                                         az.logger.Printf("Will delete %v because it is older than %s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
 528                                                         az.deleteNIC <- *result.Value().Name
 529                                                 }
 530                                         }
 531                                 }
 532                         }
 533                 }
 534         }
 535         return interfaces, nil
 536 }
 537
 538 // ManageBlobs garbage collects blobs (VM disk images) in the
 539 // configured storage account container.  It will delete blobs which
 540 // have "namePrefix", are "available" (which means they are not
 541 // leased to a VM) and haven't been modified for
 542 // DeleteDanglingResourcesAfter seconds.
 543 func (az *azureInstanceSet) manageBlobs() {
 544
 545         page := storage.ListBlobsParameters{Prefix: az.namePrefix}
 546         timestamp := time.Now()
 547
 548         for {
 549                 response, err := az.blobcont.ListBlobs(page)
 550                 if err != nil {
 551                         az.logger.WithError(err).Warn("Error listing blobs")
 552                         return
 553                 }
 554                 for _, b := range response.Blobs {
 555                         age := timestamp.Sub(time.Time(b.Properties.LastModified))
 556                         if b.Properties.BlobType == storage.BlobTypePage &&
 557                                 b.Properties.LeaseState == "available" &&
 558                                 b.Properties.LeaseStatus == "unlocked" &&
 559                                 age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
 560
 561                                 az.logger.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
 562                                 az.deleteBlob <- b
 563                         }
 564                 }
 565                 if response.NextMarker != "" {
 566                         page.Marker = response.NextMarker
 567                 } else {
 568                         break
 569                 }
 570         }
 571 }
 572
 573 func (az *azureInstanceSet) Stop() {
 574         az.stopFunc()
 575         az.stopWg.Wait()
 576         close(az.deleteNIC)
 577         close(az.deleteBlob)
 578 }
 579
 580 type azureInstance struct {
 581         provider *azureInstanceSet
 582         nic      network.Interface
 583         vm       compute.VirtualMachine
 584 }
 585
 586 func (ai *azureInstance) ID() cloud.InstanceID {
 587         return cloud.InstanceID(*ai.vm.ID)
 588 }
 589
 590 func (ai *azureInstance) String() string {
 591         return *ai.vm.Name
 592 }
 593
 594 func (ai *azureInstance) ProviderType() string {
 595         return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
 596 }
 597
 598 func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error {
 599         ai.provider.stopWg.Add(1)
 600         defer ai.provider.stopWg.Done()
 601
 602         tags := make(map[string]*string)
 603
 604         for k, v := range ai.vm.Tags {
 605                 if !strings.HasPrefix(k, "dispatch-") {
 606                         tags[k] = v
 607                 }
 608         }
 609         for k, v := range newTags {
 610                 newstr := v
 611                 tags["dispatch-"+k] = &newstr
 612         }
 613
 614         vmParameters := compute.VirtualMachine{
 615                 Location: &ai.provider.azconfig.Location,
 616                 Tags:     tags,
 617         }
 618         vm, err := ai.provider.vmClient.createOrUpdate(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
 619         if err != nil {
 620                 return wrapAzureError(err)
 621         }
 622         ai.vm = vm
 623
 624         return nil
 625 }
 626
 627 func (ai *azureInstance) Tags() cloud.InstanceTags {
 628         tags := make(map[string]string)
 629
 630         for k, v := range ai.vm.Tags {
 631                 if strings.HasPrefix(k, "dispatch-") {
 632                         tags[k[9:]] = *v
 633                 }
 634         }
 635
 636         return tags
 637 }
 638
 639 func (ai *azureInstance) Destroy() error {
 640         ai.provider.stopWg.Add(1)
 641         defer ai.provider.stopWg.Done()
 642
 643         _, err := ai.provider.vmClient.delete(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
 644         return wrapAzureError(err)
 645 }
 646
 647 func (ai *azureInstance) Address() string {
 648         if ai.nic.IPConfigurations != nil &&
 649                 len(*ai.nic.IPConfigurations) > 0 &&
 650                 (*ai.nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat != nil &&
 651                 (*ai.nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat.PrivateIPAddress != nil {
 652
 653                 return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
 654         }
 655         return ""
 656 }
 657
 658 func (ai *azureInstance) RemoteUser() string {
 659         return ai.provider.azconfig.AdminUsername
 660 }
 661
 662 func (ai *azureInstance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
 663         return cloud.ErrNotImplemented
 664 }