lib/cloud/azure/azure.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package azure
   6
   7 import (
   8         "context"
   9         "encoding/base64"
  10         "encoding/json"
  11         "fmt"
  12         "net/http"
  13         "regexp"
  14         "strconv"
  15         "strings"
  16         "sync"
  17         "time"
  18
  19         "git.curoverse.com/arvados.git/lib/cloud"
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  22         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  23         storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
  24         "github.com/Azure/azure-sdk-for-go/storage"
  25         "github.com/Azure/go-autorest/autorest"
  26         "github.com/Azure/go-autorest/autorest/azure"
  27         "github.com/Azure/go-autorest/autorest/azure/auth"
  28         "github.com/Azure/go-autorest/autorest/to"
  29         "github.com/jmcvetta/randutil"
  30         "github.com/sirupsen/logrus"
  31         "golang.org/x/crypto/ssh"
  32 )
  33
  34 // Driver is the azure implementation of the cloud.Driver interface.
  35 var Driver = cloud.DriverFunc(newAzureInstanceSet)
  36
  37 type azureInstanceSetConfig struct {
  38         SubscriptionID               string
  39         ClientID                     string
  40         ClientSecret                 string
  41         TenantID                     string
  42         CloudEnvironment             string
  43         ResourceGroup                string
  44         Location                     string
  45         Network                      string
  46         Subnet                       string
  47         StorageAccount               string
  48         BlobContainer                string
  49         DeleteDanglingResourcesAfter arvados.Duration
  50         AdminUsername                string
  51 }
  52
  53 const tagKeyInstanceSecret = "InstanceSecret"
  54
  55 type containerWrapper interface {
  56         GetBlobReference(name string) *storage.Blob
  57         ListBlobs(params storage.ListBlobsParameters) (storage.BlobListResponse, error)
  58 }
  59
  60 type virtualMachinesClientWrapper interface {
  61         createOrUpdate(ctx context.Context,
  62                 resourceGroupName string,
  63                 VMName string,
  64                 parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
  65         delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
  66         listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error)
  67 }
  68
  69 type virtualMachinesClientImpl struct {
  70         inner compute.VirtualMachinesClient
  71 }
  72
  73 func (cl *virtualMachinesClientImpl) createOrUpdate(ctx context.Context,
  74         resourceGroupName string,
  75         VMName string,
  76         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  77
  78         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
  79         if err != nil {
  80                 return compute.VirtualMachine{}, wrapAzureError(err)
  81         }
  82         future.WaitForCompletionRef(ctx, cl.inner.Client)
  83         r, err := future.Result(cl.inner)
  84         return r, wrapAzureError(err)
  85 }
  86
  87 func (cl *virtualMachinesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  88         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
  89         if err != nil {
  90                 return nil, wrapAzureError(err)
  91         }
  92         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
  93         return future.Response(), wrapAzureError(err)
  94 }
  95
  96 func (cl *virtualMachinesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  97         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
  98         return r, wrapAzureError(err)
  99 }
 100
 101 type interfacesClientWrapper interface {
 102         createOrUpdate(ctx context.Context,
 103                 resourceGroupName string,
 104                 networkInterfaceName string,
 105                 parameters network.Interface) (result network.Interface, err error)
 106         delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
 107         listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error)
 108 }
 109
 110 type interfacesClientImpl struct {
 111         inner network.InterfacesClient
 112 }
 113
 114 func (cl *interfacesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
 115         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
 116         if err != nil {
 117                 return nil, wrapAzureError(err)
 118         }
 119         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
 120         return future.Response(), wrapAzureError(err)
 121 }
 122
 123 func (cl *interfacesClientImpl) createOrUpdate(ctx context.Context,
 124         resourceGroupName string,
 125         networkInterfaceName string,
 126         parameters network.Interface) (result network.Interface, err error) {
 127
 128         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
 129         if err != nil {
 130                 return network.Interface{}, wrapAzureError(err)
 131         }
 132         future.WaitForCompletionRef(ctx, cl.inner.Client)
 133         r, err := future.Result(cl.inner)
 134         return r, wrapAzureError(err)
 135 }
 136
 137 func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
 138         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
 139         return r, wrapAzureError(err)
 140 }
 141
 142 var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`)
 143
 144 type azureRateLimitError struct {
 145         azure.RequestError
 146         firstRetry time.Time
 147 }
 148
 149 func (ar *azureRateLimitError) EarliestRetry() time.Time {
 150         return ar.firstRetry
 151 }
 152
 153 type azureQuotaError struct {
 154         azure.RequestError
 155 }
 156
 157 func (ar *azureQuotaError) IsQuotaError() bool {
 158         return true
 159 }
 160
 161 func wrapAzureError(err error) error {
 162         de, ok := err.(autorest.DetailedError)
 163         if !ok {
 164                 return err
 165         }
 166         rq, ok := de.Original.(*azure.RequestError)
 167         if !ok {
 168                 return err
 169         }
 170         if rq.Response == nil {
 171                 return err
 172         }
 173         if rq.Response.StatusCode == 429 || len(rq.Response.Header["Retry-After"]) >= 1 {
 174                 // API throttling
 175                 ra := rq.Response.Header["Retry-After"][0]
 176                 earliestRetry, parseErr := http.ParseTime(ra)
 177                 if parseErr != nil {
 178                         // Could not parse as a timestamp, must be number of seconds
 179                         dur, parseErr := strconv.ParseInt(ra, 10, 64)
 180                         if parseErr == nil {
 181                                 earliestRetry = time.Now().Add(time.Duration(dur) * time.Second)
 182                         } else {
 183                                 // Couldn't make sense of retry-after,
 184                                 // so set retry to 20 seconds
 185                                 earliestRetry = time.Now().Add(20 * time.Second)
 186                         }
 187                 }
 188                 return &azureRateLimitError{*rq, earliestRetry}
 189         }
 190         if rq.ServiceError == nil {
 191                 return err
 192         }
 193         if quotaRe.FindString(rq.ServiceError.Code) != "" || quotaRe.FindString(rq.ServiceError.Message) != "" {
 194                 return &azureQuotaError{*rq}
 195         }
 196         return err
 197 }
 198
 199 type azureInstanceSet struct {
 200         azconfig     azureInstanceSetConfig
 201         vmClient     virtualMachinesClientWrapper
 202         netClient    interfacesClientWrapper
 203         blobcont     containerWrapper
 204         azureEnv     azure.Environment
 205         interfaces   map[string]network.Interface
 206         dispatcherID string
 207         namePrefix   string
 208         ctx          context.Context
 209         stopFunc     context.CancelFunc
 210         stopWg       sync.WaitGroup
 211         deleteNIC    chan string
 212         deleteBlob   chan storage.Blob
 213         logger       logrus.FieldLogger
 214 }
 215
 216 func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
 217         azcfg := azureInstanceSetConfig{}
 218         err = json.Unmarshal(config, &azcfg)
 219         if err != nil {
 220                 return nil, err
 221         }
 222
 223         ap := azureInstanceSet{logger: logger}
 224         err = ap.setup(azcfg, string(dispatcherID))
 225         if err != nil {
 226                 return nil, err
 227         }
 228         return &ap, nil
 229 }
 230
 231 func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID string) (err error) {
 232         az.azconfig = azcfg
 233         vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
 234         netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
 235         storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
 236
 237         az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment)
 238         if err != nil {
 239                 return err
 240         }
 241
 242         authorizer, err := auth.ClientCredentialsConfig{
 243                 ClientID:     az.azconfig.ClientID,
 244                 ClientSecret: az.azconfig.ClientSecret,
 245                 TenantID:     az.azconfig.TenantID,
 246                 Resource:     az.azureEnv.ResourceManagerEndpoint,
 247                 AADEndpoint:  az.azureEnv.ActiveDirectoryEndpoint,
 248         }.Authorizer()
 249         if err != nil {
 250                 return err
 251         }
 252
 253         vmClient.Authorizer = authorizer
 254         netClient.Authorizer = authorizer
 255         storageAcctClient.Authorizer = authorizer
 256
 257         az.vmClient = &virtualMachinesClientImpl{vmClient}
 258         az.netClient = &interfacesClientImpl{netClient}
 259
 260         result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
 261         if err != nil {
 262                 az.logger.WithError(err).Warn("Couldn't get account keys")
 263                 return err
 264         }
 265
 266         key1 := *(*result.Keys)[0].Value
 267         client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv)
 268         if err != nil {
 269                 az.logger.WithError(err).Warn("Couldn't make client")
 270                 return err
 271         }
 272
 273         blobsvc := client.GetBlobService()
 274         az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer)
 275
 276         az.dispatcherID = dispatcherID
 277         az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
 278
 279         az.ctx, az.stopFunc = context.WithCancel(context.Background())
 280         go func() {
 281                 az.stopWg.Add(1)
 282                 defer az.stopWg.Done()
 283
 284                 tk := time.NewTicker(5 * time.Minute)
 285                 for {
 286                         select {
 287                         case <-az.ctx.Done():
 288                                 tk.Stop()
 289                                 return
 290                         case <-tk.C:
 291                                 az.manageBlobs()
 292                         }
 293                 }
 294         }()
 295
 296         az.deleteNIC = make(chan string)
 297         az.deleteBlob = make(chan storage.Blob)
 298
 299         for i := 0; i < 4; i++ {
 300                 go func() {
 301                         for {
 302                                 nicname, ok := <-az.deleteNIC
 303                                 if !ok {
 304                                         return
 305                                 }
 306                                 _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, nicname)
 307                                 if delerr != nil {
 308                                         az.logger.WithError(delerr).Warnf("Error deleting %v", nicname)
 309                                 } else {
 310                                         az.logger.Printf("Deleted NIC %v", nicname)
 311                                 }
 312                         }
 313                 }()
 314                 go func() {
 315                         for {
 316                                 blob, ok := <-az.deleteBlob
 317                                 if !ok {
 318                                         return
 319                                 }
 320                                 err := blob.Delete(nil)
 321                                 if err != nil {
 322                                         az.logger.WithError(err).Warnf("Error deleting %v", blob.Name)
 323                                 } else {
 324                                         az.logger.Printf("Deleted blob %v", blob.Name)
 325                                 }
 326                         }
 327                 }()
 328         }
 329
 330         return nil
 331 }
 332
 333 func (az *azureInstanceSet) Create(
 334         instanceType arvados.InstanceType,
 335         imageID cloud.ImageID,
 336         newTags cloud.InstanceTags,
 337         initCommand cloud.InitCommand,
 338         publicKey ssh.PublicKey) (cloud.Instance, error) {
 339
 340         az.stopWg.Add(1)
 341         defer az.stopWg.Done()
 342
 343         name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
 344         if err != nil {
 345                 return nil, err
 346         }
 347
 348         name = az.namePrefix + name
 349
 350         timestamp := time.Now().Format(time.RFC3339Nano)
 351
 352         tags := make(map[string]*string)
 353         tags["created-at"] = &timestamp
 354         for k, v := range newTags {
 355                 newstr := v
 356                 tags["dispatch-"+k] = &newstr
 357         }
 358
 359         nicParameters := network.Interface{
 360                 Location: &az.azconfig.Location,
 361                 Tags:     tags,
 362                 InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
 363                         IPConfigurations: &[]network.InterfaceIPConfiguration{
 364                                 network.InterfaceIPConfiguration{
 365                                         Name: to.StringPtr("ip1"),
 366                                         InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
 367                                                 Subnet: &network.Subnet{
 368                                                         ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+
 369                                                                 "/Microsoft.Network/virtualnetworks/%s/subnets/%s",
 370                                                                 az.azconfig.SubscriptionID,
 371                                                                 az.azconfig.ResourceGroup,
 372                                                                 az.azconfig.Network,
 373                                                                 az.azconfig.Subnet)),
 374                                                 },
 375                                                 PrivateIPAllocationMethod: network.Dynamic,
 376                                         },
 377                                 },
 378                         },
 379                 },
 380         }
 381         nic, err := az.netClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
 382         if err != nil {
 383                 return nil, wrapAzureError(err)
 384         }
 385
 386         blobname := fmt.Sprintf("%s-os.vhd", name)
 387         instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s",
 388                 az.azconfig.StorageAccount,
 389                 az.azureEnv.StorageEndpointSuffix,
 390                 az.azconfig.BlobContainer,
 391                 blobname)
 392
 393         customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))
 394
 395         vmParameters := compute.VirtualMachine{
 396                 Location: &az.azconfig.Location,
 397                 Tags:     tags,
 398                 VirtualMachineProperties: &compute.VirtualMachineProperties{
 399                         HardwareProfile: &compute.HardwareProfile{
 400                                 VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
 401                         },
 402                         StorageProfile: &compute.StorageProfile{
 403                                 OsDisk: &compute.OSDisk{
 404                                         OsType:       compute.Linux,
 405                                         Name:         to.StringPtr(name + "-os"),
 406                                         CreateOption: compute.FromImage,
 407                                         Image: &compute.VirtualHardDisk{
 408                                                 URI: to.StringPtr(string(imageID)),
 409                                         },
 410                                         Vhd: &compute.VirtualHardDisk{
 411                                                 URI: &instanceVhd,
 412                                         },
 413                                 },
 414                         },
 415                         NetworkProfile: &compute.NetworkProfile{
 416                                 NetworkInterfaces: &[]compute.NetworkInterfaceReference{
 417                                         compute.NetworkInterfaceReference{
 418                                                 ID: nic.ID,
 419                                                 NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
 420                                                         Primary: to.BoolPtr(true),
 421                                                 },
 422                                         },
 423                                 },
 424                         },
 425                         OsProfile: &compute.OSProfile{
 426                                 ComputerName:  &name,
 427                                 AdminUsername: to.StringPtr(az.azconfig.AdminUsername),
 428                                 LinuxConfiguration: &compute.LinuxConfiguration{
 429                                         DisablePasswordAuthentication: to.BoolPtr(true),
 430                                         SSH: &compute.SSHConfiguration{
 431                                                 PublicKeys: &[]compute.SSHPublicKey{
 432                                                         {
 433                                                                 Path:    to.StringPtr("/home/" + az.azconfig.AdminUsername + "/.ssh/authorized_keys"),
 434                                                                 KeyData: to.StringPtr(string(ssh.MarshalAuthorizedKey(publicKey))),
 435                                                         },
 436                                                 },
 437                                         },
 438                                 },
 439                                 CustomData: &customData,
 440                         },
 441                 },
 442         }
 443
 444         vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters)
 445         if err != nil {
 446                 _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil)
 447                 if delerr != nil {
 448                         az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create")
 449                 }
 450
 451                 _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name)
 452                 if delerr != nil {
 453                         az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create")
 454                 }
 455
 456                 return nil, wrapAzureError(err)
 457         }
 458
 459         return &azureInstance{
 460                 provider: az,
 461                 nic:      nic,
 462                 vm:       vm,
 463         }, nil
 464 }
 465
 466 func (az *azureInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
 467         az.stopWg.Add(1)
 468         defer az.stopWg.Done()
 469
 470         interfaces, err := az.manageNics()
 471         if err != nil {
 472                 return nil, err
 473         }
 474
 475         result, err := az.vmClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 476         if err != nil {
 477                 return nil, wrapAzureError(err)
 478         }
 479
 480         instances := make([]cloud.Instance, 0)
 481
 482         for ; result.NotDone(); err = result.Next() {
 483                 if err != nil {
 484                         return nil, wrapAzureError(err)
 485                 }
 486                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 487                         instances = append(instances, &azureInstance{
 488                                 provider: az,
 489                                 vm:       result.Value(),
 490                                 nic:      interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
 491                 }
 492         }
 493         return instances, nil
 494 }
 495
 496 // ManageNics returns a list of Azure network interface resources.
 497 // Also performs garbage collection of NICs which have "namePrefix", are
 498 // not associated with a virtual machine and have a "create-at" time
 499 // more than DeleteDanglingResourcesAfter (to prevent racing and
 500 // deleting newly created NICs) in the past are deleted.
 501 func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) {
 502         az.stopWg.Add(1)
 503         defer az.stopWg.Done()
 504
 505         result, err := az.netClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 506         if err != nil {
 507                 return nil, wrapAzureError(err)
 508         }
 509
 510         interfaces := make(map[string]network.Interface)
 511
 512         timestamp := time.Now()
 513         for ; result.NotDone(); err = result.Next() {
 514                 if err != nil {
 515                         az.logger.WithError(err).Warnf("Error listing nics")
 516                         return interfaces, nil
 517                 }
 518                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 519                         if result.Value().VirtualMachine != nil {
 520                                 interfaces[*result.Value().ID] = result.Value()
 521                         } else {
 522                                 if result.Value().Tags["created-at"] != nil {
 523                                         createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
 524                                         if err == nil {
 525                                                 if timestamp.Sub(createdAt) > az.azconfig.DeleteDanglingResourcesAfter.Duration() {
 526                                                         az.logger.Printf("Will delete %v because it is older than %s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
 527                                                         az.deleteNIC <- *result.Value().Name
 528                                                 }
 529                                         }
 530                                 }
 531                         }
 532                 }
 533         }
 534         return interfaces, nil
 535 }
 536
 537 // ManageBlobs garbage collects blobs (VM disk images) in the
 538 // configured storage account container.  It will delete blobs which
 539 // have "namePrefix", are "available" (which means they are not
 540 // leased to a VM) and haven't been modified for
 541 // DeleteDanglingResourcesAfter seconds.
 542 func (az *azureInstanceSet) manageBlobs() {
 543
 544         page := storage.ListBlobsParameters{Prefix: az.namePrefix}
 545         timestamp := time.Now()
 546
 547         for {
 548                 response, err := az.blobcont.ListBlobs(page)
 549                 if err != nil {
 550                         az.logger.WithError(err).Warn("Error listing blobs")
 551                         return
 552                 }
 553                 for _, b := range response.Blobs {
 554                         age := timestamp.Sub(time.Time(b.Properties.LastModified))
 555                         if b.Properties.BlobType == storage.BlobTypePage &&
 556                                 b.Properties.LeaseState == "available" &&
 557                                 b.Properties.LeaseStatus == "unlocked" &&
 558                                 age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
 559
 560                                 az.logger.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
 561                                 az.deleteBlob <- b
 562                         }
 563                 }
 564                 if response.NextMarker != "" {
 565                         page.Marker = response.NextMarker
 566                 } else {
 567                         break
 568                 }
 569         }
 570 }
 571
 572 func (az *azureInstanceSet) Stop() {
 573         az.stopFunc()
 574         az.stopWg.Wait()
 575         close(az.deleteNIC)
 576         close(az.deleteBlob)
 577 }
 578
 579 type azureInstance struct {
 580         provider *azureInstanceSet
 581         nic      network.Interface
 582         vm       compute.VirtualMachine
 583 }
 584
 585 func (ai *azureInstance) ID() cloud.InstanceID {
 586         return cloud.InstanceID(*ai.vm.ID)
 587 }
 588
 589 func (ai *azureInstance) String() string {
 590         return *ai.vm.Name
 591 }
 592
 593 func (ai *azureInstance) ProviderType() string {
 594         return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
 595 }
 596
 597 func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error {
 598         ai.provider.stopWg.Add(1)
 599         defer ai.provider.stopWg.Done()
 600
 601         tags := make(map[string]*string)
 602
 603         for k, v := range ai.vm.Tags {
 604                 if !strings.HasPrefix(k, "dispatch-") {
 605                         tags[k] = v
 606                 }
 607         }
 608         for k, v := range newTags {
 609                 newstr := v
 610                 tags["dispatch-"+k] = &newstr
 611         }
 612
 613         vmParameters := compute.VirtualMachine{
 614                 Location: &ai.provider.azconfig.Location,
 615                 Tags:     tags,
 616         }
 617         vm, err := ai.provider.vmClient.createOrUpdate(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
 618         if err != nil {
 619                 return wrapAzureError(err)
 620         }
 621         ai.vm = vm
 622
 623         return nil
 624 }
 625
 626 func (ai *azureInstance) Tags() cloud.InstanceTags {
 627         tags := make(map[string]string)
 628
 629         for k, v := range ai.vm.Tags {
 630                 if strings.HasPrefix(k, "dispatch-") {
 631                         tags[k[9:]] = *v
 632                 }
 633         }
 634
 635         return tags
 636 }
 637
 638 func (ai *azureInstance) Destroy() error {
 639         ai.provider.stopWg.Add(1)
 640         defer ai.provider.stopWg.Done()
 641
 642         _, err := ai.provider.vmClient.delete(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
 643         return wrapAzureError(err)
 644 }
 645
 646 func (ai *azureInstance) Address() string {
 647         if ai.nic.IPConfigurations != nil &&
 648                 len(*ai.nic.IPConfigurations) > 0 &&
 649                 (*ai.nic.IPConfigurations)[0].PrivateIPAddress != nil {
 650
 651                 return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
 652         }
 653         return ""
 654 }
 655
 656 func (ai *azureInstance) RemoteUser() string {
 657         return ai.provider.azconfig.AdminUsername
 658 }
 659
 660 func (ai *azureInstance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
 661         return cloud.ErrNotImplemented
 662 }