lib/cloud/azure/azure.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package azure
   6
   7 import (
   8         "context"
   9         "encoding/base64"
  10         "encoding/json"
  11         "fmt"
  12         "net/http"
  13         "regexp"
  14         "strconv"
  15         "strings"
  16         "sync"
  17         "time"
  18
  19         "git.curoverse.com/arvados.git/lib/cloud"
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
  22         "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
  23         storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
  24         "github.com/Azure/azure-sdk-for-go/storage"
  25         "github.com/Azure/go-autorest/autorest"
  26         "github.com/Azure/go-autorest/autorest/azure"
  27         "github.com/Azure/go-autorest/autorest/azure/auth"
  28         "github.com/Azure/go-autorest/autorest/to"
  29         "github.com/jmcvetta/randutil"
  30         "github.com/sirupsen/logrus"
  31         "golang.org/x/crypto/ssh"
  32 )
  33
  34 // Driver is the azure implementation of the cloud.Driver interface.
  35 var Driver = cloud.DriverFunc(newAzureInstanceSet)
  36
  37 type azureInstanceSetConfig struct {
  38         SubscriptionID               string
  39         ClientID                     string
  40         ClientSecret                 string
  41         TenantID                     string
  42         CloudEnvironment             string
  43         ResourceGroup                string
  44         Location                     string
  45         Network                      string
  46         Subnet                       string
  47         StorageAccount               string
  48         BlobContainer                string
  49         DeleteDanglingResourcesAfter arvados.Duration
  50         AdminUsername                string
  51 }
  52
  53 const tagKeyInstanceSecret = "InstanceSecret"
  54
  55 type containerWrapper interface {
  56         GetBlobReference(name string) *storage.Blob
  57         ListBlobs(params storage.ListBlobsParameters) (storage.BlobListResponse, error)
  58 }
  59
  60 type virtualMachinesClientWrapper interface {
  61         createOrUpdate(ctx context.Context,
  62                 resourceGroupName string,
  63                 VMName string,
  64                 parameters compute.VirtualMachine) (result compute.VirtualMachine, err error)
  65         delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error)
  66         listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error)
  67 }
  68
  69 type virtualMachinesClientImpl struct {
  70         inner compute.VirtualMachinesClient
  71 }
  72
  73 func (cl *virtualMachinesClientImpl) createOrUpdate(ctx context.Context,
  74         resourceGroupName string,
  75         VMName string,
  76         parameters compute.VirtualMachine) (result compute.VirtualMachine, err error) {
  77
  78         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, VMName, parameters)
  79         if err != nil {
  80                 return compute.VirtualMachine{}, wrapAzureError(err)
  81         }
  82         future.WaitForCompletionRef(ctx, cl.inner.Client)
  83         r, err := future.Result(cl.inner)
  84         return r, wrapAzureError(err)
  85 }
  86
  87 func (cl *virtualMachinesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
  88         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
  89         if err != nil {
  90                 return nil, wrapAzureError(err)
  91         }
  92         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
  93         return future.Response(), wrapAzureError(err)
  94 }
  95
  96 func (cl *virtualMachinesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result compute.VirtualMachineListResultIterator, err error) {
  97         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
  98         return r, wrapAzureError(err)
  99 }
 100
 101 type interfacesClientWrapper interface {
 102         createOrUpdate(ctx context.Context,
 103                 resourceGroupName string,
 104                 networkInterfaceName string,
 105                 parameters network.Interface) (result network.Interface, err error)
 106         delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (result *http.Response, err error)
 107         listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error)
 108 }
 109
 110 type interfacesClientImpl struct {
 111         inner network.InterfacesClient
 112 }
 113
 114 func (cl *interfacesClientImpl) delete(ctx context.Context, resourceGroupName string, VMName string) (result *http.Response, err error) {
 115         future, err := cl.inner.Delete(ctx, resourceGroupName, VMName)
 116         if err != nil {
 117                 return nil, wrapAzureError(err)
 118         }
 119         err = future.WaitForCompletionRef(ctx, cl.inner.Client)
 120         return future.Response(), wrapAzureError(err)
 121 }
 122
 123 func (cl *interfacesClientImpl) createOrUpdate(ctx context.Context,
 124         resourceGroupName string,
 125         networkInterfaceName string,
 126         parameters network.Interface) (result network.Interface, err error) {
 127
 128         future, err := cl.inner.CreateOrUpdate(ctx, resourceGroupName, networkInterfaceName, parameters)
 129         if err != nil {
 130                 return network.Interface{}, wrapAzureError(err)
 131         }
 132         future.WaitForCompletionRef(ctx, cl.inner.Client)
 133         r, err := future.Result(cl.inner)
 134         return r, wrapAzureError(err)
 135 }
 136
 137 func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupName string) (result network.InterfaceListResultIterator, err error) {
 138         r, err := cl.inner.ListComplete(ctx, resourceGroupName)
 139         return r, wrapAzureError(err)
 140 }
 141
 142 var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`)
 143
 144 type azureRateLimitError struct {
 145         azure.RequestError
 146         firstRetry time.Time
 147 }
 148
 149 func (ar *azureRateLimitError) EarliestRetry() time.Time {
 150         return ar.firstRetry
 151 }
 152
 153 type azureQuotaError struct {
 154         azure.RequestError
 155 }
 156
 157 func (ar *azureQuotaError) IsQuotaError() bool {
 158         return true
 159 }
 160
 161 func wrapAzureError(err error) error {
 162         de, ok := err.(autorest.DetailedError)
 163         if !ok {
 164                 return err
 165         }
 166         rq, ok := de.Original.(*azure.RequestError)
 167         if !ok {
 168                 return err
 169         }
 170         if rq.Response == nil {
 171                 return err
 172         }
 173         if rq.Response.StatusCode == 429 || len(rq.Response.Header["Retry-After"]) >= 1 {
 174                 // API throttling
 175                 ra := rq.Response.Header["Retry-After"][0]
 176                 earliestRetry, parseErr := http.ParseTime(ra)
 177                 if parseErr != nil {
 178                         // Could not parse as a timestamp, must be number of seconds
 179                         dur, parseErr := strconv.ParseInt(ra, 10, 64)
 180                         if parseErr == nil {
 181                                 earliestRetry = time.Now().Add(time.Duration(dur) * time.Second)
 182                         } else {
 183                                 // Couldn't make sense of retry-after,
 184                                 // so set retry to 20 seconds
 185                                 earliestRetry = time.Now().Add(20 * time.Second)
 186                         }
 187                 }
 188                 return &azureRateLimitError{*rq, earliestRetry}
 189         }
 190         if rq.ServiceError == nil {
 191                 return err
 192         }
 193         if quotaRe.FindString(rq.ServiceError.Code) != "" || quotaRe.FindString(rq.ServiceError.Message) != "" {
 194                 return &azureQuotaError{*rq}
 195         }
 196         return err
 197 }
 198
 199 type azureInstanceSet struct {
 200         azconfig     azureInstanceSetConfig
 201         vmClient     virtualMachinesClientWrapper
 202         netClient    interfacesClientWrapper
 203         blobcont     containerWrapper
 204         azureEnv     azure.Environment
 205         interfaces   map[string]network.Interface
 206         dispatcherID string
 207         namePrefix   string
 208         ctx          context.Context
 209         stopFunc     context.CancelFunc
 210         stopWg       sync.WaitGroup
 211         deleteNIC    chan string
 212         deleteBlob   chan storage.Blob
 213         logger       logrus.FieldLogger
 214 }
 215
 216 func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) {
 217         azcfg := azureInstanceSetConfig{}
 218         err = json.Unmarshal(config, &azcfg)
 219         if err != nil {
 220                 return nil, err
 221         }
 222
 223         az := azureInstanceSet{logger: logger}
 224         az.ctx, az.stopFunc = context.WithCancel(context.Background())
 225         err = az.setup(azcfg, string(dispatcherID))
 226         if err != nil {
 227                 az.stopFunc()
 228                 return nil, err
 229         }
 230         return &az, nil
 231 }
 232
 233 func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID string) (err error) {
 234         az.azconfig = azcfg
 235         vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
 236         netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
 237         storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID)
 238
 239         az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment)
 240         if err != nil {
 241                 return err
 242         }
 243
 244         authorizer, err := auth.ClientCredentialsConfig{
 245                 ClientID:     az.azconfig.ClientID,
 246                 ClientSecret: az.azconfig.ClientSecret,
 247                 TenantID:     az.azconfig.TenantID,
 248                 Resource:     az.azureEnv.ResourceManagerEndpoint,
 249                 AADEndpoint:  az.azureEnv.ActiveDirectoryEndpoint,
 250         }.Authorizer()
 251         if err != nil {
 252                 return err
 253         }
 254
 255         vmClient.Authorizer = authorizer
 256         netClient.Authorizer = authorizer
 257         storageAcctClient.Authorizer = authorizer
 258
 259         az.vmClient = &virtualMachinesClientImpl{vmClient}
 260         az.netClient = &interfacesClientImpl{netClient}
 261
 262         result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
 263         if err != nil {
 264                 az.logger.WithError(err).Warn("Couldn't get account keys")
 265                 return err
 266         }
 267
 268         key1 := *(*result.Keys)[0].Value
 269         client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv)
 270         if err != nil {
 271                 az.logger.WithError(err).Warn("Couldn't make client")
 272                 return err
 273         }
 274
 275         blobsvc := client.GetBlobService()
 276         az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer)
 277
 278         az.dispatcherID = dispatcherID
 279         az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID)
 280
 281         go func() {
 282                 az.stopWg.Add(1)
 283                 defer az.stopWg.Done()
 284
 285                 tk := time.NewTicker(5 * time.Minute)
 286                 for {
 287                         select {
 288                         case <-az.ctx.Done():
 289                                 tk.Stop()
 290                                 return
 291                         case <-tk.C:
 292                                 az.manageBlobs()
 293                         }
 294                 }
 295         }()
 296
 297         az.deleteNIC = make(chan string)
 298         az.deleteBlob = make(chan storage.Blob)
 299
 300         for i := 0; i < 4; i++ {
 301                 go func() {
 302                         for {
 303                                 nicname, ok := <-az.deleteNIC
 304                                 if !ok {
 305                                         return
 306                                 }
 307                                 _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, nicname)
 308                                 if delerr != nil {
 309                                         az.logger.WithError(delerr).Warnf("Error deleting %v", nicname)
 310                                 } else {
 311                                         az.logger.Printf("Deleted NIC %v", nicname)
 312                                 }
 313                         }
 314                 }()
 315                 go func() {
 316                         for {
 317                                 blob, ok := <-az.deleteBlob
 318                                 if !ok {
 319                                         return
 320                                 }
 321                                 err := blob.Delete(nil)
 322                                 if err != nil {
 323                                         az.logger.WithError(err).Warnf("Error deleting %v", blob.Name)
 324                                 } else {
 325                                         az.logger.Printf("Deleted blob %v", blob.Name)
 326                                 }
 327                         }
 328                 }()
 329         }
 330
 331         return nil
 332 }
 333
 334 func (az *azureInstanceSet) Create(
 335         instanceType arvados.InstanceType,
 336         imageID cloud.ImageID,
 337         newTags cloud.InstanceTags,
 338         initCommand cloud.InitCommand,
 339         publicKey ssh.PublicKey) (cloud.Instance, error) {
 340
 341         az.stopWg.Add(1)
 342         defer az.stopWg.Done()
 343
 344         if instanceType.AddedScratch > 0 {
 345                 return nil, fmt.Errorf("cannot create instance type %q: driver does not implement non-zero AddedScratch (%d)", instanceType.Name, instanceType.AddedScratch)
 346         }
 347
 348         name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
 349         if err != nil {
 350                 return nil, err
 351         }
 352
 353         name = az.namePrefix + name
 354
 355         timestamp := time.Now().Format(time.RFC3339Nano)
 356
 357         tags := make(map[string]*string)
 358         tags["created-at"] = &timestamp
 359         for k, v := range newTags {
 360                 newstr := v
 361                 tags["dispatch-"+k] = &newstr
 362         }
 363
 364         nicParameters := network.Interface{
 365                 Location: &az.azconfig.Location,
 366                 Tags:     tags,
 367                 InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
 368                         IPConfigurations: &[]network.InterfaceIPConfiguration{
 369                                 network.InterfaceIPConfiguration{
 370                                         Name: to.StringPtr("ip1"),
 371                                         InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
 372                                                 Subnet: &network.Subnet{
 373                                                         ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+
 374                                                                 "/Microsoft.Network/virtualnetworks/%s/subnets/%s",
 375                                                                 az.azconfig.SubscriptionID,
 376                                                                 az.azconfig.ResourceGroup,
 377                                                                 az.azconfig.Network,
 378                                                                 az.azconfig.Subnet)),
 379                                                 },
 380                                                 PrivateIPAllocationMethod: network.Dynamic,
 381                                         },
 382                                 },
 383                         },
 384                 },
 385         }
 386         nic, err := az.netClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name+"-nic", nicParameters)
 387         if err != nil {
 388                 return nil, wrapAzureError(err)
 389         }
 390
 391         blobname := fmt.Sprintf("%s-os.vhd", name)
 392         instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s",
 393                 az.azconfig.StorageAccount,
 394                 az.azureEnv.StorageEndpointSuffix,
 395                 az.azconfig.BlobContainer,
 396                 blobname)
 397
 398         customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))
 399
 400         vmParameters := compute.VirtualMachine{
 401                 Location: &az.azconfig.Location,
 402                 Tags:     tags,
 403                 VirtualMachineProperties: &compute.VirtualMachineProperties{
 404                         HardwareProfile: &compute.HardwareProfile{
 405                                 VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType),
 406                         },
 407                         StorageProfile: &compute.StorageProfile{
 408                                 OsDisk: &compute.OSDisk{
 409                                         OsType:       compute.Linux,
 410                                         Name:         to.StringPtr(name + "-os"),
 411                                         CreateOption: compute.FromImage,
 412                                         Image: &compute.VirtualHardDisk{
 413                                                 URI: to.StringPtr(string(imageID)),
 414                                         },
 415                                         Vhd: &compute.VirtualHardDisk{
 416                                                 URI: &instanceVhd,
 417                                         },
 418                                 },
 419                         },
 420                         NetworkProfile: &compute.NetworkProfile{
 421                                 NetworkInterfaces: &[]compute.NetworkInterfaceReference{
 422                                         compute.NetworkInterfaceReference{
 423                                                 ID: nic.ID,
 424                                                 NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{
 425                                                         Primary: to.BoolPtr(true),
 426                                                 },
 427                                         },
 428                                 },
 429                         },
 430                         OsProfile: &compute.OSProfile{
 431                                 ComputerName:  &name,
 432                                 AdminUsername: to.StringPtr(az.azconfig.AdminUsername),
 433                                 LinuxConfiguration: &compute.LinuxConfiguration{
 434                                         DisablePasswordAuthentication: to.BoolPtr(true),
 435                                         SSH: &compute.SSHConfiguration{
 436                                                 PublicKeys: &[]compute.SSHPublicKey{
 437                                                         {
 438                                                                 Path:    to.StringPtr("/home/" + az.azconfig.AdminUsername + "/.ssh/authorized_keys"),
 439                                                                 KeyData: to.StringPtr(string(ssh.MarshalAuthorizedKey(publicKey))),
 440                                                         },
 441                                                 },
 442                                         },
 443                                 },
 444                                 CustomData: &customData,
 445                         },
 446                 },
 447         }
 448
 449         vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters)
 450         if err != nil {
 451                 _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil)
 452                 if delerr != nil {
 453                         az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create")
 454                 }
 455
 456                 _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name)
 457                 if delerr != nil {
 458                         az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create")
 459                 }
 460
 461                 return nil, wrapAzureError(err)
 462         }
 463
 464         return &azureInstance{
 465                 provider: az,
 466                 nic:      nic,
 467                 vm:       vm,
 468         }, nil
 469 }
 470
 471 func (az *azureInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, error) {
 472         az.stopWg.Add(1)
 473         defer az.stopWg.Done()
 474
 475         interfaces, err := az.manageNics()
 476         if err != nil {
 477                 return nil, err
 478         }
 479
 480         result, err := az.vmClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 481         if err != nil {
 482                 return nil, wrapAzureError(err)
 483         }
 484
 485         instances := make([]cloud.Instance, 0)
 486
 487         for ; result.NotDone(); err = result.Next() {
 488                 if err != nil {
 489                         return nil, wrapAzureError(err)
 490                 }
 491                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 492                         instances = append(instances, &azureInstance{
 493                                 provider: az,
 494                                 vm:       result.Value(),
 495                                 nic:      interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]})
 496                 }
 497         }
 498         return instances, nil
 499 }
 500
 501 // ManageNics returns a list of Azure network interface resources.
 502 // Also performs garbage collection of NICs which have "namePrefix", are
 503 // not associated with a virtual machine and have a "create-at" time
 504 // more than DeleteDanglingResourcesAfter (to prevent racing and
 505 // deleting newly created NICs) in the past are deleted.
 506 func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) {
 507         az.stopWg.Add(1)
 508         defer az.stopWg.Done()
 509
 510         result, err := az.netClient.listComplete(az.ctx, az.azconfig.ResourceGroup)
 511         if err != nil {
 512                 return nil, wrapAzureError(err)
 513         }
 514
 515         interfaces := make(map[string]network.Interface)
 516
 517         timestamp := time.Now()
 518         for ; result.NotDone(); err = result.Next() {
 519                 if err != nil {
 520                         az.logger.WithError(err).Warnf("Error listing nics")
 521                         return interfaces, nil
 522                 }
 523                 if strings.HasPrefix(*result.Value().Name, az.namePrefix) {
 524                         if result.Value().VirtualMachine != nil {
 525                                 interfaces[*result.Value().ID] = result.Value()
 526                         } else {
 527                                 if result.Value().Tags["created-at"] != nil {
 528                                         createdAt, err := time.Parse(time.RFC3339Nano, *result.Value().Tags["created-at"])
 529                                         if err == nil {
 530                                                 if timestamp.Sub(createdAt) > az.azconfig.DeleteDanglingResourcesAfter.Duration() {
 531                                                         az.logger.Printf("Will delete %v because it is older than %s", *result.Value().Name, az.azconfig.DeleteDanglingResourcesAfter)
 532                                                         az.deleteNIC <- *result.Value().Name
 533                                                 }
 534                                         }
 535                                 }
 536                         }
 537                 }
 538         }
 539         return interfaces, nil
 540 }
 541
 542 // ManageBlobs garbage collects blobs (VM disk images) in the
 543 // configured storage account container.  It will delete blobs which
 544 // have "namePrefix", are "available" (which means they are not
 545 // leased to a VM) and haven't been modified for
 546 // DeleteDanglingResourcesAfter seconds.
 547 func (az *azureInstanceSet) manageBlobs() {
 548
 549         page := storage.ListBlobsParameters{Prefix: az.namePrefix}
 550         timestamp := time.Now()
 551
 552         for {
 553                 response, err := az.blobcont.ListBlobs(page)
 554                 if err != nil {
 555                         az.logger.WithError(err).Warn("Error listing blobs")
 556                         return
 557                 }
 558                 for _, b := range response.Blobs {
 559                         age := timestamp.Sub(time.Time(b.Properties.LastModified))
 560                         if b.Properties.BlobType == storage.BlobTypePage &&
 561                                 b.Properties.LeaseState == "available" &&
 562                                 b.Properties.LeaseStatus == "unlocked" &&
 563                                 age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() {
 564
 565                                 az.logger.Printf("Blob %v is unlocked and not modified for %v seconds, will delete", b.Name, age.Seconds())
 566                                 az.deleteBlob <- b
 567                         }
 568                 }
 569                 if response.NextMarker != "" {
 570                         page.Marker = response.NextMarker
 571                 } else {
 572                         break
 573                 }
 574         }
 575 }
 576
 577 func (az *azureInstanceSet) Stop() {
 578         az.stopFunc()
 579         az.stopWg.Wait()
 580         close(az.deleteNIC)
 581         close(az.deleteBlob)
 582 }
 583
 584 type azureInstance struct {
 585         provider *azureInstanceSet
 586         nic      network.Interface
 587         vm       compute.VirtualMachine
 588 }
 589
 590 func (ai *azureInstance) ID() cloud.InstanceID {
 591         return cloud.InstanceID(*ai.vm.ID)
 592 }
 593
 594 func (ai *azureInstance) String() string {
 595         return *ai.vm.Name
 596 }
 597
 598 func (ai *azureInstance) ProviderType() string {
 599         return string(ai.vm.VirtualMachineProperties.HardwareProfile.VMSize)
 600 }
 601
 602 func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error {
 603         ai.provider.stopWg.Add(1)
 604         defer ai.provider.stopWg.Done()
 605
 606         tags := make(map[string]*string)
 607
 608         for k, v := range ai.vm.Tags {
 609                 if !strings.HasPrefix(k, "dispatch-") {
 610                         tags[k] = v
 611                 }
 612         }
 613         for k, v := range newTags {
 614                 newstr := v
 615                 tags["dispatch-"+k] = &newstr
 616         }
 617
 618         vmParameters := compute.VirtualMachine{
 619                 Location: &ai.provider.azconfig.Location,
 620                 Tags:     tags,
 621         }
 622         vm, err := ai.provider.vmClient.createOrUpdate(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name, vmParameters)
 623         if err != nil {
 624                 return wrapAzureError(err)
 625         }
 626         ai.vm = vm
 627
 628         return nil
 629 }
 630
 631 func (ai *azureInstance) Tags() cloud.InstanceTags {
 632         tags := make(map[string]string)
 633
 634         for k, v := range ai.vm.Tags {
 635                 if strings.HasPrefix(k, "dispatch-") {
 636                         tags[k[9:]] = *v
 637                 }
 638         }
 639
 640         return tags
 641 }
 642
 643 func (ai *azureInstance) Destroy() error {
 644         ai.provider.stopWg.Add(1)
 645         defer ai.provider.stopWg.Done()
 646
 647         _, err := ai.provider.vmClient.delete(ai.provider.ctx, ai.provider.azconfig.ResourceGroup, *ai.vm.Name)
 648         return wrapAzureError(err)
 649 }
 650
 651 func (ai *azureInstance) Address() string {
 652         if iprops := ai.nic.InterfacePropertiesFormat; iprops == nil {
 653                 return ""
 654         } else if ipconfs := iprops.IPConfigurations; ipconfs == nil || len(*ipconfs) == 0 {
 655                 return ""
 656         } else if ipconfprops := (*ipconfs)[0].InterfaceIPConfigurationPropertiesFormat; ipconfprops == nil {
 657                 return ""
 658         } else if addr := ipconfprops.PrivateIPAddress; addr == nil {
 659                 return ""
 660         } else {
 661                 return *addr
 662         }
 663 }
 664
 665 func (ai *azureInstance) RemoteUser() string {
 666         return ai.provider.azconfig.AdminUsername
 667 }
 668
 669 func (ai *azureInstance) VerifyHostKey(ssh.PublicKey, *ssh.Client) error {
 670         return cloud.ErrNotImplemented
 671 }