X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/fb9f29b33a380616211a56973045dac1254977ee..c76e5bc93b066d93a668c4e00b52aa550028e1f4:/lib/cloud/azure/azure.go diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go index d19e4bef23..d2dbde96d7 100644 --- a/lib/cloud/azure/azure.go +++ b/lib/cloud/azure/azure.go @@ -8,6 +8,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "net/http" "regexp" @@ -16,9 +17,9 @@ import ( "sync" "time" - "git.curoverse.com/arvados.git/lib/cloud" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" + "git.arvados.org/arvados.git/lib/cloud" + "git.arvados.org/arvados.git/sdk/go/arvados" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage" "github.com/Azure/azure-sdk-for-go/storage" @@ -41,8 +42,10 @@ type azureInstanceSetConfig struct { TenantID string CloudEnvironment string ResourceGroup string + ImageResourceGroup string Location string Network string + NetworkResourceGroup string Subnet string StorageAccount string BlobContainer string @@ -50,8 +53,6 @@ type azureInstanceSetConfig struct { AdminUsername string } -const tagKeyInstanceSecret = "InstanceSecret" - type containerWrapper interface { GetBlobReference(name string) *storage.Blob ListBlobs(params storage.ListBlobsParameters) (storage.BlobListResponse, error) @@ -139,6 +140,25 @@ func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupN return r, wrapAzureError(err) } +type disksClientWrapper interface { + listByResourceGroup(ctx context.Context, resourceGroupName string) (result compute.DiskListPage, err error) + delete(ctx context.Context, resourceGroupName string, diskName string) (result compute.DisksDeleteFuture, err error) +} + +type disksClientImpl struct { + inner compute.DisksClient +} + +func (cl *disksClientImpl) listByResourceGroup(ctx context.Context, resourceGroupName string) (result compute.DiskListPage, err error) { + r, err := cl.inner.ListByResourceGroup(ctx, resourceGroupName) + return r, wrapAzureError(err) +} + +func (cl *disksClientImpl) delete(ctx context.Context, resourceGroupName string, diskName string) (result compute.DisksDeleteFuture, err error) { + r, err := cl.inner.Delete(ctx, resourceGroupName, diskName) + return r, wrapAzureError(err) +} + var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`) type azureRateLimitError struct { @@ -197,23 +217,26 @@ func wrapAzureError(err error) error { } type azureInstanceSet struct { - azconfig azureInstanceSetConfig - vmClient virtualMachinesClientWrapper - netClient interfacesClientWrapper - blobcont containerWrapper - azureEnv azure.Environment - interfaces map[string]network.Interface - dispatcherID string - namePrefix string - ctx context.Context - stopFunc context.CancelFunc - stopWg sync.WaitGroup - deleteNIC chan string - deleteBlob chan storage.Blob - logger logrus.FieldLogger -} - -func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) { + azconfig azureInstanceSetConfig + vmClient virtualMachinesClientWrapper + netClient interfacesClientWrapper + disksClient disksClientWrapper + imageResourceGroup string + blobcont containerWrapper + azureEnv azure.Environment + interfaces map[string]network.Interface + dispatcherID string + namePrefix string + ctx context.Context + stopFunc context.CancelFunc + stopWg sync.WaitGroup + deleteNIC chan string + deleteBlob chan storage.Blob + deleteDisk chan compute.Disk + logger logrus.FieldLogger +} + +func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) { azcfg := azureInstanceSetConfig{} err = json.Unmarshal(config, &azcfg) if err != nil { @@ -234,6 +257,7 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str az.azconfig = azcfg vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID) netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID) + disksClient := compute.NewDisksClient(az.azconfig.SubscriptionID) storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID) az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment) @@ -254,26 +278,38 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str vmClient.Authorizer = authorizer netClient.Authorizer = authorizer + disksClient.Authorizer = authorizer storageAcctClient.Authorizer = authorizer az.vmClient = &virtualMachinesClientImpl{vmClient} az.netClient = &interfacesClientImpl{netClient} + az.disksClient = &disksClientImpl{disksClient} - result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount) - if err != nil { - az.logger.WithError(err).Warn("Couldn't get account keys") - return err + az.imageResourceGroup = az.azconfig.ImageResourceGroup + if az.imageResourceGroup == "" { + az.imageResourceGroup = az.azconfig.ResourceGroup } - key1 := *(*result.Keys)[0].Value - client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv) - if err != nil { - az.logger.WithError(err).Warn("Couldn't make client") - return err - } + var client storage.Client + if az.azconfig.StorageAccount != "" && az.azconfig.BlobContainer != "" { + result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount) + if err != nil { + az.logger.WithError(err).Warn("Couldn't get account keys") + return err + } - blobsvc := client.GetBlobService() - az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer) + key1 := *(*result.Keys)[0].Value + client, err = storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv) + if err != nil { + az.logger.WithError(err).Warn("Couldn't make client") + return err + } + + blobsvc := client.GetBlobService() + az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer) + } else if az.azconfig.StorageAccount != "" || az.azconfig.BlobContainer != "" { + az.logger.Error("Invalid configuration: StorageAccount and BlobContainer must both be empty or both be set") + } az.dispatcherID = dispatcherID az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID) @@ -289,21 +325,21 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str tk.Stop() return case <-tk.C: - az.manageBlobs() + if az.blobcont != nil { + az.manageBlobs() + } + az.manageDisks() } } }() az.deleteNIC = make(chan string) az.deleteBlob = make(chan storage.Blob) + az.deleteDisk = make(chan compute.Disk) for i := 0; i < 4; i++ { go func() { - for { - nicname, ok := <-az.deleteNIC - if !ok { - return - } + for nicname := range az.deleteNIC { _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, nicname) if delerr != nil { az.logger.WithError(delerr).Warnf("Error deleting %v", nicname) @@ -313,11 +349,7 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str } }() go func() { - for { - blob, ok := <-az.deleteBlob - if !ok { - return - } + for blob := range az.deleteBlob { err := blob.Delete(nil) if err != nil { az.logger.WithError(err).Warnf("Error deleting %v", blob.Name) @@ -326,6 +358,16 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str } } }() + go func() { + for disk := range az.deleteDisk { + _, err := az.disksClient.delete(az.ctx, az.imageResourceGroup, *disk.Name) + if err != nil { + az.logger.WithError(err).Warnf("Error deleting disk %+v", *disk.Name) + } else { + az.logger.Printf("Deleted disk %v", *disk.Name) + } + } + }() } return nil @@ -341,6 +383,10 @@ func (az *azureInstanceSet) Create( az.stopWg.Add(1) defer az.stopWg.Done() + if instanceType.AddedScratch > 0 { + return nil, fmt.Errorf("cannot create instance type %q: driver does not implement non-zero AddedScratch (%d)", instanceType.Name, instanceType.AddedScratch) + } + name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789") if err != nil { return nil, err @@ -348,13 +394,15 @@ func (az *azureInstanceSet) Create( name = az.namePrefix + name - timestamp := time.Now().Format(time.RFC3339Nano) - - tags := make(map[string]*string) - tags["created-at"] = ×tamp + tags := map[string]*string{} for k, v := range newTags { - newstr := v - tags["dispatch-"+k] = &newstr + tags[k] = to.StringPtr(v) + } + tags["created-at"] = to.StringPtr(time.Now().Format(time.RFC3339Nano)) + + networkResourceGroup := az.azconfig.NetworkResourceGroup + if networkResourceGroup == "" { + networkResourceGroup = az.azconfig.ResourceGroup } nicParameters := network.Interface{ @@ -369,7 +417,7 @@ func (az *azureInstanceSet) Create( ID: to.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers"+ "/Microsoft.Network/virtualnetworks/%s/subnets/%s", az.azconfig.SubscriptionID, - az.azconfig.ResourceGroup, + networkResourceGroup, az.azconfig.Network, az.azconfig.Subnet)), }, @@ -384,14 +432,46 @@ func (az *azureInstanceSet) Create( return nil, wrapAzureError(err) } - blobname := fmt.Sprintf("%s-os.vhd", name) - instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s", - az.azconfig.StorageAccount, - az.azureEnv.StorageEndpointSuffix, - az.azconfig.BlobContainer, - blobname) - + var blobname string customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n")) + var storageProfile *compute.StorageProfile + + re := regexp.MustCompile(`^http(s?)://`) + if re.MatchString(string(imageID)) && az.blobcont != nil { + blobname = fmt.Sprintf("%s-os.vhd", name) + instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s", + az.azconfig.StorageAccount, + az.azureEnv.StorageEndpointSuffix, + az.azconfig.BlobContainer, + blobname) + az.logger.Warn("using deprecated unmanaged image, see https://doc.arvados.org/ to migrate to managed disks") + storageProfile = &compute.StorageProfile{ + OsDisk: &compute.OSDisk{ + OsType: compute.Linux, + Name: to.StringPtr(name + "-os"), + CreateOption: compute.DiskCreateOptionTypesFromImage, + Image: &compute.VirtualHardDisk{ + URI: to.StringPtr(string(imageID)), + }, + Vhd: &compute.VirtualHardDisk{ + URI: &instanceVhd, + }, + }, + } + } else if az.blobcont == nil { + storageProfile = &compute.StorageProfile{ + ImageReference: &compute.ImageReference{ + ID: to.StringPtr("/subscriptions/" + az.azconfig.SubscriptionID + "/resourceGroups/" + az.imageResourceGroup + "/providers/Microsoft.Compute/images/" + string(imageID)), + }, + OsDisk: &compute.OSDisk{ + OsType: compute.Linux, + Name: to.StringPtr(name + "-os"), + CreateOption: compute.DiskCreateOptionTypesFromImage, + }, + } + } else { + return nil, wrapAzureError(errors.New("Invalid configuration: can't configure unmanaged image URL without StorageAccount and BlobContainer")) + } vmParameters := compute.VirtualMachine{ Location: &az.azconfig.Location, @@ -400,19 +480,7 @@ func (az *azureInstanceSet) Create( HardwareProfile: &compute.HardwareProfile{ VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType), }, - StorageProfile: &compute.StorageProfile{ - OsDisk: &compute.OSDisk{ - OsType: compute.Linux, - Name: to.StringPtr(name + "-os"), - CreateOption: compute.FromImage, - Image: &compute.VirtualHardDisk{ - URI: to.StringPtr(string(imageID)), - }, - Vhd: &compute.VirtualHardDisk{ - URI: &instanceVhd, - }, - }, - }, + StorageProfile: storageProfile, NetworkProfile: &compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{ compute.NetworkInterfaceReference{ @@ -444,12 +512,15 @@ func (az *azureInstanceSet) Create( vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters) if err != nil { - _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil) - if delerr != nil { - az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create") + if az.blobcont != nil { + _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil) + if delerr != nil { + az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create") + } } + // Leave cleaning up of managed disks to the garbage collection in manageDisks() - _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name) + _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name) if delerr != nil { az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create") } @@ -478,26 +549,24 @@ func (az *azureInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, err return nil, wrapAzureError(err) } - instances := make([]cloud.Instance, 0) - + var instances []cloud.Instance for ; result.NotDone(); err = result.Next() { if err != nil { return nil, wrapAzureError(err) } - if strings.HasPrefix(*result.Value().Name, az.namePrefix) { - instances = append(instances, &azureInstance{ - provider: az, - vm: result.Value(), - nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID]}) - } + instances = append(instances, &azureInstance{ + provider: az, + vm: result.Value(), + nic: interfaces[*(*result.Value().NetworkProfile.NetworkInterfaces)[0].ID], + }) } return instances, nil } -// ManageNics returns a list of Azure network interface resources. -// Also performs garbage collection of NICs which have "namePrefix", are -// not associated with a virtual machine and have a "create-at" time -// more than DeleteDanglingResourcesAfter (to prevent racing and +// manageNics returns a list of Azure network interface resources. +// Also performs garbage collection of NICs which have "namePrefix", +// are not associated with a virtual machine and have a "created-at" +// time more than DeleteDanglingResourcesAfter (to prevent racing and // deleting newly created NICs) in the past are deleted. func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) { az.stopWg.Add(1) @@ -535,7 +604,7 @@ func (az *azureInstanceSet) manageNics() (map[string]network.Interface, error) { return interfaces, nil } -// ManageBlobs garbage collects blobs (VM disk images) in the +// manageBlobs garbage collects blobs (VM disk images) in the // configured storage account container. It will delete blobs which // have "namePrefix", are "available" (which means they are not // leased to a VM) and haven't been modified for @@ -570,11 +639,41 @@ func (az *azureInstanceSet) manageBlobs() { } } +// manageDisks garbage collects managed compute disks (VM disk images) in the +// configured resource group. It will delete disks which have "namePrefix", +// are "unattached" (which means they are not leased to a VM) and were created +// more than DeleteDanglingResourcesAfter seconds ago. (Azure provides no +// modification timestamp on managed disks, there is only a creation timestamp) +func (az *azureInstanceSet) manageDisks() { + + re := regexp.MustCompile(`^` + regexp.QuoteMeta(az.namePrefix) + `.*-os$`) + threshold := time.Now().Add(-az.azconfig.DeleteDanglingResourcesAfter.Duration()) + + response, err := az.disksClient.listByResourceGroup(az.ctx, az.imageResourceGroup) + if err != nil { + az.logger.WithError(err).Warn("Error listing disks") + return + } + + for ; response.NotDone(); err = response.Next() { + for _, d := range response.Values() { + if d.DiskProperties.DiskState == compute.Unattached && + d.Name != nil && re.MatchString(*d.Name) && + d.DiskProperties.TimeCreated.ToTime().Before(threshold) { + + az.logger.Printf("Disk %v is unlocked and was created at %+v, will delete", *d.Name, d.DiskProperties.TimeCreated.ToTime()) + az.deleteDisk <- d + } + } + } +} + func (az *azureInstanceSet) Stop() { az.stopFunc() az.stopWg.Wait() close(az.deleteNIC) close(az.deleteBlob) + close(az.deleteDisk) } type azureInstance struct { @@ -599,16 +698,12 @@ func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error { ai.provider.stopWg.Add(1) defer ai.provider.stopWg.Done() - tags := make(map[string]*string) - + tags := map[string]*string{} for k, v := range ai.vm.Tags { - if !strings.HasPrefix(k, "dispatch-") { - tags[k] = v - } + tags[k] = v } for k, v := range newTags { - newstr := v - tags["dispatch-"+k] = &newstr + tags[k] = to.StringPtr(v) } vmParameters := compute.VirtualMachine{ @@ -625,14 +720,10 @@ func (ai *azureInstance) SetTags(newTags cloud.InstanceTags) error { } func (ai *azureInstance) Tags() cloud.InstanceTags { - tags := make(map[string]string) - + tags := cloud.InstanceTags{} for k, v := range ai.vm.Tags { - if strings.HasPrefix(k, "dispatch-") { - tags[k[9:]] = *v - } + tags[k] = *v } - return tags } @@ -645,13 +736,17 @@ func (ai *azureInstance) Destroy() error { } func (ai *azureInstance) Address() string { - if ai.nic.IPConfigurations != nil && - len(*ai.nic.IPConfigurations) > 0 && - (*ai.nic.IPConfigurations)[0].PrivateIPAddress != nil { - - return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress + if iprops := ai.nic.InterfacePropertiesFormat; iprops == nil { + return "" + } else if ipconfs := iprops.IPConfigurations; ipconfs == nil || len(*ipconfs) == 0 { + return "" + } else if ipconfprops := (*ipconfs)[0].InterfaceIPConfigurationPropertiesFormat; ipconfprops == nil { + return "" + } else if addr := ipconfprops.PrivateIPAddress; addr == nil { + return "" + } else { + return *addr } - return "" } func (ai *azureInstance) RemoteUser() string {