Add a script to clean up old compute node images on Azure.
authorWard Vandewege <ward@jhvc.com>
Thu, 7 May 2020 01:30:04 +0000 (21:30 -0400)
committerWard Vandewege <ward@jhvc.com>
Thu, 7 May 2020 01:30:41 +0000 (21:30 -0400)
closes #16418

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@jhvc.com>

compute-image-cleaner/.gitignore [new file with mode: 0644]
compute-image-cleaner/Makefile [new file with mode: 0644]
compute-image-cleaner/azure.go [new file with mode: 0644]
compute-image-cleaner/compute-image-cleaner.go [new file with mode: 0644]
compute-image-cleaner/config/azure-config.go [new file with mode: 0644]
compute-image-cleaner/go.mod [new file with mode: 0644]
compute-image-cleaner/go.sum [new file with mode: 0644]
compute-image-cleaner/usage.go [new file with mode: 0644]

diff --git a/compute-image-cleaner/.gitignore b/compute-image-cleaner/.gitignore
new file mode 100644 (file)
index 0000000..b017886
--- /dev/null
@@ -0,0 +1 @@
+compute-image-cleaner
diff --git a/compute-image-cleaner/Makefile b/compute-image-cleaner/Makefile
new file mode 100644 (file)
index 0000000..3b42bc0
--- /dev/null
@@ -0,0 +1,11 @@
+
+build:
+       @go build -ldflags "-s -w"
+
+lint:
+       @gofmt -s -w *go
+       @golint
+       @cd config/; golint
+       @golangci-lint run
+       @cd config/; golangci-lint run
+
diff --git a/compute-image-cleaner/azure.go b/compute-image-cleaner/azure.go
new file mode 100644 (file)
index 0000000..c42dfcc
--- /dev/null
@@ -0,0 +1,114 @@
+// Copyright (C) The Azure-Samples Authors. All rights reserved.
+//
+// SPDX-License-Identifier: MIT
+
+// Largely borrowed from
+// https://github.com/Azure-Samples/azure-sdk-for-go-samples/blob/master/internal/iam/authorizers.go
+
+package main
+
+import (
+       "fmt"
+       "log"
+
+       "github.com/arvados/arvados-dev/compute-image-cleaner/config"
+
+       "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-06-01/storage"
+
+       "github.com/Azure/go-autorest/autorest"
+       "github.com/Azure/go-autorest/autorest/adal"
+       "github.com/Azure/go-autorest/autorest/azure/auth"
+)
+
+// OAuthGrantType specifies which grant type to use.
+type OAuthGrantType int
+
+const (
+       // OAuthGrantTypeServicePrincipal for client credentials flow
+       OAuthGrantTypeServicePrincipal OAuthGrantType = iota
+       // OAuthGrantTypeDeviceFlow for device flow
+       OAuthGrantTypeDeviceFlow
+)
+
+var (
+       armAuthorizer autorest.Authorizer
+)
+
+// GrantType returns what grant type has been configured.
+func grantType() OAuthGrantType {
+       if config.UseDeviceFlow() {
+               return OAuthGrantTypeDeviceFlow
+       }
+       return OAuthGrantTypeServicePrincipal
+}
+
+func getAuthorizerForResource(grantType OAuthGrantType, resource string) (autorest.Authorizer, error) {
+       var a autorest.Authorizer
+       var err error
+
+       switch grantType {
+
+       case OAuthGrantTypeServicePrincipal:
+               oauthConfig, err := adal.NewOAuthConfig(
+                       config.Environment().ActiveDirectoryEndpoint, config.TenantID())
+               if err != nil {
+                       return nil, err
+               }
+
+               token, err := adal.NewServicePrincipalToken(
+                       *oauthConfig, config.ClientID(), config.ClientSecret(), resource)
+               if err != nil {
+                       return nil, err
+               }
+               a = autorest.NewBearerAuthorizer(token)
+
+       case OAuthGrantTypeDeviceFlow:
+               deviceconfig := auth.NewDeviceFlowConfig(config.ClientID(), config.TenantID())
+               deviceconfig.Resource = resource
+               a, err = deviceconfig.Authorizer()
+               if err != nil {
+                       return nil, err
+               }
+
+       default:
+               return a, fmt.Errorf("invalid grant type specified")
+       }
+
+       return a, err
+}
+
+// GetResourceManagementAuthorizer gets an OAuthTokenAuthorizer for Azure Resource Manager
+func GetResourceManagementAuthorizer() (autorest.Authorizer, error) {
+       if armAuthorizer != nil {
+               return armAuthorizer, nil
+       }
+
+       var a autorest.Authorizer
+       var err error
+
+       a, err = getAuthorizerForResource(
+               grantType(), config.Environment().ResourceManagerEndpoint)
+
+       if err == nil {
+               // cache
+               armAuthorizer = a
+       } else {
+               // clear cache
+               armAuthorizer = nil
+       }
+       return armAuthorizer, err
+}
+
+func getStorageAccountsClient() storage.AccountsClient {
+       storageAccountsClient := storage.NewAccountsClient(config.SubscriptionID())
+       auth, err := GetResourceManagementAuthorizer()
+       if err != nil {
+               log.Fatal(err)
+       }
+       storageAccountsClient.Authorizer = auth
+       err = storageAccountsClient.AddToUserAgent("compute-image-cleaner")
+       if err != nil {
+               log.Fatal(err)
+       }
+       return storageAccountsClient
+}
diff --git a/compute-image-cleaner/compute-image-cleaner.go b/compute-image-cleaner/compute-image-cleaner.go
new file mode 100644 (file)
index 0000000..b812a38
--- /dev/null
@@ -0,0 +1,201 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+       "context"
+       "flag"
+       "fmt"
+       "log"
+       "net/url"
+       "os"
+       "regexp"
+       "sort"
+       "time"
+
+       "github.com/arvados/arvados-dev/compute-image-cleaner/config"
+
+       "github.com/Azure/azure-pipeline-go/pipeline"
+       "github.com/Azure/azure-storage-blob-go/azblob"
+
+       "code.cloudfoundry.org/bytefmt"
+)
+
+type blob struct {
+       name              string
+       created           time.Time
+       contentLength     int64
+       deletionCandidate bool
+}
+
+func prepAzBlob(storageKey string, account string, container string) (p pipeline.Pipeline, containerURL azblob.ContainerURL) {
+       // Create a default request pipeline using your storage account name and account key.
+       credential, err := azblob.NewSharedKeyCredential(account, storageKey)
+       if err != nil {
+               log.Fatal("Invalid credentials with error: " + err.Error())
+       }
+       p = azblob.NewPipeline(credential, azblob.PipelineOptions{})
+       // From the Azure portal, get your storage account blob service URL endpoint.
+       URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", account, container))
+
+       // Create a ContainerURL object that wraps the container URL and a request
+       // pipeline to make requests.
+       containerURL = azblob.NewContainerURL(*URL, p)
+
+       return
+}
+
+func loadBlobs(p pipeline.Pipeline, containerURL azblob.ContainerURL) (blobs []blob, blobNames map[string]*blob) {
+       blobNames = make(map[string]*blob)
+
+       ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+       defer cancel()
+
+       for marker := (azblob.Marker{}); marker.NotDone(); {
+               // Get a result segment starting with the blob indicated by the current Marker.
+               listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+               if err != nil {
+                       log.Fatal("Error getting blob list: " + err.Error())
+               }
+
+               // ListBlobs returns the start of the next segment; you MUST use this to get
+               // the next segment (after processing the current result segment).
+               marker = listBlob.NextMarker
+
+               // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute)
+               for _, blobInfo := range listBlob.Segment.BlobItems {
+                       blobs = append(blobs, blob{name: blobInfo.Name, created: *blobInfo.Properties.CreationTime, contentLength: *blobInfo.Properties.ContentLength})
+                       blobNames[blobInfo.Name] = &blobs[len(blobs)-1]
+               }
+       }
+       sort.Slice(blobs, func(i, j int) bool { return blobs[i].created.After(blobs[j].created) })
+
+       return
+}
+
+func weedBlobs(blobs []blob, blobNames map[string]*blob, containerURL azblob.ContainerURL, account string, container string, doIt bool) {
+       ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+       defer cancel()
+
+       var pairedFileName string
+       skipCount := 10
+       t := time.Now()
+       thirtyDaysAgo := t.AddDate(0, 0, -30)
+
+       // e.g. su92l-compute-osDisk.866eb426-8d1e-45ad-91be-2bb55b5a8147.vhd
+       vhd := regexp.MustCompile(`^(.*)-compute-osDisk\.(.*)\.vhd$`)
+       // e.g. su92l-compute-vmTemplate.866eb426-8d1e-45ad-91be-2bb55b5a8147.json
+       json := regexp.MustCompile(`^(.*)-compute-vmTemplate\.(.*)\.json$`)
+
+       for i, blob := range blobs {
+               matches := vhd.FindStringSubmatch(blob.name)
+               if len(matches) > 1 {
+                       // osDisk image file
+                       pairedFileName = matches[1] + "-compute-vmTemplate." + matches[2] + ".json"
+               } else {
+                       matches := json.FindStringSubmatch(blob.name)
+                       if len(matches) > 1 {
+                               // vmTemplate file
+                               pairedFileName = matches[1] + "-compute-osDisk." + matches[2] + ".vhd"
+                       } else {
+                               log.Println("Skipping blob because name does not match a known file name pattern:", blob.name, " ", blob.created)
+                               continue
+                       }
+               }
+               if blob.created.After(thirtyDaysAgo) {
+                       log.Println("Skipping blob because it was created less than 30 days ago:", blob.name, " ", blob.created)
+                       skipCount = skipCount - 1
+                       continue
+               }
+               if skipCount > 0 {
+                       log.Println("Skipping blob because it's in the top 10 most recent list:", blob.name, " ", blob.created)
+                       skipCount = skipCount - 1
+                       continue
+               }
+               if _, ok := blobNames[pairedFileName]; !ok {
+                       log.Println("Warning: paired file", pairedFileName, "not found for blob", blob.name, " ", blob.created)
+               }
+               blobs[i].deletionCandidate = true
+       }
+
+       var reclaimedSpace, otherSpace int64
+
+       for _, blob := range blobs {
+               if blob.deletionCandidate {
+                       log.Println("Candidate for deletion:", blob.name, " ", blob.created)
+                       reclaimedSpace = reclaimedSpace + blob.contentLength
+
+                       if doIt {
+                               log.Println("Deleting:", blob.name, " ", blob.created)
+                               blockBlobURL := containerURL.NewBlockBlobURL(blob.name)
+                               result, err := blockBlobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
+                               if err != nil {
+                                       log.Println(result)
+                                       log.Fatal("Error deleting blob: ", err.Error(), "\n", result)
+                               }
+                       }
+               } else {
+                       otherSpace = otherSpace + blob.contentLength
+               }
+       }
+
+       if doIt {
+               log.Println("Reclaimed", bytefmt.ByteSize(uint64(reclaimedSpace)), "or", reclaimedSpace, "bytes.")
+       } else {
+               log.Println("Deletion not requested. Able to reclaim", bytefmt.ByteSize(uint64(reclaimedSpace)), "or", reclaimedSpace, "bytes.")
+       }
+       log.Println("Kept", bytefmt.ByteSize(uint64(otherSpace)), "or", otherSpace, "bytes.")
+
+}
+
+func loadStorageAccountKey(resourceGroup string, account string) (key string) {
+       ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+       defer cancel()
+
+       storageClient := getStorageAccountsClient()
+       keys, err := storageClient.ListKeys(ctx, resourceGroup, account)
+       if err != nil {
+               log.Fatal("Error getting storage account key:", err.Error())
+       }
+
+       key = *(*keys.Keys)[0].Value
+
+       return
+}
+
+func validateInputs() (resourceGroup string, account string, container string, doIt bool) {
+       err := config.ParseEnvironment()
+       if err != nil {
+               log.Fatal("Unable to parse environment")
+       }
+
+       if config.ClientID() == "" || config.ClientSecret() == "" || config.TenantID() == "" || config.SubscriptionID() == "" {
+               log.Fatal("Please make sure the environment variables AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID and AZURE_SUBSCRIPTION_ID are set")
+       }
+
+       flags := flag.NewFlagSet("compute-image-cleaner", flag.ExitOnError)
+       flags.StringVar(&resourceGroup, "resourceGroup", "", "Name of the Azure resource group")
+       flags.StringVar(&account, "account", "", "Name of the Azure storage account")
+       flags.StringVar(&container, "container", "", "Name of the container in the Azure storage account")
+       flags.BoolVar(&doIt, "delete", false, "Delete blobs that meet criteria (default: false)")
+       flags.Usage = func() { usage(flags) }
+       err = flags.Parse(os.Args[1:])
+
+       if err != nil || resourceGroup == "" || account == "" || container == "" {
+               usage(flags)
+               os.Exit(1)
+       }
+
+       return
+}
+
+func main() {
+       resourceGroup, account, container, doIt := validateInputs()
+       storageKey := loadStorageAccountKey(resourceGroup, account)
+       p, containerURL := prepAzBlob(storageKey, account, container)
+
+       blobs, blobNames := loadBlobs(p, containerURL)
+       weedBlobs(blobs, blobNames, containerURL, account, container, doIt)
+}
diff --git a/compute-image-cleaner/config/azure-config.go b/compute-image-cleaner/config/azure-config.go
new file mode 100644 (file)
index 0000000..d411513
--- /dev/null
@@ -0,0 +1,84 @@
+// Copyright (C) The Azure-Samples Authors. All rights reserved.
+//
+// SPDX-License-Identifier: MIT
+
+// Largely borrowed from
+// https://github.com/Azure-Samples/azure-sdk-for-go-samples/tree/master/internal/config
+
+package config
+
+import (
+  "fmt"
+  "os"
+
+  "github.com/Azure/go-autorest/autorest/azure"
+)
+
+var (
+  clientID               string
+  clientSecret           string
+  tenantID               string
+  subscriptionID         string
+  cloudName              string = "AzurePublicCloud"
+  useDeviceFlow          bool
+  environment            *azure.Environment
+)
+
+// ClientID is the OAuth client ID.
+func ClientID() string {
+  return clientID
+}
+
+// ClientSecret is the OAuth client secret.
+func ClientSecret() string {
+  return clientSecret
+}
+
+// TenantID is the AAD tenant to which this client belongs.
+func TenantID() string {
+  return tenantID
+}
+
+// SubscriptionID is a target subscription for Azure resources.
+func SubscriptionID() string {
+  return subscriptionID
+}
+
+// UseDeviceFlow specifies if interactive auth should be used. Interactive
+// auth uses the OAuth Device Flow grant type.
+func UseDeviceFlow() bool {
+  return useDeviceFlow
+}
+
+// Environment returns an `azure.Environment{...}` for the current cloud.
+func Environment() *azure.Environment {
+  if environment != nil {
+    return environment
+  }
+  env, err := azure.EnvironmentFromName(cloudName)
+  if err != nil {
+    // TODO: move to initialization of var
+    panic(fmt.Sprintf(
+      "invalid cloud name '%s' specified, cannot continue\n", cloudName))
+  }
+  environment = &env
+  return environment
+}
+
+// ParseEnvironment loads the Azure environment variables for authentication
+func ParseEnvironment() error {
+  // these must be provided by environment
+  // clientID
+  clientID = os.Getenv("AZURE_CLIENT_ID")
+
+  // clientSecret
+  clientSecret = os.Getenv("AZURE_CLIENT_SECRET")
+
+  // tenantID (AAD)
+  tenantID = os.Getenv("AZURE_TENANT_ID")
+
+  // subscriptionID (ARM)
+  subscriptionID = os.Getenv("AZURE_SUBSCRIPTION_ID")
+
+  return nil
+}
diff --git a/compute-image-cleaner/go.mod b/compute-image-cleaner/go.mod
new file mode 100644 (file)
index 0000000..9330775
--- /dev/null
@@ -0,0 +1,24 @@
+module github.com/arvados/arvados-dev/compute-image-cleaner
+
+go 1.14
+
+require (
+       code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48
+       github.com/Azure/azure-pipeline-go v0.1.9
+       github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
+       github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc
+       github.com/Azure/go-autorest/autorest v0.10.1
+       github.com/Azure/go-autorest/autorest/adal v0.8.2
+       github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
+       github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
+       github.com/davecgh/go-spew v1.1.1 // indirect
+       github.com/golang/protobuf v1.3.1 // indirect
+       github.com/kr/pretty v0.1.0 // indirect
+       github.com/onsi/ginkgo v1.12.0 // indirect
+       github.com/onsi/gomega v1.9.0 // indirect
+       github.com/pkg/errors v0.8.1 // indirect
+       github.com/stretchr/testify v1.5.1 // indirect
+       golang.org/x/net v0.0.0-20190520210107-018c4d40a106 // indirect
+       golang.org/x/text v0.3.2 // indirect
+       gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+)
diff --git a/compute-image-cleaner/go.sum b/compute-image-cleaner/go.sum
new file mode 100644 (file)
index 0000000..0e8c43b
--- /dev/null
@@ -0,0 +1,106 @@
+code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE=
+code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
+github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg=
+github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc h1:BElWmFfsryQD72OcovStKpkIcd4e9ozSkdsTNQDSHGk=
+github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
+github.com/Azure/go-autorest/autorest v0.10.1 h1:uaB8A32IZU9YKs9v50+/LWIWTDHJk2vlGzbfd7FfESI=
+github.com/Azure/go-autorest/autorest v0.10.1/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
+github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk=
+github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM=
+github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U=
+github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4=
+github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI=
+github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190520210107-018c4d40a106 h1:EZofHp/BzEf3j39/+7CX1JvH0WaPG+ikBrqAdAPf+GM=
+golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/compute-image-cleaner/usage.go b/compute-image-cleaner/usage.go
new file mode 100644 (file)
index 0000000..13a6e4d
--- /dev/null
@@ -0,0 +1,30 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+       "flag"
+       "fmt"
+       "os"
+)
+
+func usage(fs *flag.FlagSet) {
+       fmt.Fprintf(os.Stderr, `
+compute-image-cleaner removes old compute images from the specified storage account/container.
+
+The following environment variables must be set:
+
+  AZURE_TENANT_ID
+  AZURE_SUBSCRIPTION_ID
+  AZURE_CLIENT_ID
+  AZURE_CLIENT_SECRET
+
+For more information about those values and for instructions to create a service principal, see
+https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal
+
+Usage:
+`)
+       fs.PrintDefaults()
+}