14324: Azure driver WIP

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/dispatchcloud/azure.go b/lib/cloud/azure.go
similarity index 95%
rename from lib/dispatchcloud/azure.go
rename to lib/cloud/azure.go
index 92970d1de37a71e200a08c3699f7b37e92f88663..734be7bef20225c28d6fa684c253f4ac76a6a787 100644 (file)
--- a/lib/dispatchcloud/azure.go
+++ b/lib/cloud/azure.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package dispatchcloud
+package cloud
 
 import (
        "context"
@@ -29,7 +29,7 @@ import (
        "golang.org/x/crypto/ssh"
 )
 
-type AzureProviderConfig struct {
+type AzureInstanceSetConfig struct {
        SubscriptionID               string  `json:"subscription_id"`
        ClientID                     string  `json:"key"`
        ClientSecret                 string  `json:"secret"`
@@ -185,8 +185,8 @@ func WrapAzureError(err error) error {
        return err
 }
 
-type AzureProvider struct {
-       azconfig          AzureProviderConfig
+type AzureInstanceSet struct {
+       azconfig          AzureInstanceSetConfig
        vmClient          VirtualMachinesClientWrapper
        netClient         InterfacesClientWrapper
        storageAcctClient storageacct.AccountsClient
@@ -196,8 +196,13 @@ type AzureProvider struct {
        namePrefix        string
 }
 
-func NewAzureProvider(azcfg AzureProviderConfig, dispatcherID string) (prv InstanceProvider, err error) {
-       ap := AzureProvider{}
+func NewAzureInstanceSet(config map[string]interface{}, dispatcherID string) (prv InstanceProvider, err error) {
+       azcfg := AzureInstanceSetConfig{}
+       err = mapstructure.Decode(config, &azcfg)
+       if err != nil {
+               return nil, err
+       }
+       ap := AzureInstanceSet{}
        err = ap.setup(azcfg, dispatcherID)
        if err != nil {
                return nil, err
@@ -205,7 +210,7 @@ func NewAzureProvider(azcfg AzureProviderConfig, dispatcherID string) (prv Insta
        return &ap, nil
 }
 
-func (az *AzureProvider) setup(azcfg AzureProviderConfig, dispatcherID string) (err error) {
+func (az *AzureInstanceSet) setup(azcfg AzureInstanceSetConfig, dispatcherID string) (err error) {
        az.azconfig = azcfg
        vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID)
        netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID)
@@ -241,7 +246,7 @@ func (az *AzureProvider) setup(azcfg AzureProviderConfig, dispatcherID string) (
        return nil
 }
 
-func (az *AzureProvider) Create(ctx context.Context,
+func (az *AzureInstanceSet) Create(ctx context.Context,
        instanceType arvados.InstanceType,
        imageId ImageID,
        newTags InstanceTags,
@@ -371,7 +376,7 @@ echo '%s-%s' > /home/crunch/node-token`, name, newTags["node-token"])))
        }, nil
 }
 
-func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
+func (az *AzureInstanceSet) Instances(ctx context.Context) ([]Instance, error) {
        interfaces, err := az.ManageNics(ctx)
        if err != nil {
                return nil, err
@@ -398,7 +403,7 @@ func (az *AzureProvider) Instances(ctx context.Context) ([]Instance, error) {
        return instances, nil
 }
 
-func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
+func (az *AzureInstanceSet) ManageNics(ctx context.Context) (map[string]network.Interface, error) {
        result, err := az.netClient.ListComplete(ctx, az.azconfig.ResourceGroup)
        if err != nil {
                return nil, WrapAzureError(err)
@@ -457,7 +462,7 @@ func (az *AzureProvider) ManageNics(ctx context.Context) (map[string]network.Int
        return interfaces, nil
 }
 
-func (az *AzureProvider) ManageBlobs(ctx context.Context) {
+func (az *AzureInstanceSet) ManageBlobs(ctx context.Context) {
        result, err := az.storageAcctClient.ListKeys(ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount)
        if err != nil {
                log.Printf("Couldn't get account keys %v", err)
@@ -527,11 +532,11 @@ func (az *AzureProvider) ManageBlobs(ctx context.Context) {
        }
 }
 
-func (az *AzureProvider) Stop() {
+func (az *AzureInstanceSet) Stop() {
 }
 
 type AzureInstance struct {
-       provider *AzureProvider
+       provider *AzureInstanceSet
        nic      network.Interface
        vm       compute.VirtualMachine
 }

diff --git a/lib/dispatchcloud/azure_test.go b/lib/cloud/azure_test.go
similarity index 99%
rename from lib/dispatchcloud/azure_test.go
rename to lib/cloud/azure_test.go
index 568a403a510c554a091f3ffa2791a76e662a7743..aaa3d2846a28be0ab7fbfe55be5c619a298ba86c 100644 (file)
--- a/lib/dispatchcloud/azure_test.go
+++ b/lib/cloud/azure_test.go
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package dispatchcloud
+package cloud
 
 import (
        "context"

diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index 295fd6105b2ed86fecc71415e11c3fdae3260a66..fe5362437299bb00ac04dac2b444cb0551afe202 100644 (file)
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -11,7 +11,9 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/arvados"
 )
 
-var drivers = map[string]cloud.Driver{}
+var drivers = map[string]cloud.Driver{
+       "azure": "",
+}
 
 func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID) (cloud.InstanceSet, error) {
        driver, ok := drivers[cluster.CloudVMs.Driver]

diff --git a/lib/dispatchcloud/provider.go b/lib/dispatchcloud/provider.go
deleted file mode 100644 (file)
index 9e3af07..0000000
--- a/lib/dispatchcloud/provider.go
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package dispatchcloud
-
-import (
-       "context"
-       "time"
-
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "golang.org/x/crypto/ssh"
-)
-
-// A RateLimitError should be returned by a Provider when the cloud
-// service indicates it is rejecting all API calls for some time
-// interval.
-type RateLimitError interface {
-       // Time before which the caller should expect requests to
-       // fail.
-       EarliestRetry() time.Time
-       error
-}
-
-// A QuotaError should be returned by a Provider when the cloud
-// service indicates the account cannot create more VMs than already
-// exist.
-type QuotaError interface {
-       // If true, don't create more instances until some existing
-       // instances are destroyed. If false, don't handle the error
-       // as a quota error.
-       IsQuotaError() bool
-       error
-}
-
-type InstanceTags map[string]string
-type InstanceID string
-type ImageID string
-
-// instance is implemented by the provider-specific instance types.
-type Instance interface {
-       // ID returns the provider's instance ID. It must be stable
-       // for the life of the instance.
-       ID() InstanceID
-
-       // String typically returns the cloud-provided instance ID.
-       String() string
-
-       // Get tags
-       Tags(context.Context) (InstanceTags, error)
-
-       // Replace tags with the given tags
-       SetTags(context.Context, InstanceTags) error
-
-       // Shut down the node
-       Destroy(context.Context) error
-
-       // SSH server hostname or IP address, or empty string if unknown pending creation.
-       Address() string
-
-       // Return nil if the given public key matches the instance's
-       // SSH server key. If the provided ssh client is not nil,
-       // VerifyPublicKey can use it to make outgoing network
-       // connections from the instance -- e.g., to use the cloud's
-       // "this instance's metadata" API.
-       VerifyPublicKey(context.Context, ssh.PublicKey, *ssh.Client) error
-}
-
-type InstanceProvider interface {
-       // Create a new instance. If supported by the driver, add the
-       // provided public key to /root/.ssh/authorized_keys.
-       //
-       // The returned error should implement RateLimitError and
-       // QuotaError where applicable.
-       Create(context.Context, arvados.InstanceType, ImageID, InstanceTags, ssh.PublicKey) (Instance, error)
-
-       // Return all instances, including ones that are booting or
-       // shutting down.
-       //
-       // An instance returned by successive calls to Instances() may
-       // -- but does not need to -- be represented by the same
-       // Instance object each time. Thus, the caller is responsible
-       // for de-duplicating the returned instances by comparing the
-       // InstanceIDs returned by the instances' ID() methods.
-       Instances(context.Context) ([]Instance, error)
-
-       // Stop any background tasks and release other resources.
-       Stop()
-}