15003: Update dispatchcloud to new config struct.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 71ff9c784e958fa7927cb3ca57214593d74eecd7..3bf0ee9bd558eeecc27919388874af786baca6c9 100644 (file)
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -95,7 +95,7 @@ func (disp *dispatcher) Close() {
 // Make a worker.Executor for the given instance.
 func (disp *dispatcher) newExecutor(inst cloud.Instance) worker.Executor {
        exr := ssh_executor.New(inst)
-       exr.SetTargetPort(disp.Cluster.CloudVMs.SSHPort)
+       exr.SetTargetPort(disp.Cluster.Containers.CloudVMs.SSHPort)
        exr.SetSigners(disp.sshKey)
        return exr
 }
@@ -126,8 +126,8 @@ func (disp *dispatcher) initialize() {
        disp.stop = make(chan struct{}, 1)
        disp.stopped = make(chan struct{})
 
-       if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Dispatch.PrivateKey)); err != nil {
-               disp.logger.Fatalf("error parsing configured Dispatch.PrivateKey: %s", err)
+       if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Containers.DispatchPrivateKey)); err != nil {
+               disp.logger.Fatalf("error parsing configured Containers.DispatchPrivateKey: %s", err)
        } else {
                disp.sshKey = key
        }
@@ -167,11 +167,11 @@ func (disp *dispatcher) run() {
        defer disp.instanceSet.Stop()
        defer disp.pool.Stop()
 
-       staleLockTimeout := time.Duration(disp.Cluster.Dispatch.StaleLockTimeout)
+       staleLockTimeout := time.Duration(disp.Cluster.Containers.StaleLockTimeout)
        if staleLockTimeout == 0 {
                staleLockTimeout = defaultStaleLockTimeout
        }
-       pollInterval := time.Duration(disp.Cluster.Dispatch.PollInterval)
+       pollInterval := time.Duration(disp.Cluster.Containers.CloudVMs.PollInterval)
        if pollInterval <= 0 {
                pollInterval = defaultPollInterval
        }

diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 00157b75c649226880898c802973e9cd03a82173..6b8620ade3d3dbf4f36a1230cb17ae12426c31c5 100644 (file)
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -49,23 +49,23 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
        }
 
        s.cluster = &arvados.Cluster{
-               CloudVMs: arvados.CloudVMs{
-                       Driver:               "test",
-                       SyncInterval:         arvados.Duration(10 * time.Millisecond),
-                       TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
-                       TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
-                       TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
-                       TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
-                       MaxCloudOpsPerSecond: 500,
-               },
-               Dispatch: arvados.Dispatch{
-                       PrivateKey:         string(dispatchprivraw),
-                       PollInterval:       arvados.Duration(5 * time.Millisecond),
-                       ProbeInterval:      arvados.Duration(5 * time.Millisecond),
+               Containers: arvados.ContainersConfig{
+                       DispatchPrivateKey: string(dispatchprivraw),
                        StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
-                       MaxProbesPerSecond: 1000,
-                       TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
-                       TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
+                       CloudVMs: arvados.CloudVMsConfig{
+                               Driver:               "test",
+                               SyncInterval:         arvados.Duration(10 * time.Millisecond),
+                               TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
+                               TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
+                               TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
+                               TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
+                               MaxCloudOpsPerSecond: 500,
+                               PollInterval:         arvados.Duration(5 * time.Millisecond),
+                               ProbeInterval:        arvados.Duration(5 * time.Millisecond),
+                               MaxProbesPerSecond:   1000,
+                               TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
+                               TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
+                       },
                },
                InstanceTypes: arvados.InstanceTypeMap{
                        test.InstanceType(1).Name:  test.InstanceType(1),
@@ -76,16 +76,9 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
                        test.InstanceType(8).Name:  test.InstanceType(8),
                        test.InstanceType(16).Name: test.InstanceType(16),
                },
-               NodeProfiles: map[string]arvados.NodeProfile{
-                       "*": {
-                               Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
-                               DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
-                       },
-               },
-               Services: arvados.Services{
-                       Controller: arvados.Service{ExternalURL: arvados.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}},
-               },
        }
+       arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
+       arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
 
        arvClient, err := arvados.NewClientFromConfig(s.cluster)
        c.Check(err, check.IsNil)
@@ -242,7 +235,7 @@ func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 
 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
        s.cluster.ManagementToken = "abcdefgh"
-       s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
+       s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
        drivers["test"] = s.stubDriver
        s.disp.setupOnce.Do(s.disp.initialize)
        s.disp.queue = &test.Queue{}

diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index eb1e48737c8b131cbb919ca71e8f6bbc377c553a..5ec0f73e75bc035a3f2dad725c028e8c5b8502a8 100644 (file)
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -22,12 +22,12 @@ var drivers = map[string]cloud.Driver{
 }
 
 func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID, logger logrus.FieldLogger) (cloud.InstanceSet, error) {
-       driver, ok := drivers[cluster.CloudVMs.Driver]
+       driver, ok := drivers[cluster.Containers.CloudVMs.Driver]
        if !ok {
-               return nil, fmt.Errorf("unsupported cloud driver %q", cluster.CloudVMs.Driver)
+               return nil, fmt.Errorf("unsupported cloud driver %q", cluster.Containers.CloudVMs.Driver)
        }
-       is, err := driver.InstanceSet(cluster.CloudVMs.DriverParameters, setID, logger)
-       if maxops := cluster.CloudVMs.MaxCloudOpsPerSecond; maxops > 0 {
+       is, err := driver.InstanceSet(cluster.Containers.CloudVMs.DriverParameters, setID, logger)
+       if maxops := cluster.Containers.CloudVMs.MaxCloudOpsPerSecond; maxops > 0 {
                is = &rateLimitedInstanceSet{
                        InstanceSet: is,
                        ticker:      time.NewTicker(time.Second / time.Duration(maxops)),

diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 014ab93bfe9c7289bcd99286379a3a26bbc38b18..84b61fc006c239e3cc67e9a7828e6ebe29b6546e 100644 (file)
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -97,18 +97,18 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe
                arvClient:          arvClient,
                instanceSet:        &throttledInstanceSet{InstanceSet: instanceSet},
                newExecutor:        newExecutor,
-               bootProbeCommand:   cluster.CloudVMs.BootProbeCommand,
-               imageID:            cloud.ImageID(cluster.CloudVMs.ImageID),
+               bootProbeCommand:   cluster.Containers.CloudVMs.BootProbeCommand,
+               imageID:            cloud.ImageID(cluster.Containers.CloudVMs.ImageID),
                instanceTypes:      cluster.InstanceTypes,
-               maxProbesPerSecond: cluster.Dispatch.MaxProbesPerSecond,
-               probeInterval:      duration(cluster.Dispatch.ProbeInterval, defaultProbeInterval),
-               syncInterval:       duration(cluster.CloudVMs.SyncInterval, defaultSyncInterval),
-               timeoutIdle:        duration(cluster.CloudVMs.TimeoutIdle, defaultTimeoutIdle),
-               timeoutBooting:     duration(cluster.CloudVMs.TimeoutBooting, defaultTimeoutBooting),
-               timeoutProbe:       duration(cluster.CloudVMs.TimeoutProbe, defaultTimeoutProbe),
-               timeoutShutdown:    duration(cluster.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown),
-               timeoutTERM:        duration(cluster.Dispatch.TimeoutTERM, defaultTimeoutTERM),
-               timeoutSignal:      duration(cluster.Dispatch.TimeoutSignal, defaultTimeoutSignal),
+               maxProbesPerSecond: cluster.Containers.CloudVMs.MaxProbesPerSecond,
+               probeInterval:      duration(cluster.Containers.CloudVMs.ProbeInterval, defaultProbeInterval),
+               syncInterval:       duration(cluster.Containers.CloudVMs.SyncInterval, defaultSyncInterval),
+               timeoutIdle:        duration(cluster.Containers.CloudVMs.TimeoutIdle, defaultTimeoutIdle),
+               timeoutBooting:     duration(cluster.Containers.CloudVMs.TimeoutBooting, defaultTimeoutBooting),
+               timeoutProbe:       duration(cluster.Containers.CloudVMs.TimeoutProbe, defaultTimeoutProbe),
+               timeoutShutdown:    duration(cluster.Containers.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown),
+               timeoutTERM:        duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM),
+               timeoutSignal:      duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal),
                installPublicKey:   installPublicKey,
                stop:               make(chan bool),
        }

diff --git a/lib/dispatchcloud/worker/pool_test.go b/lib/dispatchcloud/worker/pool_test.go
index fc33a7ab235d7a733903903219302a81c8fc44d0..69395366895d3eb515c45705032dd722894f083e 100644 (file)
--- a/lib/dispatchcloud/worker/pool_test.go
+++ b/lib/dispatchcloud/worker/pool_test.go
@@ -76,13 +76,13 @@ func (suite *PoolSuite) TestResumeAfterRestart(c *check.C) {
        }
 
        cluster := &arvados.Cluster{
-               Dispatch: arvados.Dispatch{
-                       MaxProbesPerSecond: 1000,
-                       ProbeInterval:      arvados.Duration(time.Millisecond * 10),
-               },
-               CloudVMs: arvados.CloudVMs{
-                       BootProbeCommand: "true",
-                       SyncInterval:     arvados.Duration(time.Millisecond * 10),
+               Containers: arvados.ContainersConfig{
+                       CloudVMs: arvados.CloudVMsConfig{
+                               BootProbeCommand:   "true",
+                               MaxProbesPerSecond: 1000,
+                               ProbeInterval:      arvados.Duration(time.Millisecond * 10),
+                               SyncInterval:       arvados.Duration(time.Millisecond * 10),
+                       },
                },
                InstanceTypes: arvados.InstanceTypeMap{
                        type1.Name: type1,

diff --git a/sdk/go/arvadostest/stub.go b/sdk/go/arvadostest/stub.go
index 6b24a38fd79eb0139f6a1b38da3d4e39b725b3ab..2b6e40c950d4a7cc2ab54c18278162566563d238 100644 (file)
--- a/sdk/go/arvadostest/stub.go
+++ b/sdk/go/arvadostest/stub.go
@@ -51,4 +51,5 @@ func SetServiceURL(service *arvados.Service, internalURL string) {
                panic(err)
        }
        service.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL(*u): {}}
+       service.ExternalURL = arvados.URL(*u)
 }