From 3b1947092b856e2c3bdb733828b1c951ac158b06 Mon Sep 17 00:00:00 2001 From: Eric Biagiotti Date: Fri, 6 Sep 2019 14:39:39 -0400 Subject: [PATCH] 14714: Adds keep-balance to cluster config loading Also adds a deprecated config loading test and fixes the service file Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti --- lib/config/config.default.yml | 30 ++++++++++ lib/config/deprecated.go | 68 ++++++++++++++++++++++ lib/config/deprecated_test.go | 45 ++++++++++++++ lib/config/export.go | 4 ++ lib/config/generated_config.go | 30 ++++++++++ lib/config/load.go | 6 ++ sdk/go/arvados/config.go | 5 ++ services/keep-balance/keep-balance.service | 1 - 8 files changed, 188 insertions(+), 1 deletion(-) diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index 572a2558ed..4338c18ed1 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -382,6 +382,36 @@ Clusters: # The default is 2 weeks. BlobSigningTTL: 336h + # When running keep-balance, this is the destination filename for the + # list of lost block hashes if there are any, one per line. Updated atomically during + # each successful run. + BlobMissingReport: "" + + # keep-balance operates periodically, i.e.: do a + # scan/balance operation, sleep, repeat. + # + # BalancePeriod determines the interval between start times of + # successive scan/balance operations. If a scan/balance operation + # takes longer than RunPeriod, the next one will follow it + # immediately. + # + # If SIGUSR1 is received during an idle period between operations, + # the next operation will start immediately. + BalancePeriod: 10m + + # Limits the number of collections retrieved by keep-balance per + # API transaction. If this is zero, page size is + # determined by the API server's own page size limits (see + # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead). + BalanceCollectionBatch: 100000 + + # The size of keep-balance's internal queue of + # collections. Higher values use more memory and improve throughput + # by allowing keep-balance to fetch the next page of collections + # while the current page is still being processed. If this is zero + # or omitted, pages are processed serially. + BalanceCollectionBuffers: 1000 + # Default lifetime for ephemeral collections: 2 weeks. This must not # be less than BlobSigningTTL. DefaultTrashLifetime: 336h diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go index 0a030fb040..ba2c79acf1 100644 --- a/lib/config/deprecated.go +++ b/lib/config/deprecated.go @@ -474,3 +474,71 @@ func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error { cfg.Clusters[cluster.ClusterID] = *cluster return nil } + +const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml" + +type oldKeepBalanceConfig struct { + Client *arvados.Client + Listen *string + KeepServiceTypes *[]string + KeepServiceList *arvados.KeepServiceList + RunPeriod *arvados.Duration + CollectionBatchSize *int + CollectionBuffers *int + RequestTimeout *arvados.Duration + LostBlocksFile *string + ManagementToken *string +} + +func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error { + if ldr.KeepBalancePath == "" { + return nil + } + var oc oldKeepBalanceConfig + err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc) + if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath { + return nil + } else if err != nil { + return err + } + + cluster, err := cfg.GetCluster("") + if err != nil { + return err + } + + loadOldClientConfig(cluster, oc.Client) + + if oc.Listen != nil { + cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{} + } + if oc.ManagementToken != nil { + cluster.ManagementToken = *oc.ManagementToken + } + if oc.RunPeriod != nil { + cluster.Collections.BalancePeriod = *oc.RunPeriod + } + if oc.LostBlocksFile != nil { + cluster.Collections.BlobMissingReport = *oc.LostBlocksFile + } + if oc.CollectionBatchSize != nil { + cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize + } + if oc.CollectionBuffers != nil { + cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers + } + if oc.RequestTimeout != nil { + cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout + } + + msg := "To balance specfic keep services, please update to the cluster config." + if oc.KeepServiceTypes != nil && len(*oc.KeepServiceTypes) > 0 { + ldr.Logger.Warnf("The KeepServiceType configuration option is not longer supported and is being ignored. %s", msg) + } + if oc.KeepServiceList != nil { + return fmt.Errorf("The KeepServiceList configuration option is no longer supported. Please remove it from your configuration file. %s", msg) + } + + cfg.Clusters[cluster.ClusterID] = *cluster + return nil +} diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go index ea9b50d035..8b80d6275b 100644 --- a/lib/config/deprecated_test.go +++ b/lib/config/deprecated_test.go @@ -216,3 +216,48 @@ func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) { c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot") c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{}) } + +func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) { + f := "-legacy-keepbalance-config" + content := []byte(fmtKeepBalanceConfig("")) + cluster, err := testLoadLegacyConfig(content, f, c) + + c.Check(err, check.IsNil) + c.Check(cluster, check.NotNil) + c.Check(cluster.ManagementToken, check.Equals, "xyzzy") + c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{}) + c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000) + c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000) + c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m") + c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile") + c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m") + + content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`)) + _, err = testLoadLegacyConfig(content, f, c) + c.Check(err, check.IsNil) + + content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`)) + _, err = testLoadLegacyConfig(content, f, c) + c.Check(err, check.NotNil) +} + +func fmtKeepBalanceConfig(param string) string { + return fmt.Sprintf(` +{ + "Client": { + "Scheme": "", + "APIHost": "example.com", + "AuthToken": "abcdefg", + "Insecure": false + }, + "Listen": ":80", + %s + "RunPeriod": "10m", + "CollectionBatchSize": 100000, + "CollectionBuffers": 1000, + "RequestTimeout": "30m", + "ManagementToken": "xyzzy", + "LostBlocksFile": "testfile" +} +`, param) +} diff --git a/lib/config/export.go b/lib/config/export.go index 8df561c00f..5437836f6f 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -99,6 +99,10 @@ var whitelist = map[string]bool{ "Collections.TrashSweepInterval": false, "Collections.TrustAllContent": false, "Collections.WebDAVCache": false, + "Collections.BalanceCollectionBatch": false, + "Collections.BalancePeriod": false, + "Collections.BlobMissingReport": false, + "Collections.BalanceCollectionBuffers": false, "Containers": true, "Containers.CloudVMs": false, "Containers.CrunchRunCommand": false, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 32c101a5a0..3806bbd8ad 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -388,6 +388,36 @@ Clusters: # The default is 2 weeks. BlobSigningTTL: 336h + # When running keep-balance, this is the destination filename for the + # list of lost block hashes if there are any, one per line. Updated atomically during + # each successful run. + BlobMissingReport: "" + + # keep-balance operates periodically, i.e.: do a + # scan/balance operation, sleep, repeat. + # + # BalancePeriod determines the interval between start times of + # successive scan/balance operations. If a scan/balance operation + # takes longer than RunPeriod, the next one will follow it + # immediately. + # + # If SIGUSR1 is received during an idle period between operations, + # the next operation will start immediately. + BalancePeriod: 10m + + # Limits the number of collections retrieved by keep-balance per + # API transaction. If this is zero, page size is + # determined by the API server's own page size limits (see + # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead). + BalanceCollectionBatch: 100000 + + # The size of keep-balance's internal queue of + # collections. Higher values use more memory and improve throughput + # by allowing keep-balance to fetch the next page of collections + # while the current page is still being processed. If this is zero + # or omitted, pages are processed serially. + BalanceCollectionBuffers: 1000 + # Default lifetime for ephemeral collections: 2 weeks. This must not # be less than BlobSigningTTL. DefaultTrashLifetime: 336h diff --git a/lib/config/load.go b/lib/config/load.go index 93c36f69ed..8c335f4c7c 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -37,6 +37,7 @@ type Loader struct { WebsocketPath string KeepproxyPath string GitHttpdPath string + KeepBalancePath string configdata []byte } @@ -69,6 +70,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`") + flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`") flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") } @@ -149,6 +151,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string, if legacyConfigArg != "-legacy-git-httpd-config" { ldr.GitHttpdPath = "" } + if legacyConfigArg != "-legacy-keepbalance-config" { + ldr.KeepBalancePath = "" + } return munged } @@ -251,6 +256,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) { ldr.loadOldWebsocketConfig(&cfg), ldr.loadOldKeepproxyConfig(&cfg), ldr.loadOldGitHttpdConfig(&cfg), + ldr.loadOldKeepBalanceConfig(&cfg), } { if err != nil { return nil, err diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 076a3c44d7..7c1c353809 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -119,6 +119,11 @@ type Cluster struct { TrashSweepInterval Duration TrustAllContent bool + BlobMissingReport string + BalancePeriod Duration + BalanceCollectionBatch int + BalanceCollectionBuffers int + WebDAVCache WebDAVCacheConfig } Git struct { diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service index 5638716078..1b71fb4e44 100644 --- a/services/keep-balance/keep-balance.service +++ b/services/keep-balance/keep-balance.service @@ -6,7 +6,6 @@ Description=Arvados Keep Balance Documentation=https://doc.arvados.org/ After=network.target -AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml # systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section StartLimitInterval=0 -- 2.30.2