From 0e5ee27bb7bef018395a73f1fa2617050dc18d7a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Jul 2019 16:25:50 -0400 Subject: [PATCH] 14717: Migrate websockets to new config Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- lib/config/config.default.yml | 4 + lib/config/deprecated.go | 98 ++++++++++++++--- lib/config/export.go | 3 + lib/config/generated_config.go | 4 + lib/config/load.go | 9 ++ sdk/go/arvados/config.go | 3 + .../crunch-dispatch-slurm_test.go | 3 +- services/ws/config.go | 49 --------- services/ws/main.go | 54 +++++++--- services/ws/router.go | 12 ++- services/ws/server.go | 30 ++++-- services/ws/server_test.go | 102 +++++++++++++++--- 12 files changed, 264 insertions(+), 107 deletions(-) delete mode 100644 services/ws/config.go diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index 15bae9af89..1a1f1c5ac5 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -198,6 +198,10 @@ Clusters: # Maximum wall clock time to spend handling an incoming request. RequestTimeout: 5m + WebsocketKeepaliveTimeout: 60s + WebsocketClientEventQueue: 64 + WebsocketServerEventQueue: 4 + Users: # Config parameters to automatically setup new users. If enabled, # this users will be able to self-activate. Enable this if you want diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go index 8f3f3d7edd..4526706a3d 100644 --- a/lib/config/deprecated.go +++ b/lib/config/deprecated.go @@ -178,6 +178,24 @@ type oldCrunchDispatchSlurmConfig struct { const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml" +func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) { + if client == nil { + return + } + if client.APIHost != "" { + cluster.Services.Controller.ExternalURL.Host = client.APIHost + } + if client.Scheme != "" { + cluster.Services.Controller.ExternalURL.Scheme = client.Scheme + } else { + cluster.Services.Controller.ExternalURL.Scheme = "https" + } + if client.AuthToken != "" { + cluster.SystemRootToken = client.AuthToken + } + cluster.TLS.Insecure = client.Insecure +} + // update config using values from an crunch-dispatch-slurm config file. func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error { var oc oldCrunchDispatchSlurmConfig @@ -193,18 +211,7 @@ func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error { return err } - if oc.Client != nil { - u := arvados.URL{} - u.Host = oc.Client.APIHost - if oc.Client.Scheme != "" { - u.Scheme = oc.Client.Scheme - } else { - u.Scheme = "https" - } - cluster.Services.Controller.ExternalURL = u - cluster.SystemRootToken = oc.Client.AuthToken - cluster.TLS.Insecure = oc.Client.Insecure - } + loadOldClientConfig(cluster, oc.Client) if oc.SbatchArguments != nil { cluster.Containers.SLURM.SbatchArgumentsList = *oc.SbatchArguments @@ -236,3 +243,70 @@ func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error { cfg.Clusters[cluster.ClusterID] = *cluster return nil } + +type oldWsConfig struct { + Client *arvados.Client + Postgres *arvados.PostgreSQLConnection + PostgresPool *int + Listen *string + LogLevel *string + LogFormat *string + + PingTimeout *arvados.Duration + ClientEventQueue *int + ServerEventQueue *int + + ManagementToken *string +} + +const defaultWebsocketsConfigPath = "/etc/arvados/ws/ws.yml" + +// update config using values from an crunch-dispatch-slurm config file. +func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error { + var oc oldWsConfig + err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketsPath, &oc) + if os.IsNotExist(err) && ldr.WebsocketsPath == defaultWebsocketsConfigPath { + return nil + } else if err != nil { + return err + } + + cluster, err := cfg.GetCluster("") + if err != nil { + return err + } + + loadOldClientConfig(cluster, oc.Client) + fmt.Printf("Clllllllllllient %v %v", *oc.Client, cluster.Services.Controller.ExternalURL) + + if oc.Postgres != nil { + cluster.PostgreSQL.Connection = *oc.Postgres + } + if oc.PostgresPool != nil { + cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool + } + if oc.Listen != nil { + cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{} + } + if oc.LogLevel != nil { + cluster.SystemLogs.LogLevel = *oc.LogLevel + } + if oc.LogFormat != nil { + cluster.SystemLogs.Format = *oc.LogFormat + } + if oc.PingTimeout != nil { + cluster.API.WebsocketKeepaliveTimeout = *oc.PingTimeout + } + if oc.ClientEventQueue != nil { + cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue + } + if oc.ServerEventQueue != nil { + cluster.API.WebsocketServerEventQueue = *oc.ServerEventQueue + } + if oc.ManagementToken != nil { + cluster.ManagementToken = *oc.ManagementToken + } + + cfg.Clusters[cluster.ClusterID] = *cluster + return nil +} diff --git a/lib/config/export.go b/lib/config/export.go index dbbaac1274..1cb84a2a18 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -64,6 +64,9 @@ var whitelist = map[string]bool{ "API.MaxRequestSize": true, "API.RailsSessionSecretToken": false, "API.RequestTimeout": true, + "API.WebsocketClientEventQueue": false, + "API.WebsocketKeepaliveTimeout": true, + "API.WebsocketServerEventQueue": false, "AuditLogs": false, "AuditLogs.MaxAge": false, "AuditLogs.MaxDeleteBatch": false, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 58a7690f48..7bcb384416 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -204,6 +204,10 @@ Clusters: # Maximum wall clock time to spend handling an incoming request. RequestTimeout: 5m + WebsocketKeepaliveTimeout: 60s + WebsocketClientEventQueue: 64 + WebsocketServerEventQueue: 4 + Users: # Config parameters to automatically setup new users. If enabled, # this users will be able to self-activate. Enable this if you want diff --git a/lib/config/load.go b/lib/config/load.go index bce57d7598..63b6ac7d98 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -31,6 +31,7 @@ type Loader struct { Path string KeepstorePath string CrunchDispatchSlurmPath string + WebsocketsPath string configdata []byte } @@ -59,6 +60,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)") flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`") flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") + flagset.StringVar(&ldr.WebsocketsPath, "legacy-ws-config", defaultWebsocketsConfigPath, "Legacy arvados-ws configuration `file`") } // MungeLegacyConfigArgs checks args for a -config flag whose argument @@ -132,6 +134,12 @@ func (ldr *Loader) loadBytes(path string) ([]byte, error) { return ioutil.ReadAll(f) } +func (ldr *Loader) LoadDefaults() (*arvados.Config, error) { + ldr.configdata = []byte(`Clusters: {zzzzz: {}}`) + defer func() { ldr.configdata = nil }() + return ldr.Load() +} + func (ldr *Loader) Load() (*arvados.Config, error) { if ldr.configdata == nil { buf, err := ldr.loadBytes(ldr.Path) @@ -208,6 +216,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) { for _, err := range []error{ ldr.loadOldKeepstoreConfig(&cfg), ldr.loadOldCrunchDispatchSlurmConfig(&cfg), + ldr.loadOldWebsocketsConfig(&cfg), } { if err != nil { return nil, err diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 12ec8e6b26..9072b73192 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -76,6 +76,9 @@ type Cluster struct { MaxRequestSize int RailsSessionSecretToken string RequestTimeout Duration + WebsocketKeepaliveTimeout Duration + WebsocketClientEventQueue int + WebsocketServerEventQueue int } AuditLogs struct { MaxAge Duration diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go index ca3944d76e..6007c6d4a8 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go @@ -395,7 +395,7 @@ func (s *StubbedSuite) TestLoadLegacyConfig(c *C) { content := []byte(` Client: APIHost: example.com - APIToken: abcdefg + AuthToken: abcdefg SbatchArguments: ["--foo", "bar"] PollPeriod: 12s PrioritySpread: 42 @@ -422,6 +422,7 @@ BatchSize: 99 c.Check(err, IsNil) c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"}) + c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg") c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"}) c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second)) c.Check(s.disp.cluster.Containers.SLURM.PrioritySpread, Equals, int64(42)) diff --git a/services/ws/config.go b/services/ws/config.go deleted file mode 100644 index ead1ec20c6..0000000000 --- a/services/ws/config.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) The Arvados Authors. All rights reserved. -// -// SPDX-License-Identifier: AGPL-3.0 - -package main - -import ( - "time" - - "git.curoverse.com/arvados.git/sdk/go/arvados" -) - -type wsConfig struct { - Client arvados.Client - Postgres arvados.PostgreSQLConnection - PostgresPool int - Listen string - LogLevel string - LogFormat string - - PingTimeout arvados.Duration - ClientEventQueue int - ServerEventQueue int - - ManagementToken string -} - -func defaultConfig() wsConfig { - return wsConfig{ - Client: arvados.Client{ - APIHost: "localhost:443", - }, - Postgres: arvados.PostgreSQLConnection{ - "dbname": "arvados_production", - "user": "arvados", - "password": "xyzzy", - "host": "localhost", - "connect_timeout": "30", - "sslmode": "require", - "fallback_application_name": "arvados-ws", - }, - PostgresPool: 64, - LogLevel: "info", - LogFormat: "json", - PingTimeout: arvados.Duration(time.Minute), - ClientEventQueue: 64, - ServerEventQueue: 4, - } -} diff --git a/services/ws/main.go b/services/ws/main.go index a0006a4f8a..0556c77d61 100644 --- a/services/ws/main.go +++ b/services/ws/main.go @@ -7,47 +7,71 @@ package main import ( "flag" "fmt" + "os" - "git.curoverse.com/arvados.git/sdk/go/config" + "git.curoverse.com/arvados.git/lib/config" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/ctxlog" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) var logger = ctxlog.FromContext var version = "dev" -func main() { - log := logger(nil) +func configure(log logrus.FieldLogger, args []string) *arvados.Cluster { + flags := flag.NewFlagSet(args[0], flag.ExitOnError) + dumpConfig := flags.Bool("dump-config", false, "show current configuration and exit") + getVersion := flags.Bool("version", false, "Print version information and exit.") + + loader := config.NewLoader(nil, log) + loader.SetupFlags(flags) + args = loader.MungeLegacyConfigArgs(log, args[1:], "-legacy-ws-config") - configPath := flag.String("config", "/etc/arvados/ws/ws.yml", "`path` to config file") - dumpConfig := flag.Bool("dump-config", false, "show current configuration and exit") - getVersion := flag.Bool("version", false, "Print version information and exit.") - cfg := defaultConfig() - flag.Parse() + flags.Parse(args) // Print version information if requested if *getVersion { fmt.Printf("arvados-ws %s\n", version) - return + return nil } - err := config.LoadFile(&cfg, *configPath) + cfg, err := loader.Load() if err != nil { log.Fatal(err) } - ctxlog.SetLevel(cfg.LogLevel) - ctxlog.SetFormat(cfg.LogFormat) + cluster, err := cfg.GetCluster("") + if err != nil { + log.Fatal(err) + } + + ctxlog.SetLevel(cluster.SystemLogs.LogLevel) + ctxlog.SetFormat(cluster.SystemLogs.Format) if *dumpConfig { - txt, err := config.Dump(&cfg) + out, err := yaml.Marshal(cfg) if err != nil { log.Fatal(err) } - fmt.Print(string(txt)) + _, err = os.Stdout.Write(out) + if err != nil { + log.Fatal(err) + } + return nil + } + return cluster +} + +func main() { + log := logger(nil) + + cluster := configure(log, os.Args) + if cluster == nil { return } log.Printf("arvados-ws %s started", version) - srv := &server{wsConfig: &cfg} + srv := &server{cluster: cluster} log.Fatal(srv.Run()) } diff --git a/services/ws/router.go b/services/ws/router.go index a408b58bdd..5a5b7c53b2 100644 --- a/services/ws/router.go +++ b/services/ws/router.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/ctxlog" "git.curoverse.com/arvados.git/sdk/go/health" "github.com/sirupsen/logrus" @@ -27,7 +28,8 @@ type wsConn interface { } type router struct { - Config *wsConfig + client arvados.Client + cluster *arvados.Cluster eventSource eventSource newPermChecker func() permChecker @@ -52,8 +54,8 @@ type debugStatuser interface { func (rtr *router) setup() { rtr.handler = &handler{ - PingTimeout: rtr.Config.PingTimeout.Duration(), - QueueSize: rtr.Config.ClientEventQueue, + PingTimeout: time.Duration(rtr.cluster.API.WebsocketKeepaliveTimeout), + QueueSize: rtr.cluster.API.WebsocketClientEventQueue, } rtr.mux = http.NewServeMux() rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0)) @@ -62,7 +64,7 @@ func (rtr *router) setup() { rtr.mux.Handle("/status.json", rtr.jsonHandler(rtr.Status)) rtr.mux.Handle("/_health/", &health.Handler{ - Token: rtr.Config.ManagementToken, + Token: rtr.cluster.ManagementToken, Prefix: "/_health/", Routes: health.Routes{ "db": rtr.eventSource.DBHealth, @@ -87,7 +89,7 @@ func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server { stats := rtr.handler.Handle(ws, rtr.eventSource, func(ws wsConn, sendq chan<- interface{}) (session, error) { - return newSession(ws, sendq, rtr.eventSource.DB(), rtr.newPermChecker(), &rtr.Config.Client) + return newSession(ws, sendq, rtr.eventSource.DB(), rtr.newPermChecker(), &rtr.client) }) log.WithFields(logrus.Fields{ diff --git a/services/ws/server.go b/services/ws/server.go index eda7ff2a48..081ff53b30 100644 --- a/services/ws/server.go +++ b/services/ws/server.go @@ -10,13 +10,14 @@ import ( "sync" "time" + "git.curoverse.com/arvados.git/sdk/go/arvados" "github.com/coreos/go-systemd/daemon" ) type server struct { httpServer *http.Server listener net.Listener - wsConfig *wsConfig + cluster *arvados.Cluster eventSource *pgEventSource setupOnce sync.Once } @@ -40,27 +41,38 @@ func (srv *server) Run() error { func (srv *server) setup() { log := logger(nil) - ln, err := net.Listen("tcp", srv.wsConfig.Listen) + var listen arvados.URL + for listen, _ = range srv.cluster.Services.Websocket.InternalURLs { + break + } + ln, err := net.Listen("tcp", listen.Host) if err != nil { - log.WithField("Listen", srv.wsConfig.Listen).Fatal(err) + log.WithField("Listen", listen).Fatal(err) } log.WithField("Listen", ln.Addr().String()).Info("listening") + client := arvados.Client{} + client.APIHost = srv.cluster.Services.Controller.ExternalURL.Host + client.AuthToken = srv.cluster.SystemRootToken + client.Insecure = srv.cluster.TLS.Insecure + srv.listener = ln srv.eventSource = &pgEventSource{ - DataSource: srv.wsConfig.Postgres.String(), - MaxOpenConns: srv.wsConfig.PostgresPool, - QueueSize: srv.wsConfig.ServerEventQueue, + DataSource: srv.cluster.PostgreSQL.Connection.String(), + MaxOpenConns: srv.cluster.PostgreSQL.ConnectionPool, + QueueSize: srv.cluster.API.WebsocketServerEventQueue, } + srv.httpServer = &http.Server{ - Addr: srv.wsConfig.Listen, + Addr: listen.Host, ReadTimeout: time.Minute, WriteTimeout: time.Minute, MaxHeaderBytes: 1 << 20, Handler: &router{ - Config: srv.wsConfig, + cluster: srv.cluster, + client: client, eventSource: srv.eventSource, - newPermChecker: func() permChecker { return newPermChecker(srv.wsConfig.Client) }, + newPermChecker: func() permChecker { return newPermChecker(client) }, }, } diff --git a/services/ws/server_test.go b/services/ws/server_test.go index b1f943857a..097889c987 100644 --- a/services/ws/server_test.go +++ b/services/ws/server_test.go @@ -7,10 +7,13 @@ package main import ( "encoding/json" "io/ioutil" + "log" "net/http" + "os" "sync" "time" + "git.curoverse.com/arvados.git/lib/config" "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadostest" check "gopkg.in/check.v1" @@ -19,29 +22,42 @@ import ( var _ = check.Suite(&serverSuite{}) type serverSuite struct { - cfg *wsConfig - srv *server - wg sync.WaitGroup + cluster *arvados.Cluster + srv *server + wg sync.WaitGroup } func (s *serverSuite) SetUpTest(c *check.C) { - s.cfg = s.testConfig() - s.srv = &server{wsConfig: s.cfg} + var err error + s.cluster, err = s.testConfig() + c.Assert(err, check.IsNil) + s.srv = &server{cluster: s.cluster} } -func (*serverSuite) testConfig() *wsConfig { - cfg := defaultConfig() - cfg.Client = *(arvados.NewClientFromEnv()) - cfg.Postgres = testDBConfig() - cfg.Listen = ":" - cfg.ManagementToken = arvadostest.ManagementToken - return &cfg +func (*serverSuite) testConfig() (*arvados.Cluster, error) { + ldr := config.NewLoader(nil, nil) + cfg, err := ldr.LoadDefaults() + if err != nil { + return nil, err + } + cluster, err := cfg.GetCluster("") + if err != nil { + return nil, err + } + client := arvados.NewClientFromEnv() + cluster.Services.Controller.ExternalURL.Host = client.APIHost + cluster.SystemRootToken = client.AuthToken + cluster.TLS.Insecure = client.Insecure + cluster.PostgreSQL.Connection = testDBConfig() + cluster.Services.Websocket.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":"}: arvados.ServiceInstance{}} + cluster.ManagementToken = arvadostest.ManagementToken + return cluster, nil } // TestBadDB ensures Run() returns an error (instead of panicking or // deadlocking) if it can't connect to the database server at startup. func (s *serverSuite) TestBadDB(c *check.C) { - s.cfg.Postgres["password"] = "1234" + s.cluster.PostgreSQL.Connection["password"] = "1234" var wg sync.WaitGroup wg.Add(1) @@ -72,7 +88,7 @@ func (s *serverSuite) TestHealth(c *check.C) { go s.srv.Run() defer s.srv.Close() s.srv.WaitReady() - for _, token := range []string{"", "foo", s.cfg.ManagementToken} { + for _, token := range []string{"", "foo", s.cluster.ManagementToken} { req, err := http.NewRequest("GET", "http://"+s.srv.listener.Addr().String()+"/_health/ping", nil) c.Assert(err, check.IsNil) if token != "" { @@ -80,7 +96,7 @@ func (s *serverSuite) TestHealth(c *check.C) { } resp, err := http.DefaultClient.Do(req) c.Check(err, check.IsNil) - if token == s.cfg.ManagementToken { + if token == s.cluster.ManagementToken { c.Check(resp.StatusCode, check.Equals, http.StatusOK) buf, err := ioutil.ReadAll(resp.Body) c.Check(err, check.IsNil) @@ -107,7 +123,7 @@ func (s *serverSuite) TestStatus(c *check.C) { } func (s *serverSuite) TestHealthDisabled(c *check.C) { - s.cfg.ManagementToken = "" + s.cluster.ManagementToken = "" go s.srv.Run() defer s.srv.Close() @@ -122,3 +138,57 @@ func (s *serverSuite) TestHealthDisabled(c *check.C) { c.Check(resp.StatusCode, check.Equals, http.StatusNotFound) } } + +func (s *serverSuite) TestLoadLegacyConfig(c *check.C) { + content := []byte(` +Client: + APIHost: example.com + AuthToken: abcdefg +Postgres: + "dbname": "arvados_production" + "user": "arvados" + "password": "xyzzy" + "host": "localhost" + "connect_timeout": "30" + "sslmode": "require" + "fallback_application_name": "arvados-ws" +PostgresPool: 63 +Listen: ":8765" +LogLevel: "debug" +LogFormat: "text" +PingTimeout: 61s +ClientEventQueue: 62 +ServerEventQueue: 5 +ManagementToken: qqqqq +`) + tmpfile, err := ioutil.TempFile("", "example") + if err != nil { + log.Fatal(err) + } + + defer os.Remove(tmpfile.Name()) // clean up + + if _, err := tmpfile.Write(content); err != nil { + log.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + log.Fatal(err) + + } + cluster := configure(logger(nil), []string{"arvados-ws", "-config", tmpfile.Name()}) + c.Check(cluster, check.NotNil) + + c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"}) + c.Check(cluster.SystemRootToken, check.Equals, "abcdefg") + + c.Check(cluster.PostgreSQL.Connection.String(), check.Equals, "connect_timeout='30' dbname='arvados_production' fallback_application_name='arvados-ws' host='localhost' password='xyzzy' sslmode='require' user='arvados' ") + c.Check(cluster.PostgreSQL.ConnectionPool, check.Equals, 63) + c.Check(cluster.Services.Websocket.InternalURLs, check.DeepEquals, map[arvados.URL]arvados.ServiceInstance{ + arvados.URL{Host: ":8765"}: arvados.ServiceInstance{}}) + c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug") + c.Check(cluster.SystemLogs.Format, check.Equals, "text") + c.Check(cluster.API.WebsocketKeepaliveTimeout, check.Equals, arvados.Duration(61*time.Second)) + c.Check(cluster.API.WebsocketClientEventQueue, check.Equals, 62) + c.Check(cluster.API.WebsocketServerEventQueue, check.Equals, 5) + c.Check(cluster.ManagementToken, check.Equals, "qqqqq") +} -- 2.30.2