From 56f01599bd6345ad65712db7eed1940726b2f8c2 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Fri, 22 Sep 2023 18:57:22 +0200 Subject: [PATCH 1/2] sql: read dbstat once in 60 minutes by default allow to disable it by setting interval to 0 --- config/config.go | 38 ++++++++++++++++++++------------------ config/mainnet.go | 13 +++++++------ config/presets/testnet.go | 11 ++++++----- node/node.go | 4 ++-- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index b0795bfbc5..2feb7022a2 100644 --- a/config/config.go +++ b/config/config.go @@ -108,9 +108,10 @@ type BaseConfig struct { OptFilterThreshold int `mapstructure:"optimistic-filtering-threshold"` TickSize uint64 `mapstructure:"tick-size"` - DatabaseConnections int `mapstructure:"db-connections"` - DatabaseLatencyMetering bool `mapstructure:"db-latency-metering"` - DatabasePruneInterval time.Duration `mapstructure:"db-prune-interval"` + DatabaseConnections int `mapstructure:"db-connections"` + DatabaseLatencyMetering bool `mapstructure:"db-latency-metering"` + DatabaseSizeMeteringInterval time.Duration `mapstructure:"db-size-metering-interval"` + DatabasePruneInterval time.Duration `mapstructure:"db-prune-interval"` NetworkHRP string `mapstructure:"network-hrp"` @@ -174,21 +175,22 @@ func DefaultTestConfig() Config { // DefaultBaseConfig returns a default configuration for spacemesh. func defaultBaseConfig() BaseConfig { return BaseConfig{ - DataDirParent: defaultDataDir, - FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), - CollectMetrics: false, - MetricsPort: 1010, - ProfilerName: "gp-spacemesh", - LayerDuration: 30 * time.Second, - LayersPerEpoch: 3, - PoETServers: []string{"127.0.0.1"}, - TxsPerProposal: 100, - BlockGasLimit: math.MaxUint64, - OptFilterThreshold: 90, - TickSize: 100, - DatabaseConnections: 16, - DatabasePruneInterval: 30 * time.Minute, - NetworkHRP: "sm", + DataDirParent: defaultDataDir, + FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), + CollectMetrics: false, + MetricsPort: 1010, + ProfilerName: "gp-spacemesh", + LayerDuration: 30 * time.Second, + LayersPerEpoch: 3, + PoETServers: []string{"127.0.0.1"}, + TxsPerProposal: 100, + BlockGasLimit: math.MaxUint64, + OptFilterThreshold: 90, + TickSize: 100, + DatabaseConnections: 16, + DatabaseSizeMeteringInterval: 10 * time.Minute, + DatabasePruneInterval: 30 * time.Minute, + NetworkHRP: "sm", } } diff --git a/config/mainnet.go b/config/mainnet.go index b8472f8a76..e76c36b1f5 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -42,12 +42,13 @@ func MainnetConfig() Config { logging.TrtlLoggerLevel = zapcore.WarnLevel.String() return Config{ BaseConfig: BaseConfig{ - DataDirParent: defaultDataDir, - FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), - MetricsPort: 1010, - DatabaseConnections: 16, - DatabasePruneInterval: 30 * time.Minute, - NetworkHRP: "sm", + DataDirParent: defaultDataDir, + FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), + MetricsPort: 1010, + DatabaseConnections: 16, + DatabaseSizeMeteringInterval: 60 * time.Minute, + DatabasePruneInterval: 30 * time.Minute, + NetworkHRP: "sm", LayerDuration: 5 * time.Minute, LayerAvgSize: 50, diff --git a/config/presets/testnet.go b/config/presets/testnet.go index 03b005daec..2c7c404ba1 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -47,11 +47,12 @@ func testnet() config.Config { defaultdir := filepath.Join(home, "spacemesh-testnet", "/") return config.Config{ BaseConfig: config.BaseConfig{ - DataDirParent: defaultdir, - FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), - MetricsPort: 1010, - DatabaseConnections: 16, - NetworkHRP: "stest", + DataDirParent: defaultdir, + FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), + MetricsPort: 1010, + DatabaseConnections: 16, + DatabaseSizeMeteringInterval: 10 * time.Minute, + NetworkHRP: "stest", LayerDuration: 5 * time.Minute, LayerAvgSize: 50, diff --git a/node/node.go b/node/node.go index 95d68455b8..d86c04c1e9 100644 --- a/node/node.go +++ b/node/node.go @@ -1341,8 +1341,8 @@ func (app *App) setupDBs(ctx context.Context, lg log.Log) error { return fmt.Errorf("open sqlite db %w", err) } app.db = sqlDB - if app.Config.CollectMetrics { - app.dbMetrics = dbmetrics.NewDBMetricsCollector(ctx, sqlDB, app.addLogger(StateDbLogger, lg), 5*time.Minute) + if app.Config.CollectMetrics && app.Config.DatabaseSizeMeteringInterval != 0 { + app.dbMetrics = dbmetrics.NewDBMetricsCollector(ctx, sqlDB, app.addLogger(StateDbLogger, lg), app.Config.DatabaseSizeMeteringInterval) } app.cachedDB = datastore.NewCachedDB(sqlDB, app.addLogger(CachedDBLogger, lg), datastore.WithConfig(app.Config.Cache)) return nil From a6c7f4f476b060358adf080b6cf31f393365d6cc Mon Sep 17 00:00:00 2001 From: Dmitry Date: Fri, 22 Sep 2023 19:06:08 +0200 Subject: [PATCH 2/2] disable and add changelog --- CHANGELOG.md | 8 ++++++++ config/mainnet.go | 13 ++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a9347f18..1a76dda5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,14 @@ Support for old certificate sync protocol is dropped. This update is incompatibl ### Features * [#5031](https://github.com/spacemeshos/go-spacemesh/pull/5031) Nodes will also fetch from PoET 112 for round 4 if they were able to register to PoET 110. +* [#5067](https://github.com/spacemeshos/go-spacemesh/pull/5067) dbstat virtual table can be read periodically to collect table/index sizes. + +In order to enable provide following configuration: +```json +"main": { + "db-size-metering-interval": "10m" +} +``` ### Improvements diff --git a/config/mainnet.go b/config/mainnet.go index e76c36b1f5..b8472f8a76 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -42,13 +42,12 @@ func MainnetConfig() Config { logging.TrtlLoggerLevel = zapcore.WarnLevel.String() return Config{ BaseConfig: BaseConfig{ - DataDirParent: defaultDataDir, - FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), - MetricsPort: 1010, - DatabaseConnections: 16, - DatabaseSizeMeteringInterval: 60 * time.Minute, - DatabasePruneInterval: 30 * time.Minute, - NetworkHRP: "sm", + DataDirParent: defaultDataDir, + FileLock: filepath.Join(os.TempDir(), "spacemesh.lock"), + MetricsPort: 1010, + DatabaseConnections: 16, + DatabasePruneInterval: 30 * time.Minute, + NetworkHRP: "sm", LayerDuration: 5 * time.Minute, LayerAvgSize: 50,