Skip to content

Commit

Permalink
code review feedback and delay pruning of activeset to next update
Browse files Browse the repository at this point in the history
  • Loading branch information
countvonzero committed Sep 17, 2023
1 parent 8d879e0 commit 5a20c04
Show file tree
Hide file tree
Showing 17 changed files with 309 additions and 187 deletions.
8 changes: 0 additions & 8 deletions blocks/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,6 @@ var (
failFetchCnt = blockGenCount.WithLabelValues(failFetch)
failGenCnt = blockGenCount.WithLabelValues(failGen)
failErrCnt = blockGenCount.WithLabelValues(internalErr)

deleteLatency = metrics.NewHistogramWithBuckets(
"delete_duration",
namespace,
"duration in second to delete old proposals",
[]string{},
prometheus.ExponentialBuckets(0.01, 2, 10),
).WithLabelValues()
)

type collector struct {
Expand Down
112 changes: 0 additions & 112 deletions mesh/janitor.go

This file was deleted.

11 changes: 0 additions & 11 deletions mesh/metrics/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,4 @@ var (
[]string{},
prometheus.ExponentialBuckets(1, 2, 16),
)

pruneLatency = metrics.NewHistogramWithBuckets(
"prune_seconds",
Subsystem,
"prune time in seconds",
[]string{"step"},
prometheus.ExponentialBuckets(0.01, 2, 10),
)
PruneProposalLatency = pruneLatency.WithLabelValues("proposal")
PruneCertLatency = pruneLatency.WithLabelValues("cert")
PrunePropTxLatency = pruneLatency.WithLabelValues("proptxs")
)
6 changes: 4 additions & 2 deletions node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,10 @@ import (
"github.com/spacemeshos/go-spacemesh/p2p"
"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
"github.com/spacemeshos/go-spacemesh/proposals"
"github.com/spacemeshos/go-spacemesh/prune"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/ballots/util"
"github.com/spacemeshos/go-spacemesh/sql/layers"
dbmetrics "github.com/spacemeshos/go-spacemesh/sql/metrics"
"github.com/spacemeshos/go-spacemesh/syncer"
Expand Down Expand Up @@ -643,7 +645,7 @@ func (app *App) initServices(ctx context.Context) error {
}

app.eg.Go(func() error {
mesh.Prune(ctx, mlog.Zap(), app.db, app.clock, app.Config.HareEligibility.ConfidenceParam, app.Config.DatabasePruneInterval)
prune.Prune(ctx, mlog.Zap(), app.db, app.clock, app.Config.Tortoise.Hdist, app.Config.DatabasePruneInterval)
return nil
})

Expand Down Expand Up @@ -1336,7 +1338,7 @@ func (app *App) setupDBs(ctx context.Context, lg log.Log) error {
sqlDB, err := sql.Open("file:"+filepath.Join(dbPath, dbFile),
sql.WithConnections(app.Config.DatabaseConnections),
sql.WithLatencyMetering(app.Config.DatabaseLatencyMetering),
sql.WithV4PreMigration(mesh.ExtractActiveSet),
sql.WithV4Migration(util.ExtractActiveSet),
)
if err != nil {
return fmt.Errorf("open sqlite db %w", err)
Expand Down
11 changes: 11 additions & 0 deletions prune/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package prune

import (
"github.com/spacemeshos/go-spacemesh/common/types"
)

//go:generate mockgen -typed -package=prune -destination=./mocks.go -source=./interface.go

type layerClock interface {
CurrentLayer() types.LayerID
}
22 changes: 22 additions & 0 deletions prune/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package prune

import (
"github.com/prometheus/client_golang/prometheus"

"github.com/spacemeshos/go-spacemesh/metrics"
)

const namespace = "prune"

var (
pruneLatency = metrics.NewHistogramWithBuckets(
"prune_seconds",
namespace,
"prune time in seconds",
[]string{"step"},
prometheus.ExponentialBuckets(0.01, 2, 10),
)
proposalLatency = pruneLatency.WithLabelValues("proposal")
certLatency = pruneLatency.WithLabelValues("cert")
propTxLatency = pruneLatency.WithLabelValues("proptxs")
)
73 changes: 73 additions & 0 deletions prune/mocks.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

60 changes: 60 additions & 0 deletions prune/prune.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package prune

import (
"context"
"time"

"go.uber.org/zap"

"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/certificates"
"github.com/spacemeshos/go-spacemesh/sql/proposals"
"github.com/spacemeshos/go-spacemesh/sql/transactions"
)

func Prune(
ctx context.Context,
logger *zap.Logger,
db sql.Executor,
lc layerClock,
safeDist uint32,
interval time.Duration,
) {
logger.With().Info("db pruning launched",
zap.Uint32("dist", safeDist),
zap.Duration("interval", interval),
)
for {
select {
case <-ctx.Done():
return
case <-time.After(interval):
oldest := lc.CurrentLayer() - types.LayerID(safeDist)
t0 := time.Now()
if err := proposals.DeleteBefore(db, oldest); err != nil {
logger.Error("failed to delete proposals",
zap.Stringer("lid", oldest),
zap.Error(err),
)
}

Check warning on line 40 in prune/prune.go

View check run for this annotation

Codecov / codecov/patch

prune/prune.go#L36-L40

Added lines #L36 - L40 were not covered by tests
proposalLatency.Observe(time.Since(t0).Seconds())
t1 := time.Now()
if err := certificates.DeleteCertBefore(db, oldest); err != nil {
logger.Error("failed to delete certificates",
zap.Stringer("lid", oldest),
zap.Error(err),
)
}

Check warning on line 48 in prune/prune.go

View check run for this annotation

Codecov / codecov/patch

prune/prune.go#L44-L48

Added lines #L44 - L48 were not covered by tests
certLatency.Observe(time.Since(t1).Seconds())
t2 := time.Now()
if err := transactions.DeleteProposalTxsBefore(db, oldest); err != nil {
logger.Error("failed to delete proposal tx mapping",
zap.Stringer("lid", oldest),
zap.Error(err),
)
}

Check warning on line 56 in prune/prune.go

View check run for this annotation

Codecov / codecov/patch

prune/prune.go#L52-L56

Added lines #L52 - L56 were not covered by tests
propTxLatency.Observe(time.Since(t2).Seconds())
}
}
}
Loading

0 comments on commit 5a20c04

Please sign in to comment.