diff --git a/sql/database.go b/sql/database.go index df95109e6f5..732f9fd10f0 100644 --- a/sql/database.go +++ b/sql/database.go @@ -115,7 +115,18 @@ func Open(uri string, opts ...Opt) (*Database, error) { if config.enableLatency { db.latency = newQueryLatency() } + for i := 0; i < config.connections; i++ { + conn := pool.Get(context.Background()) + if err := registerFunctions(conn); err != nil { + return nil, err + } + pool.Put(conn) + } if config.migrations != nil { + before, err := version(db) + if err != nil { + return nil, err + } tx, err := db.Tx(context.Background()) if err != nil { return nil, err @@ -128,13 +139,11 @@ func Open(uri string, opts ...Opt) (*Database, error) { if err != nil { return nil, err } - } - for i := 0; i < config.connections; i++ { - conn := pool.Get(context.Background()) - if err := registerFunctions(conn); err != nil { - return nil, err + if before == 3 { + if err := Vacuum(db); err != nil { + return nil, err + } } - defer pool.Put(conn) } return db, nil } diff --git a/sql/functions.go b/sql/functions.go index 07f2c21c851..fb2ce92b241 100644 --- a/sql/functions.go +++ b/sql/functions.go @@ -4,6 +4,9 @@ import ( "fmt" sqlite "github.com/go-llsqlite/crawshaw" + + "github.com/spacemeshos/go-spacemesh/codec" + "github.com/spacemeshos/go-spacemesh/common/types" ) func registerFunctions(conn *sqlite.Conn) error { @@ -16,5 +19,21 @@ func registerFunctions(conn *sqlite.Conn) error { }, nil, nil); err != nil { return fmt.Errorf("registering add_uint64: %w", err) } + // function to prune active set from old ballots + if err := conn.CreateFunction("prune_actives", true, 1, func(ctx sqlite.Context, values ...sqlite.Value) { + var ballot types.Ballot + if err := codec.Decode(values[0].Blob(), &ballot); err != nil { + ctx.ResultError(err) + } else { + ballot.ActiveSet = nil + if blob, err := codec.Encode(&ballot); err != nil { + ctx.ResultError(err) + } else { + ctx.ResultBlob(blob) + } + } + }, nil, nil); err != nil { + return fmt.Errorf("registering prune_actives: %w", err) + } return nil } diff --git a/sql/migrations.go b/sql/migrations.go index c6bf08f9fe8..aeec91e0bda 100644 --- a/sql/migrations.go +++ b/sql/migrations.go @@ -23,6 +23,17 @@ type migration struct { // Migrations is interface for migrations provider. type Migrations func(Executor) error +func version(db Executor) (int, error) { + var current int + if _, err := db.Exec("PRAGMA user_version;", nil, func(stmt *Statement) bool { + current = stmt.ColumnInt(0) + return true + }); err != nil { + return 0, fmt.Errorf("read user_version %w", err) + } + return current, nil +} + func embeddedMigrations(db Executor) error { var migrations []migration fs.WalkDir(embedded, "migrations", func(path string, d fs.DirEntry, err error) error { @@ -62,13 +73,9 @@ func embeddedMigrations(db Executor) error { return migrations[i].order < migrations[j].order }) - var current int - - if _, err := db.Exec("PRAGMA user_version;", nil, func(stmt *Statement) bool { - current = stmt.ColumnInt(0) - return true - }); err != nil { - return fmt.Errorf("read user_version %w", err) + current, err := version(db) + if err != nil { + return err } for _, m := range migrations { diff --git a/sql/migrations/0003_next.sql b/sql/migrations/0003_v1.1.5.sql similarity index 100% rename from sql/migrations/0003_next.sql rename to sql/migrations/0003_v1.1.5.sql diff --git a/sql/migrations/0004_next.sql b/sql/migrations/0004_next.sql new file mode 100644 index 00000000000..f74ef85c5f7 --- /dev/null +++ b/sql/migrations/0004_next.sql @@ -0,0 +1,5 @@ +DELETE FROM proposals WHERE layer < 19000; +DELETE FROM proposal_transactions WHERE layer < 19000; +UPDATE certificates SET cert = NULL WHERE layer < 19000; +UPDATE ballots SET ballot = prune_actives(ballot); + diff --git a/sql/migrations_test.go b/sql/migrations_test.go index 8a0c637c467..2cb0de2b0f6 100644 --- a/sql/migrations_test.go +++ b/sql/migrations_test.go @@ -15,5 +15,5 @@ func TestMigrationsAppliedOnce(t *testing.T) { return true }) require.NoError(t, err) - require.Equal(t, version, 3) + require.Equal(t, version, 4) } diff --git a/sql/vacuum.go b/sql/vacuum.go new file mode 100644 index 00000000000..e53b7c234d4 --- /dev/null +++ b/sql/vacuum.go @@ -0,0 +1,15 @@ +package sql + +import ( + "fmt" +) + +func Vacuum(db Executor) error { + if _, err := db.Exec("vacuum", nil, nil); err != nil { + return fmt.Errorf("vacuum %w", err) + } + if _, err := db.Exec("pragma wal_checkpoint(TRUNCATE)", nil, nil); err != nil { + return fmt.Errorf("wal checkpoint %w", err) + } + return nil +} diff --git a/sql/vacuum/vacuum.go b/sql/vacuum/vacuum.go deleted file mode 100644 index 0b56a98db0e..00000000000 --- a/sql/vacuum/vacuum.go +++ /dev/null @@ -1,29 +0,0 @@ -package vacuum - -import ( - "fmt" - - "github.com/spacemeshos/go-spacemesh/sql" -) - -func VacuumDB(db sql.Executor) error { - if _, err := db.Exec("vacuum", nil, nil); err != nil { - return fmt.Errorf("vacuum %w", err) - } - if _, err := db.Exec("pragma wal_checkpoint(TRUNCATE)", nil, nil); err != nil { - return fmt.Errorf("wal checkpoint %w", err) - } - return nil -} - -func FreePct(db sql.Executor) (int, error) { - var pct int - if _, err := db.Exec(`SELECT freelist_count, page_count FROM pragma_freelist_count(), pragma_page_count();`, - nil, func(stmt *sql.Statement) bool { - pct = 100 * stmt.ColumnInt(0) / stmt.ColumnInt(1) - return false - }); err != nil { - return 0, fmt.Errorf("db page size %w", err) - } - return pct, nil -} diff --git a/sql/vacuum/vacuum_test.go b/sql/vacuum/vacuum_test.go deleted file mode 100644 index ef6071474eb..00000000000 --- a/sql/vacuum/vacuum_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package vacuum - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" - "github.com/spacemeshos/go-spacemesh/sql/ballots" - "github.com/spacemeshos/go-spacemesh/sql/proposals" -) - -func TestVacuumDB(t *testing.T) { - db := sql.InMemory() - require.NoError(t, VacuumDB(db)) -} - -func TestFreedPct(t *testing.T) { - db := sql.InMemory() - - got, err := FreePct(db) - require.NoError(t, err) - require.Equal(t, 0, got) - - for i := 0; i < 100; i++ { - ballot := types.NewExistingBallot(types.RandomBallotID(), types.RandomEdSignature(), types.NodeID{1}, types.LayerID(i)) - proposal := &types.Proposal{ - InnerProposal: types.InnerProposal{ - Ballot: ballot, - }, - Signature: types.RandomEdSignature(), - } - proposal.SetID(types.RandomProposalID()) - require.NoError(t, ballots.Add(db, &ballot)) - require.NoError(t, proposals.Add(db, proposal)) - } - - got, err = FreePct(db) - require.NoError(t, err) - require.Equal(t, 0, got) - - require.NoError(t, proposals.Delete(db, 80)) - got, err = FreePct(db) - require.NoError(t, err) - require.Greater(t, got, 10) -} diff --git a/sql/vacuum_test.go b/sql/vacuum_test.go new file mode 100644 index 00000000000..b994516279e --- /dev/null +++ b/sql/vacuum_test.go @@ -0,0 +1,12 @@ +package sql + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVacuumDB(t *testing.T) { + db := InMemory() + require.NoError(t, Vacuum(db)) +}