Skip to content

Commit

Permalink
sql: improve database schema handling
Browse files Browse the repository at this point in the history 
Use consistent package/folder structure for local.sql and state.sql
databases.

When a new database is created, use schema script instead of running
all the migrations. Also, check that there's no schema drift by
diffing database schema against the schema script after running
migrations.
  • Loading branch information
@ivan4th
ivan4th committed Jun 1, 2024
1 parent c21567c commit f33a283
Show file tree
Hide file tree
Showing 148 changed files with 1,354 additions and 724 deletions.
3 changes: 2 additions & 1 deletion activation/activation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/spacemeshos/go-spacemesh/sql/localsql"
"github.com/spacemeshos/go-spacemesh/sql/localsql/nipost"
sqlmocks "github.com/spacemeshos/go-spacemesh/sql/mocks"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

// ========== Vars / Consts ==========
Expand Down Expand Up @@ -78,7 +79,7 @@ func newTestBuilder(tb testing.TB, numSigners int, opts ...BuilderOption) *testA

ctrl := gomock.NewController(tb)
tab := &testAtxBuilder{
db: sql.InMemory(),
db: statesql.InMemory(),
localDb: localsql.InMemory(sql.WithConnections(numSigners)),
goldenATXID: types.ATXID(types.HexToHash32("77777")),

Expand Down
7 changes: 4 additions & 3 deletions activation/certifier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/spacemeshos/go-spacemesh/sql/localsql"
certdb "github.com/spacemeshos/go-spacemesh/sql/localsql/certifier"
"github.com/spacemeshos/go-spacemesh/sql/localsql/nipost"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func TestPersistsCerts(t *testing.T) {
Expand Down Expand Up @@ -113,15 +114,15 @@ func TestObtainingPost(t *testing.T) {
id := types.RandomNodeID()

t.Run("no POST or ATX", func(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
localDb := localsql.InMemory()

certifier := NewCertifierClient(db, localDb, zaptest.NewLogger(t))
_, err := certifier.obtainPost(context.Background(), id)
require.ErrorContains(t, err, "PoST not found")
})
t.Run("initial POST available", func(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
localDb := localsql.InMemory()

post := nipost.Post{
Expand All @@ -142,7 +143,7 @@ func TestObtainingPost(t *testing.T) {
require.Equal(t, post, *got)
})
t.Run("initial POST unavailable but ATX exists", func(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
localDb := localsql.InMemory()

atx := newInitialATXv1(t, types.RandomATXID())
Expand Down
4 changes: 2 additions & 2 deletions activation/e2e/activation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ import (
"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
"github.com/spacemeshos/go-spacemesh/p2p/pubsub/mocks"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/localsql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
"github.com/spacemeshos/go-spacemesh/timesync"
)

Expand All @@ -52,7 +52,7 @@ func Test_BuilderWithMultipleClients(t *testing.T) {
logger := zaptest.NewLogger(t)
goldenATX := types.ATXID{2, 3, 4}
cfg := activation.DefaultPostConfig()
db := sql.InMemory()
db := statesql.InMemory()
localDB := localsql.InMemory()

syncer := activation.NewMocksyncer(ctrl)
Expand Down
4 changes: 2 additions & 2 deletions activation/e2e/certifier_client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ import (
"github.com/spacemeshos/go-spacemesh/atxsdata"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/localsql"
"github.com/spacemeshos/go-spacemesh/sql/localsql/nipost"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func TestCertification(t *testing.T) {
Expand All @@ -36,7 +36,7 @@ func TestCertification(t *testing.T) {

logger := zaptest.NewLogger(t)
cfg := activation.DefaultPostConfig()
db := sql.InMemory()
db := statesql.InMemory()
localDb := localsql.InMemory()

syncer := activation.NewMocksyncer(gomock.NewController(t))
Expand Down
6 changes: 3 additions & 3 deletions activation/e2e/nipost_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ import (
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/datastore"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/localsql"
"github.com/spacemeshos/go-spacemesh/sql/localsql/nipost"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

const (
Expand Down Expand Up @@ -134,7 +134,7 @@ func TestNIPostBuilderWithClients(t *testing.T) {
logger := zaptest.NewLogger(t)
goldenATX := types.ATXID{2, 3, 4}
cfg := activation.DefaultPostConfig()
db := sql.InMemory()
db := statesql.InMemory()
cdb := datastore.NewCachedDB(db, logger)
localDb := localsql.InMemory()

Expand Down Expand Up @@ -263,7 +263,7 @@ func Test_NIPostBuilderWithMultipleClients(t *testing.T) {
logger := zaptest.NewLogger(t)
goldenATX := types.ATXID{2, 3, 4}
cfg := activation.DefaultPostConfig()
db := sql.InMemory()
db := statesql.InMemory()

syncer := activation.NewMocksyncer(ctrl)
syncer.EXPECT().RegisterForATXSynced().AnyTimes().DoAndReturn(func() <-chan struct{} {
Expand Down
6 changes: 3 additions & 3 deletions activation/e2e/validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
"github.com/spacemeshos/go-spacemesh/atxsdata"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/localsql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func TestValidator_Validate(t *testing.T) {
Expand All @@ -30,7 +30,7 @@ func TestValidator_Validate(t *testing.T) {
logger := zaptest.NewLogger(t)
goldenATX := types.ATXID{2, 3, 4}
cfg := activation.DefaultPostConfig()
db := sql.InMemory()
db := statesql.InMemory()

validator := activation.NewMocknipostValidator(gomock.NewController(t))
syncer := activation.NewMocksyncer(gomock.NewController(t))
Expand Down Expand Up @@ -67,7 +67,7 @@ func TestValidator_Validate(t *testing.T) {
WithPhaseShift(poetCfg.PhaseShift),
WithCycleGap(poetCfg.CycleGap),
)
poetDb := activation.NewPoetDb(sql.InMemory(), logger.Named("poetDb"))
poetDb := activation.NewPoetDb(statesql.InMemory(), logger.Named("poetDb"))
client, err := activation.NewPoetClient(
poetDb,
types.PoetServer{Address: poetProver.RestURL().String()},
Expand Down
3 changes: 2 additions & 1 deletion activation/handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/identities"
"github.com/spacemeshos/go-spacemesh/sql/localsql/nipost"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
"github.com/spacemeshos/go-spacemesh/system/mocks"
)

Expand Down Expand Up @@ -194,7 +195,7 @@ func newTestHandlerMocks(tb testing.TB, golden types.ATXID) handlerMocks {

func newTestHandler(tb testing.TB, goldenATXID types.ATXID, opts ...HandlerOption) *testHandler {
lg := zaptest.NewLogger(tb)
cdb := datastore.NewCachedDB(sql.InMemory(), lg)
cdb := datastore.NewCachedDB(statesql.InMemory(), lg)
edVerifier := signing.NewEdVerifier()

mocks := newTestHandlerMocks(tb, goldenATXID)
Expand Down
3 changes: 2 additions & 1 deletion activation/handler_v1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/identities"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

type v1TestHandler struct {
Expand All @@ -34,7 +35,7 @@ type v1TestHandler struct {

func newV1TestHandler(tb testing.TB, goldenATXID types.ATXID) *v1TestHandler {
lg := zaptest.NewLogger(tb)
cdb := datastore.NewCachedDB(sql.InMemory(), lg)
cdb := datastore.NewCachedDB(statesql.InMemory(), lg)
mocks := newTestHandlerMocks(tb, goldenATXID)
return &v1TestHandler{
HandlerV1: &HandlerV1{
Expand Down
3 changes: 2 additions & 1 deletion activation/handler_v2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

type v2TestHandler struct {
Expand All @@ -31,7 +32,7 @@ type v2TestHandler struct {

func newV2TestHandler(tb testing.TB, golden types.ATXID) *v2TestHandler {
lg := zaptest.NewLogger(tb)
cdb := datastore.NewCachedDB(sql.InMemory(), lg)
cdb := datastore.NewCachedDB(statesql.InMemory(), lg)
mocks := newTestHandlerMocks(tb, golden)
return &v2TestHandler{
HandlerV2: &HandlerV2{
Expand Down
5 changes: 3 additions & 2 deletions activation/poetdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,19 @@ import (
"github.com/spacemeshos/go-spacemesh/p2p"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/poets"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

var ErrObjectExists = sql.ErrObjectExists

// PoetDb is a database for PoET proofs.
type PoetDb struct {
sqlDB *sql.Database
sqlDB *statesql.Database
logger *zap.Logger
}

// NewPoetDb returns a new PoET handler.
func NewPoetDb(db *sql.Database, log *zap.Logger) *PoetDb {
func NewPoetDb(db *statesql.Database, log *zap.Logger) *PoetDb {
return &PoetDb{sqlDB: db, logger: log}
}

Expand Down
10 changes: 5 additions & 5 deletions activation/poetdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (

"github.com/spacemeshos/go-spacemesh/codec"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

var (
Expand Down Expand Up @@ -63,7 +63,7 @@ func getPoetProof(t *testing.T) types.PoetProofMessage {
func TestPoetDbHappyFlow(t *testing.T) {
r := require.New(t)
msg := getPoetProof(t)
poetDb := NewPoetDb(sql.InMemory(), zaptest.NewLogger(t))
poetDb := NewPoetDb(statesql.InMemory(), zaptest.NewLogger(t))

r.NoError(poetDb.Validate(msg.Statement[:], msg.PoetProof, msg.PoetServiceID, msg.RoundID, types.EmptyEdSignature))
ref, err := msg.Ref()
Expand All @@ -83,7 +83,7 @@ func TestPoetDbHappyFlow(t *testing.T) {
func TestPoetDbInvalidPoetProof(t *testing.T) {
r := require.New(t)
msg := getPoetProof(t)
poetDb := NewPoetDb(sql.InMemory(), zaptest.NewLogger(t))
poetDb := NewPoetDb(statesql.InMemory(), zaptest.NewLogger(t))
msg.PoetProof.Root = []byte("some other root")

err := poetDb.Validate(msg.Statement[:], msg.PoetProof, msg.PoetServiceID, msg.RoundID, types.EmptyEdSignature)
Expand All @@ -99,7 +99,7 @@ func TestPoetDbInvalidPoetProof(t *testing.T) {
func TestPoetDbInvalidPoetStatement(t *testing.T) {
r := require.New(t)
msg := getPoetProof(t)
poetDb := NewPoetDb(sql.InMemory(), zaptest.NewLogger(t))
poetDb := NewPoetDb(statesql.InMemory(), zaptest.NewLogger(t))
msg.Statement = types.CalcHash32([]byte("some other statement"))

err := poetDb.Validate(msg.Statement[:], msg.PoetProof, msg.PoetServiceID, msg.RoundID, types.EmptyEdSignature)
Expand All @@ -115,7 +115,7 @@ func TestPoetDbInvalidPoetStatement(t *testing.T) {
func TestPoetDbNonExistingKeys(t *testing.T) {
r := require.New(t)
msg := getPoetProof(t)
poetDb := NewPoetDb(sql.InMemory(), zaptest.NewLogger(t))
poetDb := NewPoetDb(statesql.InMemory(), zaptest.NewLogger(t))

_, err := poetDb.GetProofRef(msg.PoetServiceID, "0")
r.EqualError(
Expand Down
4 changes: 2 additions & 2 deletions activation/post_supervisor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"github.com/spacemeshos/go-spacemesh/atxsdata"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func closedChan() <-chan struct{} {
Expand Down Expand Up @@ -56,7 +56,7 @@ func newPostManager(t *testing.T, cfg PostConfig, opts PostSetupOpts) *PostSetup
close(ch)
return ch
})
db := sql.InMemory()
db := statesql.InMemory()
atxsdata := atxsdata.New()
mgr, err := NewPostSetupManager(cfg, zaptest.NewLogger(t), db, atxsdata, types.RandomATXID(), syncer, validator)
require.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions activation/post_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/datastore"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func TestPostSetupManager(t *testing.T) {
Expand Down Expand Up @@ -365,7 +365,7 @@ func newTestPostManager(tb testing.TB) *testPostManager {
syncer.EXPECT().RegisterForATXSynced().AnyTimes().Return(synced)

logger := zaptest.NewLogger(tb)
cdb := datastore.NewCachedDB(sql.InMemory(), logger)
cdb := datastore.NewCachedDB(statesql.InMemory(), logger)
mgr, err := NewPostSetupManager(DefaultPostConfig(), logger, cdb, atxsdata.New(), goldenATXID, syncer, validator)
require.NoError(tb, err)

Expand Down
6 changes: 3 additions & 3 deletions activation/validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
"github.com/spacemeshos/go-spacemesh/activation/wire"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func Test_Validation_VRFNonce(t *testing.T) {
Expand Down Expand Up @@ -476,7 +476,7 @@ func TestValidateMerkleProof(t *testing.T) {
}

func TestVerifyChainDeps(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
ctx := context.Background()
goldenATXID := types.ATXID{2, 3, 4}
signer, err := signing.NewEdSigner()
Expand Down Expand Up @@ -579,7 +579,7 @@ func TestVerifyChainDeps(t *testing.T) {
}

func TestVerifyChainDepsAfterCheckpoint(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
ctx := context.Background()
goldenATXID := types.ATXID{2, 3, 4}
signer, err := signing.NewEdSigner()
Expand Down
4 changes: 2 additions & 2 deletions activation/verify_state_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ import (
"github.com/spacemeshos/go-spacemesh/activation/wire"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
"github.com/spacemeshos/go-spacemesh/sql/identities"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func Test_CheckPrevATXs(t *testing.T) {
db := sql.InMemory()
db := statesql.InMemory()
logger := zaptest.NewLogger(t)

// Arrange
Expand Down
3 changes: 2 additions & 1 deletion api/grpcserver/activation_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/events"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

func Test_Highest_ReturnsGoldenAtxOnError(t *testing.T) {
Expand Down Expand Up @@ -137,7 +138,7 @@ func TestGet_IdentityCanceled(t *testing.T) {
atxProvider := grpcserver.NewMockatxProvider(ctrl)
activationService := grpcserver.NewActivationService(atxProvider, types.ATXID{1})

smesher, proof := grpcserver.BallotMalfeasance(t, sql.InMemory())
smesher, proof := grpcserver.BallotMalfeasance(t, statesql.InMemory())
id := types.RandomATXID()
atx := types.ActivationTx{
Sequence: rand.Uint64(),
Expand Down
6 changes: 3 additions & 3 deletions api/grpcserver/admin_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"github.com/spacemeshos/go-spacemesh/checkpoint"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/events"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/statesql"
)

const (
Expand All @@ -32,14 +32,14 @@ const (

// AdminService exposes endpoints for node administration.
type AdminService struct {
db *sql.Database
db *statesql.Database
dataDir string
recover func()
p peers
}

// NewAdminService creates a new admin grpc service.
func NewAdminService(db *sql.Database, dataDir string, p peers) *AdminService {
func NewAdminService(db *statesql.Database, dataDir string, p peers) *AdminService {
return &AdminService{
db: db,
dataDir: dataDir,
Expand Down
Loading

0 comments on commit f33a283

Please sign in to comment.