From ca2d5c6f46e48c4bba2b0702c541ded782a990a9 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 11 Nov 2024 12:52:51 +0400 Subject: [PATCH 01/22] sql: add Database.Connection/WithConnection, interface cleanup This adds a possibility to take a connection from the pool to use it via the Executor interface, and return it later when it's no longer needed. This avoids connection pool overhead in cases when a lot of quries need to be made, but the use of read transactions is not needed. Using read transactions instead of simple connections has the side effect of blocking WAL checkpoints. --- sql/database.go | 136 +++++++++++++++--- sql/database_test.go | 37 ++++- sql/interface.go | 5 +- sql/migrations.go | 5 - sql/mocks.go | 38 ----- sql/schema.go | 19 +-- .../migrations/state_0021_migration.go | 4 - .../migrations/state_0025_migration.go | 4 - 8 files changed, 163 insertions(+), 85 deletions(-) diff --git a/sql/database.go b/sql/database.go index 94a9322563..1620f633fc 100644 --- a/sql/database.go +++ b/sql/database.go @@ -572,24 +572,83 @@ type Interceptor func(query string) error type Database interface { Executor QueryCache + // Close closes the database. Close() error + // QueryCount returns the number of queries executed on the database. QueryCount() int + // QueryCache returns the query cache for this database, if it's present, + // or nil otherwise. QueryCache() QueryCache + // Tx creates deferred sqlite transaction. + // + // Deferred transactions are not started until the first statement. + // Transaction may be started in read mode and automatically upgraded to write mode + // after one of the write statements. + // + // https://www.sqlite.org/lang_transaction.html Tx(ctx context.Context) (Transaction, error) + // WithTx starts a new transaction and passes it to the exec function. + // It then commits the transaction if the exec function doesn't return an error, + // and rolls it back otherwise. + // If the context is canceled, the currently running SQL statement is interrupted. WithTx(ctx context.Context, exec func(Transaction) error) error + // TxImmediate begins a new immediate transaction on the database, that is, + // a transaction that starts a write immediately without waiting for a write + // statement. + // The transaction returned from this function must always be released by calling + // its Release method. Release rolls back the transaction if it hasn't been + // committed. + // If the context is canceled, the currently running SQL statement is interrupted. TxImmediate(ctx context.Context) (Transaction, error) + // WithTxImmediate starts a new immediate transaction and passes it to the exec + // function. + // An immediate transaction is started immediately, without waiting for a write + // statement. + // It then commits the transaction if the exec function doesn't return an error, + // and rolls it back otherwise. + // If the context is canceled, the currently running SQL statement is interrupted. WithTxImmediate(ctx context.Context, exec func(Transaction) error) error + // Connection returns a connection from the database pool. + // If many queries are to be executed in a row, but there's no need for an + // explicit transaction which may be long-running and thus block + // WAL checkpointing, it may be preferable to use a single connection for + // it to avoid database pool overhead. + // The connection needs to be always returned to the pool by calling its Release + // method. + // If the context is canceled, the currently running SQL statement is interrupted. + Connection(ctx context.Context) (Connection, error) + // WithConnection executes the provided function with a connection from the + // database pool. + // The connection is released back to the pool after the function returns. + // If the context is canceled, the currently running SQL statement is interrupted. + WithConnection(ctx context.Context, exec func(Connection) error) error + // Intercept adds an interceptor function to the database. The interceptor + // functions are invoked upon each query on the database, including queries + // executed within transactions. + // The query will fail if the interceptor returns an error. + // The interceptor can later be removed using RemoveInterceptor with the same key. Intercept(key string, fn Interceptor) + // RemoveInterceptor removes the interceptor function with specified key from the database. RemoveInterceptor(key string) } // Transaction represents a transaction. type Transaction interface { Executor + // Commit commits the transaction. Commit() error + // Release releases the transaction. If the transaction hasn't been committed, + // it's rolled back. Release() error } +// Connection represents a database connection. +type Connection interface { + Executor + // Release releases the connection back to the connection pool. + Release() +} + type sqliteDatabase struct { *queryCache pool *sqlitex.Pool @@ -684,34 +743,22 @@ func (db *sqliteDatabase) startExclusive() error { return nil } -// Tx creates deferred sqlite transaction. -// -// Deferred transactions are not started until the first statement. -// Transaction may be started in read mode and automatically upgraded to write mode -// after one of the write statements. -// -// https://www.sqlite.org/lang_transaction.html +// Tx implements Database. func (db *sqliteDatabase) Tx(ctx context.Context) (Transaction, error) { return db.getTx(ctx, beginDefault) } -// WithTx will pass initialized deferred transaction to exec callback. -// Will commit only if error is nil. +// WithTx implements Database. func (db *sqliteDatabase) WithTx(ctx context.Context, exec func(Transaction) error) error { return db.withTx(ctx, beginDefault, exec) } -// TxImmediate creates immediate transaction. -// -// IMMEDIATE cause the database connection to start a new write immediately, without waiting -// for a write statement. The BEGIN IMMEDIATE might fail with SQLITE_BUSY if another write -// transaction is already active on another database connection. +// TxImmediate implements Database. func (db *sqliteDatabase) TxImmediate(ctx context.Context) (Transaction, error) { return db.getTx(ctx, beginImmediate) } -// WithTxImmediate will pass initialized immediate transaction to exec callback. -// Will commit only if error is nil. +// WithTxImmediate implements Database. func (db *sqliteDatabase) WithTxImmediate(ctx context.Context, exec func(Transaction) error) error { return db.withTx(ctx, beginImmediate, exec) } @@ -727,7 +774,7 @@ func (db *sqliteDatabase) runInterceptors(query string) error { return nil } -// Exec statement using one of the connection from the pool. +// Exec implements Executor. // // If you care about atomicity of the operation (for example writing rewards to multiple accounts) // Tx should be used. Otherwise sqlite will not guarantee that all side-effects of operations are @@ -758,7 +805,7 @@ func (db *sqliteDatabase) Exec(query string, encoder Encoder, decoder Decoder) ( return exec(conn, query, encoder, decoder) } -// Close closes all pooled connections. +// Close implements Database. func (db *sqliteDatabase) Close() error { db.closeMux.Lock() defer db.closeMux.Unlock() @@ -772,6 +819,30 @@ func (db *sqliteDatabase) Close() error { return nil } +// Connection implements Database. +func (db *sqliteDatabase) Connection(ctx context.Context) (Connection, error) { + if db.closed { + return nil, ErrClosed + } + conCtx, cancel := context.WithCancel(ctx) + conn := db.getConn(conCtx) + if conn == nil { + cancel() + return nil, ErrNoConnection + } + return &sqliteConn{queryCache: db.queryCache, db: db, conn: conn, freeConn: cancel}, nil +} + +// WithConnection implements Database. +func (db *sqliteDatabase) WithConnection(ctx context.Context, exec func(Connection) error) error { + conn, err := db.Connection(ctx) + if err != nil { + return err + } + defer conn.Release() + return exec(conn) +} + // Intercept adds an interceptor function to the database. The interceptor functions // are invoked upon each query. The query will fail if the interceptor returns an error. // The interceptor can later be removed using RemoveInterceptor with the same key. @@ -1093,6 +1164,35 @@ func (tx *sqliteTx) Exec(query string, encoder Encoder, decoder Decoder) (int, e return exec(tx.conn, query, encoder, decoder) } +type sqliteConn struct { + *queryCache + db *sqliteDatabase + conn *sqlite.Conn + freeConn func() +} + +var _ Connection = &sqliteConn{} + +func (c *sqliteConn) Release() { + c.freeConn() + c.db.pool.Put(c.conn) +} + +func (c *sqliteConn) Exec(query string, encoder Encoder, decoder Decoder) (int, error) { + if err := c.db.runInterceptors(query); err != nil { + return 0, fmt.Errorf("running query interceptors: %w", err) + } + + c.db.queryCount.Add(1) + if c.db.latency != nil { + start := time.Now() + defer func() { + c.db.latency.WithLabelValues(query).Observe(float64(time.Since(start))) + }() + } + return exec(c.conn, query, encoder, decoder) +} + func mapSqliteError(err error) error { switch sqlite.ErrCode(err) { case sqlite.SQLITE_CONSTRAINT_PRIMARYKEY, sqlite.SQLITE_CONSTRAINT_UNIQUE: diff --git a/sql/database_test.go b/sql/database_test.go index d197d5e497..899ef2b493 100644 --- a/sql/database_test.go +++ b/sql/database_test.go @@ -93,8 +93,6 @@ func Test_Migration_Rollback(t *testing.T) { migration1.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(nil) migration2.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(errors.New("migration 2 failed")) - migration2.EXPECT().Rollback().Return(nil) - dbFile := filepath.Join(t.TempDir(), "test.sql") _, err := Open("file:"+dbFile, WithDatabaseSchema(&Schema{ @@ -129,7 +127,6 @@ func Test_Migration_Rollback_Only_NewMigrations(t *testing.T) { migration2.EXPECT().Name().Return("test").AnyTimes() migration2.EXPECT().Order().Return(2).AnyTimes() migration2.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(errors.New("migration 2 failed")) - migration2.EXPECT().Rollback().Return(nil) _, err = Open("file:"+dbFile, WithLogger(logger), @@ -638,3 +635,37 @@ func TestExclusive(t *testing.T) { }) } } + +func TestConnection(t *testing.T) { + db := InMemoryTest(t) + c, err := db.Connection(context.Background()) + require.NoError(t, err) + var r int + n, err := c.Exec("select ?", func(stmt *Statement) { + stmt.BindInt64(1, 42) + }, func(stmt *Statement) bool { + r = stmt.ColumnInt(0) + return true + }) + require.NoError(t, err) + require.Equal(t, 1, n) + require.Equal(t, 42, r) + c.Release() + + require.NoError(t, db.WithConnection(context.Background(), func(c Connection) error { + n, err := c.Exec("select ?", func(stmt *Statement) { + stmt.BindInt64(1, 42) + }, func(stmt *Statement) bool { + r = stmt.ColumnInt(0) + return true + }) + require.NoError(t, err) + require.Equal(t, 1, n) + require.Equal(t, 42, r) + return nil + })) + + require.Error(t, db.WithConnection(context.Background(), func(c Connection) error { + return errors.New("error") + })) +} diff --git a/sql/interface.go b/sql/interface.go index c9b0ee1441..14efae19c0 100644 --- a/sql/interface.go +++ b/sql/interface.go @@ -6,13 +6,16 @@ import "go.uber.org/zap" // Executor is an interface for executing raw statement. type Executor interface { + // Exec executes a statement. Exec(string, Encoder, Decoder) (int, error) } // Migration is interface for migrations provider. type Migration interface { + // Apply applies the migration. Apply(db Executor, logger *zap.Logger) error - Rollback() error + // Name returns the name of the migration. Name() string + // Order returns the sequential number of the migration. Order() int } diff --git a/sql/migrations.go b/sql/migrations.go index 92d1a1d516..b601f30685 100644 --- a/sql/migrations.go +++ b/sql/migrations.go @@ -89,11 +89,6 @@ func (m *sqlMigration) Order() int { return m.order } -func (sqlMigration) Rollback() error { - // handled by the DB itself - return nil -} - func version(db Executor) (int, error) { var current int if _, err := db.Exec("PRAGMA user_version;", nil, func(stmt *Statement) bool { diff --git a/sql/mocks.go b/sql/mocks.go index 2be336b646..ae5e3413e2 100644 --- a/sql/mocks.go +++ b/sql/mocks.go @@ -216,41 +216,3 @@ func (c *MockMigrationOrderCall) DoAndReturn(f func() int) *MockMigrationOrderCa c.Call = c.Call.DoAndReturn(f) return c } - -// Rollback mocks base method. -func (m *MockMigration) Rollback() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Rollback") - ret0, _ := ret[0].(error) - return ret0 -} - -// Rollback indicates an expected call of Rollback. -func (mr *MockMigrationMockRecorder) Rollback() *MockMigrationRollbackCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rollback", reflect.TypeOf((*MockMigration)(nil).Rollback)) - return &MockMigrationRollbackCall{Call: call} -} - -// MockMigrationRollbackCall wrap *gomock.Call -type MockMigrationRollbackCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockMigrationRollbackCall) Return(arg0 error) *MockMigrationRollbackCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockMigrationRollbackCall) Do(f func() error) *MockMigrationRollbackCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockMigrationRollbackCall) DoAndReturn(f func() error) *MockMigrationRollbackCall { - c.Call = c.Call.DoAndReturn(f) - return c -} diff --git a/sql/schema.go b/sql/schema.go index f393d7534f..144ad43eff 100644 --- a/sql/schema.go +++ b/sql/schema.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "errors" "fmt" "io" "os" @@ -31,14 +30,17 @@ func LoadDBSchemaScript(db Executor) (string, error) { return "", err } fmt.Fprintf(&sb, "PRAGMA user_version = %d;\n", version) + // The following SQL query ensures that tables are listed first, + // ordered by name, and then all other objects, ordered by their table name + // and then by their own name. if _, err = db.Exec(` SELECT tbl_name, sql || ';' FROM sqlite_master WHERE sql IS NOT NULL AND tbl_name NOT LIKE 'sqlite_%' ORDER BY - CASE WHEN type = 'table' THEN 1 ELSE 2 END, -- ensures tables are first - tbl_name, -- tables are sorted by name, then all other objects - name -- (indexes, triggers, etc.) also by name + CASE WHEN type = 'table' THEN 1 ELSE 2 END, + tbl_name, + name `, nil, func(st *Statement) bool { fmt.Fprintln(&sb, st.ColumnText(1)) return true @@ -143,20 +145,13 @@ func (s *Schema) Migrate(logger *zap.Logger, db Database, before, vacuumState in db.Intercept("logQueries", logQueryInterceptor(logger)) defer db.RemoveInterceptor("logQueries") } - for i, m := range s.Migrations { + for _, m := range s.Migrations { if m.Order() <= before { continue } if err := db.WithTxImmediate(context.Background(), func(tx Transaction) error { if _, ok := s.skipMigration[m.Order()]; !ok { if err := m.Apply(tx, logger); err != nil { - for j := i; j >= 0 && s.Migrations[j].Order() > before; j-- { - if e := s.Migrations[j].Rollback(); e != nil { - err = errors.Join(err, fmt.Errorf("rollback %s: %w", m.Name(), e)) - break - } - } - return fmt.Errorf("apply %s: %w", m.Name(), err) } } diff --git a/sql/statesql/migrations/state_0021_migration.go b/sql/statesql/migrations/state_0021_migration.go index b88471fbeb..874e558d8a 100644 --- a/sql/statesql/migrations/state_0021_migration.go +++ b/sql/statesql/migrations/state_0021_migration.go @@ -32,10 +32,6 @@ func (*migration0021) Order() int { return 21 } -func (*migration0021) Rollback() error { - return nil -} - func (m *migration0021) Apply(db sql.Executor, logger *zap.Logger) error { if err := m.applySql(db); err != nil { return err diff --git a/sql/statesql/migrations/state_0025_migration.go b/sql/statesql/migrations/state_0025_migration.go index 71fee844fa..c3869d74c6 100644 --- a/sql/statesql/migrations/state_0025_migration.go +++ b/sql/statesql/migrations/state_0025_migration.go @@ -40,10 +40,6 @@ func (*migration0025) Order() int { return 25 } -func (*migration0025) Rollback() error { - return nil -} - func (m *migration0025) Apply(db sql.Executor, logger *zap.Logger) error { updates := map[types.NodeID][]byte{} From 6e3aa1659baab8675b0c12418d504f504c856922 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 11 Nov 2024 13:12:45 +0400 Subject: [PATCH 02/22] sync2: dbset: use single connection for each sync session Using single connection for multiple SQL queries which are executed during sync avoids noticeable overhead due to SQLite connection pool delays. Also, this change fixes memory overuse in DBSet. When initializing DBSet from a database table, there's no need to use an FPTree with big preallocated pool for the new entries that are added during recent sync. --- sync2/dbset/dbset.go | 54 ++++++++++--- sync2/dbset/dbset_test.go | 121 +++++++++++++++++----------- sync2/dbset/p2p_test.go | 8 +- sync2/fptree/dbbackedstore.go | 3 +- sync2/fptree/dbbackedstore_test.go | 2 +- sync2/fptree/fptree_test.go | 2 +- sync2/multipeer/interface.go | 6 +- sync2/multipeer/mocks_test.go | 45 +++++------ sync2/multipeer/multipeer.go | 10 ++- sync2/multipeer/multipeer_test.go | 31 +++---- sync2/multipeer/setsyncbase.go | 27 ++++--- sync2/multipeer/setsyncbase_test.go | 48 ++++++----- sync2/multipeer/split_sync.go | 23 +++--- sync2/multipeer/split_sync_test.go | 6 +- sync2/p2p.go | 11 ++- sync2/p2p_test.go | 8 +- sync2/rangesync/dumbset.go | 16 ++-- sync2/rangesync/interface.go | 6 +- sync2/rangesync/mocks/mocks.go | 72 +++++++++++++---- 19 files changed, 316 insertions(+), 183 deletions(-) diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index 25c2101909..564c6afb7a 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -1,6 +1,7 @@ package dbset import ( + "context" "fmt" "maps" "sync" @@ -49,7 +50,16 @@ func (d *DBSet) handleIDfromDB(stmt *sql.Statement) bool { return true } +// Loaded returns true if the DBSet is loaded. +// Implements rangesync.OrderedSet. +func (d *DBSet) Loaded() bool { + d.loadMtx.Lock() + defer d.loadMtx.Unlock() + return d.ft != nil +} + // EnsureLoaded ensures that the DBSet is loaded and ready to be used. +// Implements rangesync.OrderedSet. func (d *DBSet) EnsureLoaded() error { d.loadMtx.Lock() defer d.loadMtx.Unlock() @@ -65,7 +75,7 @@ func (d *DBSet) EnsureLoaded() error { if err != nil { return fmt.Errorf("error loading count: %w", err) } - d.dbStore = fptree.NewDBBackedStore(d.db, d.snapshot, count, d.keyLen) + d.dbStore = fptree.NewDBBackedStore(d.db, d.snapshot, d.keyLen) d.ft = fptree.NewFPTree(count, d.dbStore, d.keyLen, d.maxDepth) return d.snapshot.Load(d.db, d.handleIDfromDB) } @@ -219,26 +229,39 @@ func (d *DBSet) Advance() error { // Copy creates a copy of the DBSet. // Implements rangesync.OrderedSet. -func (d *DBSet) Copy(syncScope bool) rangesync.OrderedSet { +func (d *DBSet) Copy(ctx context.Context, syncScope bool) (rangesync.OrderedSet, error) { + if err := d.EnsureLoaded(); err != nil { + return nil, fmt.Errorf("loading DBSet: %w", err) + } d.loadMtx.Lock() defer d.loadMtx.Unlock() - if d.ft == nil { - // FIXME - panic("BUG: can't copy the DBItemStore before it's loaded") - } ft := d.ft.Clone().(*fptree.FPTree) + ex := d.db + if syncScope { + db, ok := d.db.(sql.Database) + if ok { + // We might want to pass a real context here, but FPTree relies on + var err error + ex, err = db.Connection(context.Background()) + if err != nil { + return nil, fmt.Errorf("get connection: %w", err) + } + } + } return &DBSet{ - db: d.db, + db: ex, ft: ft, st: d.st, + snapshot: d.snapshot, keyLen: d.keyLen, maxDepth: d.maxDepth, dbStore: d.dbStore, received: maps.Clone(d.received), - } + }, nil } // Has returns true if the DBSet contains the given item. +// Implements rangesync.OrderedSet. func (d *DBSet) Has(k rangesync.KeyBytes) (bool, error) { if err := d.EnsureLoaded(); err != nil { return false, err @@ -258,17 +281,22 @@ func (d *DBSet) Has(k rangesync.KeyBytes) (bool, error) { } // Recent returns a sequence of items that have been added to the DBSet since the given time. +// Implements rangesync.OrderedSet. func (d *DBSet) Recent(since time.Time) (rangesync.SeqResult, int) { return d.dbStore.Since(make(rangesync.KeyBytes, d.keyLen), since.UnixNano()) } // Release releases resources associated with the DBSet. -func (d *DBSet) Release() error { +// Implements rangesync.OrderedSet. +func (d *DBSet) Release() { d.loadMtx.Lock() defer d.loadMtx.Unlock() - if d.ft != nil { - d.ft.Release() - d.ft = nil + if d.ft == nil { + return + } + d.ft.Release() + d.ft = nil + if c, ok := d.db.(sql.Connection); ok { + c.Release() } - return nil } diff --git a/sync2/dbset/dbset_test.go b/sync2/dbset/dbset_test.go index 23625d6444..6dd60582be 100644 --- a/sync2/dbset/dbset_test.go +++ b/sync2/dbset/dbset_test.go @@ -1,11 +1,13 @@ package dbset_test import ( + "context" "fmt" "testing" "github.com/stretchr/testify/require" + "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sync2/dbset" "github.com/spacemeshos/go-spacemesh/sync2/rangesync" "github.com/spacemeshos/go-spacemesh/sync2/sqlstore" @@ -220,7 +222,8 @@ func TestDBSet_Copy(t *testing.T) { require.Equal(t, "0000000000000000000000000000000000000000000000000000000000000000", firstKey(t, s.Items()).String()) - copy := s.Copy(false) + copy, err := s.Copy(context.Background(), false) + require.NoError(t, err) info, err := copy.GetRangeInfo(ids[2], ids[0]) require.NoError(t, err) @@ -259,64 +262,84 @@ func TestDBItemStore_Advance(t *testing.T) { rangesync.MustParseHexKeyBytes("5555555555555555555555555555555555555555555555555555555555555555"), rangesync.MustParseHexKeyBytes("8888888888888888888888888888888888888888888888888888888888888888"), } - db := sqlstore.PopulateDB(t, testKeyLen, ids) + st := &sqlstore.SyncedTable{ TableName: "foo", IDColumn: "id", } - s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() - require.NoError(t, s.EnsureLoaded()) - copy := s.Copy(false) + verifyDS := func(db sql.Database, os rangesync.OrderedSet) { + require.NoError(t, os.EnsureLoaded()) - info, err := s.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) - - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) - - sqlstore.InsertDBItems(t, db, []rangesync.KeyBytes{ - rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000"), - }) + copy, err := os.Copy(context.Background(), false) + require.NoError(t, err) - info, err = s.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err := os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - require.NoError(t, s.Advance()) + sqlstore.InsertDBItems(t, db, []rangesync.KeyBytes{ + rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000"), + }) - info, err = s.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 5, info.Count) - require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err = os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + + require.NoError(t, os.Advance()) + + info, err = os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 5, info.Count) + require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + + copy1, err := os.Copy(context.Background(), false) + require.NoError(t, err) + info, err = copy1.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 5, info.Count) + require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + } - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + t.Run("original DBSet", func(t *testing.T) { + db := sqlstore.PopulateDB(t, testKeyLen, ids) + dbSet := dbset.NewDBSet(db, st, testKeyLen, testDepth) + defer dbSet.Release() + verifyDS(db, dbSet) + }) - info, err = s.Copy(false).GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 5, info.Count) - require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + t.Run("DBSet copy", func(t *testing.T) { + db := sqlstore.PopulateDB(t, testKeyLen, ids) + origSet := dbset.NewDBSet(db, st, testKeyLen, testDepth) + defer origSet.Release() + os, err := origSet.Copy(context.Background(), false) + require.NoError(t, err) + verifyDS(db, os) + }) } func TestDBSet_Added(t *testing.T) { @@ -353,7 +376,9 @@ func TestDBSet_Added(t *testing.T) { rangesync.MustParseHexKeyBytes("4444444444444444444444444444444444444444444444444444444444444444"), }, added) - added1, err := s.Copy(false).(*dbset.DBSet).Received().FirstN(3) + copy, err := s.Copy(context.Background(), false) + require.NoError(t, err) + added1, err := copy.(*dbset.DBSet).Received().FirstN(3) require.NoError(t, err) require.ElementsMatch(t, added, added1) } diff --git a/sync2/dbset/p2p_test.go b/sync2/dbset/p2p_test.go index 9321f0fcc4..fc5f9842b0 100644 --- a/sync2/dbset/p2p_test.go +++ b/sync2/dbset/p2p_test.go @@ -185,7 +185,9 @@ func runSync( cfg.MaxReconcDiff = 1 // always reconcile pssA := rangesync.NewPairwiseSetSyncerInternal(syncLogger.Named("sideA"), nil, "test", cfg, &tr, clock) d := rangesync.NewDispatcher(log) - syncSetA := setA.Copy(false).(*dbset.DBSet) + copyA, err := setA.Copy(context.Background(), false) + require.NoError(t, err) + syncSetA := copyA.(*dbset.DBSet) pssA.Register(d, syncSetA) srv := server.New(mesh.Hosts()[0], proto, d.Dispatch, @@ -223,7 +225,9 @@ func runSync( pssB := rangesync.NewPairwiseSetSyncerInternal(syncLogger.Named("sideB"), client, "test", cfg, &tr, clock) tStart := time.Now() - syncSetB := setB.Copy(false).(*dbset.DBSet) + copyB, err := setB.Copy(context.Background(), false) + require.NoError(t, err) + syncSetB := copyB.(*dbset.DBSet) require.NoError(t, pssB.Sync(ctx, srvPeerID, syncSetB, x, x)) stopTimer(t) t.Logf("synced in %v, sent %d, recv %d", time.Since(tStart), pssB.Sent(), pssB.Received()) diff --git a/sync2/fptree/dbbackedstore.go b/sync2/fptree/dbbackedstore.go index d0a3e6e72f..fe987f945b 100644 --- a/sync2/fptree/dbbackedstore.go +++ b/sync2/fptree/dbbackedstore.go @@ -21,12 +21,11 @@ var _ sqlstore.IDStore = &DBBackedStore{} func NewDBBackedStore( db sql.Executor, sts *sqlstore.SyncedTableSnapshot, - sizeHint int, keyLen int, ) *DBBackedStore { return &DBBackedStore{ SQLIDStore: sqlstore.NewSQLIDStore(db, sts, keyLen), - FPTree: NewFPTreeWithValues(sizeHint, keyLen), + FPTree: NewFPTreeWithValues(0, keyLen), } } diff --git a/sync2/fptree/dbbackedstore_test.go b/sync2/fptree/dbbackedstore_test.go index 5c76c40e69..d18844b0be 100644 --- a/sync2/fptree/dbbackedstore_test.go +++ b/sync2/fptree/dbbackedstore_test.go @@ -55,7 +55,7 @@ func TestDBBackedStore(t *testing.T) { sts, err := st.Snapshot(db) require.NoError(t, err) - store := NewDBBackedStore(db, sts, 0, keyLen) + store := NewDBBackedStore(db, sts, keyLen) actualIDs, err := store.From(rangesync.KeyBytes{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5).FirstN(5) require.NoError(t, err) require.Equal(t, []rangesync.KeyBytes{ diff --git a/sync2/fptree/fptree_test.go b/sync2/fptree/fptree_test.go index 6136715d49..87e3a9f3e0 100644 --- a/sync2/fptree/fptree_test.go +++ b/sync2/fptree/fptree_test.go @@ -635,7 +635,7 @@ func makeDBBackedFPTree(t *testing.T) []*fptree.FPTree { t.Cleanup(func() { tx.Release() }) sts, err := st.Snapshot(tx) require.NoError(t, err) - store := fptree.NewDBBackedStore(tx, sts, 0, testKeyLen) + store := fptree.NewDBBackedStore(tx, sts, testKeyLen) ft := fptree.NewFPTree(0, store, testKeyLen, testDepth) return []*fptree.FPTree{ft, store.FPTree} } diff --git a/sync2/multipeer/interface.go b/sync2/multipeer/interface.go index ef8dc6f444..6ecbed2135 100644 --- a/sync2/multipeer/interface.go +++ b/sync2/multipeer/interface.go @@ -18,7 +18,7 @@ type SyncBase interface { // Count returns the number of items in the set. Count() (int, error) // Derive creates a Syncer for the specified peer. - Derive(p p2p.Peer) PeerSyncer + Derive(ctx context.Context, p p2p.Peer) (PeerSyncer, error) // Probe probes the specified peer, obtaining its set fingerprint, // the number of items and the similarity value. Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) @@ -36,7 +36,7 @@ type PeerSyncer interface { Serve(ctx context.Context, stream io.ReadWriter) error // Release releases the resources associated with the syncer. // Calling Release on a syncer that is already released is a no-op. - Release() error + Release() } // SyncKeyHandler is a handler for keys that are received from peers. @@ -44,7 +44,7 @@ type SyncKeyHandler interface { // Receive handles a key that was received from a peer. Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) // Commit is invoked at the end of synchronization to apply the changes. - Commit(peer p2p.Peer, base, new rangesync.OrderedSet) error + Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error } // PairwiseSyncer is used to probe a peer or sync against a single peer. diff --git a/sync2/multipeer/mocks_test.go b/sync2/multipeer/mocks_test.go index ffe4c90d6b..b9e3c79fd9 100644 --- a/sync2/multipeer/mocks_test.go +++ b/sync2/multipeer/mocks_test.go @@ -84,17 +84,18 @@ func (c *MockSyncBaseCountCall) DoAndReturn(f func() (int, error)) *MockSyncBase } // Derive mocks base method. -func (m *MockSyncBase) Derive(p p2p.Peer) multipeer.PeerSyncer { +func (m *MockSyncBase) Derive(ctx context.Context, p p2p.Peer) (multipeer.PeerSyncer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Derive", p) + ret := m.ctrl.Call(m, "Derive", ctx, p) ret0, _ := ret[0].(multipeer.PeerSyncer) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // Derive indicates an expected call of Derive. -func (mr *MockSyncBaseMockRecorder) Derive(p any) *MockSyncBaseDeriveCall { +func (mr *MockSyncBaseMockRecorder) Derive(ctx, p any) *MockSyncBaseDeriveCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Derive", reflect.TypeOf((*MockSyncBase)(nil).Derive), p) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Derive", reflect.TypeOf((*MockSyncBase)(nil).Derive), ctx, p) return &MockSyncBaseDeriveCall{Call: call} } @@ -104,19 +105,19 @@ type MockSyncBaseDeriveCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockSyncBaseDeriveCall) Return(arg0 multipeer.PeerSyncer) *MockSyncBaseDeriveCall { - c.Call = c.Call.Return(arg0) +func (c *MockSyncBaseDeriveCall) Return(arg0 multipeer.PeerSyncer, arg1 error) *MockSyncBaseDeriveCall { + c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockSyncBaseDeriveCall) Do(f func(p2p.Peer) multipeer.PeerSyncer) *MockSyncBaseDeriveCall { +func (c *MockSyncBaseDeriveCall) Do(f func(context.Context, p2p.Peer) (multipeer.PeerSyncer, error)) *MockSyncBaseDeriveCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncBaseDeriveCall) DoAndReturn(f func(p2p.Peer) multipeer.PeerSyncer) *MockSyncBaseDeriveCall { +func (c *MockSyncBaseDeriveCall) DoAndReturn(f func(context.Context, p2p.Peer) (multipeer.PeerSyncer, error)) *MockSyncBaseDeriveCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -261,11 +262,9 @@ func (c *MockPeerSyncerPeerCall) DoAndReturn(f func() p2p.Peer) *MockPeerSyncerP } // Release mocks base method. -func (m *MockPeerSyncer) Release() error { +func (m *MockPeerSyncer) Release() { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Release") - ret0, _ := ret[0].(error) - return ret0 + m.ctrl.Call(m, "Release") } // Release indicates an expected call of Release. @@ -281,19 +280,19 @@ type MockPeerSyncerReleaseCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockPeerSyncerReleaseCall) Return(arg0 error) *MockPeerSyncerReleaseCall { - c.Call = c.Call.Return(arg0) +func (c *MockPeerSyncerReleaseCall) Return() *MockPeerSyncerReleaseCall { + c.Call = c.Call.Return() return c } // Do rewrite *gomock.Call.Do -func (c *MockPeerSyncerReleaseCall) Do(f func() error) *MockPeerSyncerReleaseCall { +func (c *MockPeerSyncerReleaseCall) Do(f func()) *MockPeerSyncerReleaseCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockPeerSyncerReleaseCall) DoAndReturn(f func() error) *MockPeerSyncerReleaseCall { +func (c *MockPeerSyncerReleaseCall) DoAndReturn(f func()) *MockPeerSyncerReleaseCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -399,17 +398,17 @@ func (m *MockSyncKeyHandler) EXPECT() *MockSyncKeyHandlerMockRecorder { } // Commit mocks base method. -func (m *MockSyncKeyHandler) Commit(peer p2p.Peer, base, new rangesync.OrderedSet) error { +func (m *MockSyncKeyHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Commit", peer, base, new) + ret := m.ctrl.Call(m, "Commit", ctx, peer, base, new) ret0, _ := ret[0].(error) return ret0 } // Commit indicates an expected call of Commit. -func (mr *MockSyncKeyHandlerMockRecorder) Commit(peer, base, new any) *MockSyncKeyHandlerCommitCall { +func (mr *MockSyncKeyHandlerMockRecorder) Commit(ctx, peer, base, new any) *MockSyncKeyHandlerCommitCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockSyncKeyHandler)(nil).Commit), peer, base, new) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockSyncKeyHandler)(nil).Commit), ctx, peer, base, new) return &MockSyncKeyHandlerCommitCall{Call: call} } @@ -425,13 +424,13 @@ func (c *MockSyncKeyHandlerCommitCall) Return(arg0 error) *MockSyncKeyHandlerCom } // Do rewrite *gomock.Call.Do -func (c *MockSyncKeyHandlerCommitCall) Do(f func(p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { +func (c *MockSyncKeyHandlerCommitCall) Do(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncKeyHandlerCommitCall) DoAndReturn(f func(p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { +func (c *MockSyncKeyHandlerCommitCall) DoAndReturn(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/sync2/multipeer/multipeer.go b/sync2/multipeer/multipeer.go index 3b35a057ee..46b1d7cf48 100644 --- a/sync2/multipeer/multipeer.go +++ b/sync2/multipeer/multipeer.go @@ -3,6 +3,7 @@ package multipeer import ( "context" "errors" + "fmt" "math" "time" @@ -260,21 +261,24 @@ func (mpr *MultiPeerReconciler) needSplitSync(s syncability) bool { func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Peer) error { var eg errgroup.Group for _, p := range syncPeers { - syncer := mpr.syncBase.Derive(p) + syncer, err := mpr.syncBase.Derive(ctx, p) + if err != nil { + return fmt.Errorf("derive syncer: %w", err) + } eg.Go(func() error { + defer syncer.Release() err := syncer.Sync(ctx, nil, nil) switch { case err == nil: mpr.sl.NoteSync() case errors.Is(err, context.Canceled): - syncer.Release() return err default: // failing to sync against a particular peer is not considered // a fatal sync failure, so we just log the error mpr.logger.Error("error syncing peer", zap.Stringer("peer", p), zap.Error(err)) } - return syncer.Release() + return nil }) } return eg.Wait() diff --git a/sync2/multipeer/multipeer_test.go b/sync2/multipeer/multipeer_test.go index ab0a516839..353fa705fd 100644 --- a/sync2/multipeer/multipeer_test.go +++ b/sync2/multipeer/multipeer_test.go @@ -148,21 +148,22 @@ func (mt *multiPeerSyncTester) expectFullSync(pl *peerList, times, numFails int) // delegate to the real fullsync return mt.reconciler.FullSync(ctx, peers) }) - mt.syncBase.EXPECT().Derive(gomock.Any()).DoAndReturn(func(p p2p.Peer) multipeer.PeerSyncer { - mt.mtx.Lock() - defer mt.mtx.Unlock() - require.Contains(mt, pl.get(), p) - s := NewMockPeerSyncer(mt.ctrl) - s.EXPECT().Peer().Return(p).AnyTimes() - // TODO: do better job at tracking Release() calls - s.EXPECT().Release().AnyTimes() - expSync := s.EXPECT().Sync(gomock.Any(), gomock.Nil(), gomock.Nil()) - if numFails != 0 { - expSync.Return(errors.New("sync failed")) - numFails-- - } - return s - }).Times(times) + mt.syncBase.EXPECT().Derive(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, p p2p.Peer) (multipeer.PeerSyncer, error) { + mt.mtx.Lock() + defer mt.mtx.Unlock() + require.Contains(mt, pl.get(), p) + s := NewMockPeerSyncer(mt.ctrl) + s.EXPECT().Peer().Return(p).AnyTimes() + // TODO: do better job at tracking Release() calls + s.EXPECT().Release().AnyTimes() + expSync := s.EXPECT().Sync(gomock.Any(), gomock.Nil(), gomock.Nil()) + if numFails != 0 { + expSync.Return(errors.New("sync failed")) + numFails-- + } + return s, nil + }).Times(times) } // satisfy waits until all the expected mocked calls are made. diff --git a/sync2/multipeer/setsyncbase.go b/sync2/multipeer/setsyncbase.go index d3b5bface2..c20e4f1b86 100644 --- a/sync2/multipeer/setsyncbase.go +++ b/sync2/multipeer/setsyncbase.go @@ -67,30 +67,37 @@ func (ssb *SetSyncBase) Count() (int, error) { } // Derive implements SyncBase. -func (ssb *SetSyncBase) Derive(p p2p.Peer) PeerSyncer { +func (ssb *SetSyncBase) Derive(ctx context.Context, p p2p.Peer) (PeerSyncer, error) { ssb.mtx.Lock() defer ssb.mtx.Unlock() + os, err := ssb.os.Copy(ctx, true) + if err != nil { + return nil, fmt.Errorf("copy set: %w", err) + } return &peerSetSyncer{ SetSyncBase: ssb, - OrderedSet: ssb.os.Copy(true), + OrderedSet: os, p: p, handler: ssb.handler, - } + }, nil } // Probe implements SyncBase. func (ssb *SetSyncBase) Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) { // Use a snapshot of the store to avoid holding the mutex for a long time ssb.mtx.Lock() - os := ssb.os.Copy(true) + os, err := ssb.os.Copy(ctx, true) + defer os.Release() ssb.mtx.Unlock() + if err != nil { + return rangesync.ProbeResult{}, fmt.Errorf("copy set: %w", err) + } pr, err := ssb.ps.Probe(ctx, p, os, nil, nil) if err != nil { - os.Release() return rangesync.ProbeResult{}, fmt.Errorf("probing peer %s: %w", p, err) } - return pr, os.Release() + return pr, nil } func (ssb *SetSyncBase) receiveKey(k rangesync.KeyBytes, p p2p.Peer) error { @@ -170,7 +177,7 @@ func (pss *peerSetSyncer) Sync(ctx context.Context, x, y rangesync.KeyBytes) err if err := pss.ps.Sync(ctx, pss.p, pss, x, y); err != nil { return err } - return pss.commit() + return pss.commit(ctx) } // Serve implements Syncer. @@ -178,7 +185,7 @@ func (pss *peerSetSyncer) Serve(ctx context.Context, stream io.ReadWriter) error if err := pss.ps.Serve(ctx, stream, pss); err != nil { return err } - return pss.commit() + return pss.commit(ctx) } // Receive implements OrderedSet. @@ -189,8 +196,8 @@ func (pss *peerSetSyncer) Receive(k rangesync.KeyBytes) error { return pss.OrderedSet.Receive(k) } -func (pss *peerSetSyncer) commit() error { - if err := pss.handler.Commit(pss.p, pss.SetSyncBase.os, pss.OrderedSet); err != nil { +func (pss *peerSetSyncer) commit(ctx context.Context) error { + if err := pss.handler.Commit(ctx, pss.p, pss.SetSyncBase.os, pss.OrderedSet); err != nil { return err } return pss.SetSyncBase.advance() diff --git a/sync2/multipeer/setsyncbase_test.go b/sync2/multipeer/setsyncbase_test.go index cc0e5e84b1..5b00aa8eba 100644 --- a/sync2/multipeer/setsyncbase_test.go +++ b/sync2/multipeer/setsyncbase_test.go @@ -69,17 +69,18 @@ func (st *setSyncBaseTester) getWaitCh(k rangesync.KeyBytes) chan error { func (st *setSyncBaseTester) expectCopy(addedKeys ...rangesync.KeyBytes) *mocks.MockOrderedSet { copy := mocks.NewMockOrderedSet(st.ctrl) - st.os.EXPECT().Copy(true).DoAndReturn(func(bool) rangesync.OrderedSet { - copy.EXPECT().Items().DoAndReturn(func() rangesync.SeqResult { - return rangesync.EmptySeqResult() - }).AnyTimes() - for _, k := range addedKeys { - copy.EXPECT().Receive(k) - } - // TODO: do better job at tracking Release() calls - copy.EXPECT().Release().AnyTimes() - return copy - }) + st.os.EXPECT().Copy(gomock.Any(), true).DoAndReturn( + func(context.Context, bool) (rangesync.OrderedSet, error) { + copy.EXPECT().Items().DoAndReturn(func() rangesync.SeqResult { + return rangesync.EmptySeqResult() + }).AnyTimes() + for _, k := range addedKeys { + copy.EXPECT().Receive(k) + } + // TODO: do better job at tracking Release() calls + copy.EXPECT().Release().AnyTimes() + return copy, nil + }) return copy } @@ -138,12 +139,13 @@ func TestSetSyncBase(t *testing.T) { addedKey := rangesync.RandomKeyBytes(32) st.expectCopy(addedKey) - ss := st.ssb.Derive(p2p.Peer("p1")) + ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) + require.NoError(t, err) require.Equal(t, p2p.Peer("p1"), ss.Peer()) x := rangesync.RandomKeyBytes(32) y := rangesync.RandomKeyBytes(32) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any()) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) st.os.EXPECT().Advance() st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), ss, x, y) require.NoError(t, ss.Sync(context.Background(), x, y)) @@ -151,7 +153,7 @@ func TestSetSyncBase(t *testing.T) { st.os.EXPECT().Has(addedKey) st.os.EXPECT().Receive(addedKey) st.expectSync(p2p.Peer("p1"), ss, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any()) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) st.os.EXPECT().Advance() require.NoError(t, ss.Sync(context.Background(), nil, nil)) close(st.getWaitCh(addedKey)) @@ -167,7 +169,8 @@ func TestSetSyncBase(t *testing.T) { addedKey := rangesync.RandomKeyBytes(32) st.expectCopy(addedKey, addedKey, addedKey) - ss := st.ssb.Derive(p2p.Peer("p1")) + ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) + require.NoError(t, err) require.Equal(t, p2p.Peer("p1"), ss.Peer()) // added just once @@ -175,7 +178,7 @@ func TestSetSyncBase(t *testing.T) { for i := 0; i < 3; i++ { st.os.EXPECT().Has(addedKey) st.expectSync(p2p.Peer("p1"), ss, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any()) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) st.os.EXPECT().Advance() require.NoError(t, ss.Sync(context.Background(), nil, nil)) } @@ -193,7 +196,8 @@ func TestSetSyncBase(t *testing.T) { k1 := rangesync.RandomKeyBytes(32) k2 := rangesync.RandomKeyBytes(32) st.expectCopy(k1, k2) - ss := st.ssb.Derive(p2p.Peer("p1")) + ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) + require.NoError(t, err) require.Equal(t, p2p.Peer("p1"), ss.Peer()) st.os.EXPECT().Has(k1) @@ -201,7 +205,7 @@ func TestSetSyncBase(t *testing.T) { st.os.EXPECT().Receive(k1) st.os.EXPECT().Receive(k2) st.expectSync(p2p.Peer("p1"), ss, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any()) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) st.os.EXPECT().Advance() require.NoError(t, ss.Sync(context.Background(), nil, nil)) close(st.getWaitCh(k1)) @@ -219,7 +223,8 @@ func TestSetSyncBase(t *testing.T) { k1 := rangesync.RandomKeyBytes(32) k2 := rangesync.RandomKeyBytes(32) st.expectCopy(k1, k2) - ss := st.ssb.Derive(p2p.Peer("p1")) + ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) + require.NoError(t, err) require.Equal(t, p2p.Peer("p1"), ss.Peer()) st.os.EXPECT().Has(k1) @@ -227,7 +232,7 @@ func TestSetSyncBase(t *testing.T) { // k1 is not propagated to syncBase due to the handler failure st.os.EXPECT().Receive(k2) st.expectSync(p2p.Peer("p1"), ss, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any()) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) st.os.EXPECT().Advance() require.NoError(t, ss.Sync(context.Background(), nil, nil)) st.getWaitCh(k1) <- errors.New("fail") @@ -248,7 +253,8 @@ func TestSetSyncBase(t *testing.T) { os.AddUnchecked(hs[0]) os.AddUnchecked(hs[1]) st := newSetSyncBaseTester(t, &os) - ss := st.ssb.Derive(p2p.Peer("p1")) + ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) + require.NoError(t, err) ss.(rangesync.OrderedSet).Receive(hs[2]) ss.(rangesync.OrderedSet).Add(hs[2]) ss.(rangesync.OrderedSet).Receive(hs[3]) diff --git a/sync2/multipeer/split_sync.go b/sync2/multipeer/split_sync.go index 7bfbe8394b..b084c30e0d 100644 --- a/sync2/multipeer/split_sync.go +++ b/sync2/multipeer/split_sync.go @@ -3,6 +3,7 @@ package multipeer import ( "context" "errors" + "fmt" "slices" "time" @@ -80,22 +81,23 @@ func (s *splitSync) nextPeer() p2p.Peer { return p } -func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange) { - syncer := s.syncBase.Derive(p) +func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange) error { + syncer, err := s.syncBase.Derive(ctx, p) + if err != nil { + return fmt.Errorf("derive syncer: %w", err) + } sr.NumSyncers++ s.numRunning++ doneCh := make(chan struct{}) s.eg.Go(func() error { - defer func() { - syncer.Release() - close(doneCh) - }() + defer syncer.Release() err := syncer.Sync(ctx, sr.X, sr.Y) + close(doneCh) select { case <-ctx.Done(): return ctx.Err() case s.resCh <- syncResult{s: syncer, err: err}: - return syncer.Release() + return nil } }) gpTimer := s.clock.After(s.gracePeriod) @@ -115,6 +117,7 @@ func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange } return nil }) + return nil } func (s *splitSync) handleSyncResult(r syncResult) error { @@ -184,13 +187,15 @@ func (s *splitSync) Sync(ctx context.Context) error { } p := s.nextPeer() s.syncMap[p] = sr - s.startPeerSync(syncCtx, p, sr) + if err := s.startPeerSync(syncCtx, p, sr); err != nil { + return err + } break } s.clearDeadPeers() for s.numRemaining > 0 && (s.sq.empty() || len(s.syncPeers) == 0) { if s.numRunning == 0 && len(s.syncPeers) == 0 { - return errors.New("all peers dropped before full sync has completed") + return errors.New("all peers dropped before split sync has completed") } select { case sr = <-s.slowRangeCh: diff --git a/sync2/multipeer/split_sync_test.go b/sync2/multipeer/split_sync_test.go index 7a8e584009..195cf45561 100644 --- a/sync2/multipeer/split_sync_test.go +++ b/sync2/multipeer/split_sync_test.go @@ -74,8 +74,8 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { } for index, p := range tst.syncPeers { tst.syncBase.EXPECT(). - Derive(p). - DoAndReturn(func(peer p2p.Peer) multipeer.PeerSyncer { + Derive(gomock.Any(), p). + DoAndReturn(func(_ context.Context, peer p2p.Peer) (multipeer.PeerSyncer, error) { s := NewMockPeerSyncer(ctrl) s.EXPECT().Peer().Return(p).AnyTimes() // TODO: do better job at tracking Release() calls @@ -103,7 +103,7 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { } return nil }) - return s + return s, nil }). AnyTimes() } diff --git a/sync2/p2p.go b/sync2/p2p.go index d36d7546fd..4310adff2c 100644 --- a/sync2/p2p.go +++ b/sync2/p2p.go @@ -80,7 +80,16 @@ func NewP2PHashSync( func (s *P2PHashSync) serve(ctx context.Context, peer p2p.Peer, stream io.ReadWriter) error { // We derive a dedicated Syncer for the peer being served to pass all the received // items through the handler before adding them to the main ItemStore - return s.syncBase.Derive(peer).Serve(ctx, stream) + syncer, err := s.syncBase.Derive(ctx, peer) + if err != nil { + return fmt.Errorf("derive syncer: %w", err) + } + defer syncer.Release() + if err := syncer.Serve(ctx, stream); err != nil { + syncer.Release() + return err + } + return nil } // Set returns the OrderedSet that is being synchronized. diff --git a/sync2/p2p_test.go b/sync2/p2p_test.go index a36e4ebf27..54b63c4a60 100644 --- a/sync2/p2p_test.go +++ b/sync2/p2p_test.go @@ -46,7 +46,7 @@ func (fh *fakeHandler) Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error return true, nil } -func (fh *fakeHandler) Commit(peer p2p.Peer, base, new rangesync.OrderedSet) error { +func (fh *fakeHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { fh.mtx.Lock() defer fh.mtx.Unlock() for k := range fh.synced { @@ -128,7 +128,8 @@ func TestP2P(t *testing.T) { if !hsync.Synced() { return false } - os := hsync.Set().Copy(false) + os, err := hsync.Set().Copy(context.Background(), false) + require.NoError(t, err) for _, k := range handlers[n].committedItems() { os.(*rangesync.DumbSet).AddUnchecked(k) } @@ -150,7 +151,8 @@ func TestP2P(t *testing.T) { for n, hsync := range hs { hsync.Stop() - os := hsync.Set().Copy(false) + os, err := hsync.Set().Copy(context.Background(), false) + require.NoError(t, err) for _, k := range handlers[n].committedItems() { os.(*rangesync.DumbSet).AddUnchecked(k) } diff --git a/sync2/rangesync/dumbset.go b/sync2/rangesync/dumbset.go index 6a1f6134c2..ac4e734312 100644 --- a/sync2/rangesync/dumbset.go +++ b/sync2/rangesync/dumbset.go @@ -1,6 +1,7 @@ package rangesync import ( + "context" "crypto/md5" "errors" "slices" @@ -307,10 +308,10 @@ func (ds *DumbSet) Items() SeqResult { } // Copy implements OrderedSet. -func (ds *DumbSet) Copy(syncScope bool) OrderedSet { +func (ds *DumbSet) Copy(_ context.Context, syncScope bool) (OrderedSet, error) { return &DumbSet{ keys: slices.Clone(ds.keys), - } + }, nil } // Recent implements OrderedSet. @@ -318,7 +319,12 @@ func (ds *DumbSet) Recent(since time.Time) (SeqResult, int) { return EmptySeqResult(), 0 } -// Advance implements OrderedSet. +// Loaded implements OrderedSet. +func (ds *DumbSet) Loaded() bool { + return true +} + +// EnsureLoaded implements OrderedSet. func (ds *DumbSet) EnsureLoaded() error { return nil } @@ -346,6 +352,4 @@ func (ds *DumbSet) Has(k KeyBytes) (bool, error) { } // Release implements OrderedSet. -func (ds *DumbSet) Release() error { - return nil -} +func (ds *DumbSet) Release() {} diff --git a/sync2/rangesync/interface.go b/sync2/rangesync/interface.go index 7770ab246d..12d42ae2c7 100644 --- a/sync2/rangesync/interface.go +++ b/sync2/rangesync/interface.go @@ -64,11 +64,13 @@ type OrderedSet interface { // a synchronization run. // If syncScope if false, then the lifetime of the copy is not clearly defined. // The list of received items as returned by Received is also inherited by the copy. - Copy(syncScope bool) OrderedSet + Copy(ctx context.Context, syncScope bool) (OrderedSet, error) // Recent returns an Iterator that yields the items added since the specified // timestamp. Some OrderedSet implementations may not have Recent implemented, in // which case it should return an empty sequence. Recent(since time.Time) (SeqResult, int) + // Loaded returns true if the set is loaded and ready for use. + Loaded() bool // EnsureLoaded ensures that the set is loaded and ready for use. // It may do nothing in case of in-memory sets, but may trigger loading // from database in case of database-backed sets. @@ -80,7 +82,7 @@ type OrderedSet interface { Has(KeyBytes) (bool, error) // Release releases the resources associated with the set. // Calling Release on a set that is already released is a no-op. - Release() error + Release() } type Requester interface { diff --git a/sync2/rangesync/mocks/mocks.go b/sync2/rangesync/mocks/mocks.go index 280edddb27..481ea093ec 100644 --- a/sync2/rangesync/mocks/mocks.go +++ b/sync2/rangesync/mocks/mocks.go @@ -10,6 +10,7 @@ package mocks import ( + context "context" reflect "reflect" time "time" @@ -118,17 +119,18 @@ func (c *MockOrderedSetAdvanceCall) DoAndReturn(f func() error) *MockOrderedSetA } // Copy mocks base method. -func (m *MockOrderedSet) Copy(syncScope bool) rangesync.OrderedSet { +func (m *MockOrderedSet) Copy(ctx context.Context, syncScope bool) (rangesync.OrderedSet, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Copy", syncScope) + ret := m.ctrl.Call(m, "Copy", ctx, syncScope) ret0, _ := ret[0].(rangesync.OrderedSet) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // Copy indicates an expected call of Copy. -func (mr *MockOrderedSetMockRecorder) Copy(syncScope any) *MockOrderedSetCopyCall { +func (mr *MockOrderedSetMockRecorder) Copy(ctx, syncScope any) *MockOrderedSetCopyCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockOrderedSet)(nil).Copy), syncScope) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockOrderedSet)(nil).Copy), ctx, syncScope) return &MockOrderedSetCopyCall{Call: call} } @@ -138,19 +140,19 @@ type MockOrderedSetCopyCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetCopyCall) Return(arg0 rangesync.OrderedSet) *MockOrderedSetCopyCall { - c.Call = c.Call.Return(arg0) +func (c *MockOrderedSetCopyCall) Return(arg0 rangesync.OrderedSet, arg1 error) *MockOrderedSetCopyCall { + c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetCopyCall) Do(f func(bool) rangesync.OrderedSet) *MockOrderedSetCopyCall { +func (c *MockOrderedSetCopyCall) Do(f func(context.Context, bool) (rangesync.OrderedSet, error)) *MockOrderedSetCopyCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetCopyCall) DoAndReturn(f func(bool) rangesync.OrderedSet) *MockOrderedSetCopyCall { +func (c *MockOrderedSetCopyCall) DoAndReturn(f func(context.Context, bool) (rangesync.OrderedSet, error)) *MockOrderedSetCopyCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -348,6 +350,44 @@ func (c *MockOrderedSetItemsCall) DoAndReturn(f func() rangesync.SeqResult) *Moc return c } +// Loaded mocks base method. +func (m *MockOrderedSet) Loaded() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Loaded") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Loaded indicates an expected call of Loaded. +func (mr *MockOrderedSetMockRecorder) Loaded() *MockOrderedSetLoadedCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Loaded", reflect.TypeOf((*MockOrderedSet)(nil).Loaded)) + return &MockOrderedSetLoadedCall{Call: call} +} + +// MockOrderedSetLoadedCall wrap *gomock.Call +type MockOrderedSetLoadedCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockOrderedSetLoadedCall) Return(arg0 bool) *MockOrderedSetLoadedCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockOrderedSetLoadedCall) Do(f func() bool) *MockOrderedSetLoadedCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockOrderedSetLoadedCall) DoAndReturn(f func() bool) *MockOrderedSetLoadedCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Receive mocks base method. func (m *MockOrderedSet) Receive(k rangesync.KeyBytes) error { m.ctrl.T.Helper() @@ -464,11 +504,9 @@ func (c *MockOrderedSetRecentCall) DoAndReturn(f func(time.Time) (rangesync.SeqR } // Release mocks base method. -func (m *MockOrderedSet) Release() error { +func (m *MockOrderedSet) Release() { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Release") - ret0, _ := ret[0].(error) - return ret0 + m.ctrl.Call(m, "Release") } // Release indicates an expected call of Release. @@ -484,19 +522,19 @@ type MockOrderedSetReleaseCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetReleaseCall) Return(arg0 error) *MockOrderedSetReleaseCall { - c.Call = c.Call.Return(arg0) +func (c *MockOrderedSetReleaseCall) Return() *MockOrderedSetReleaseCall { + c.Call = c.Call.Return() return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetReleaseCall) Do(f func() error) *MockOrderedSetReleaseCall { +func (c *MockOrderedSetReleaseCall) Do(f func()) *MockOrderedSetReleaseCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetReleaseCall) DoAndReturn(f func() error) *MockOrderedSetReleaseCall { +func (c *MockOrderedSetReleaseCall) DoAndReturn(f func()) *MockOrderedSetReleaseCall { c.Call = c.Call.DoAndReturn(f) return c } From 4828b6794fa73e1024b85edf1da3f093ed6978e5 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 11 Nov 2024 13:37:39 +0400 Subject: [PATCH 03/22] sync2: ATX integration This adds set reconciliation for ATXs. There are per-epoch syncers, with lower FPTree depth (16 by default) used for older epochs and greater FPTree depth (21 by default) used for current epoch. Both active syncv2 and passive (server-only) syncv2 are disabled by default. It is possible to enable syncv2 in server-only or full (active) mode. --- config/mainnet.go | 14 ++ config/presets/testnet.go | 13 + fetch/fetch.go | 9 + fetch/mesh_data.go | 16 +- fetch/mesh_data_test.go | 21 +- sync2/atxs.go | 336 +++++++++++++++++++++++++ sync2/atxs_test.go | 414 +++++++++++++++++++++++++++++++ sync2/interface.go | 36 +++ sync2/mocks_test.go | 498 ++++++++++++++++++++++++++++++++++++++ syncer/interface.go | 5 + syncer/mocks/mocks.go | 99 ++++++++ syncer/syncer.go | 172 +++++++++++-- syncer/syncer_test.go | 156 ++++++++++-- system/fetcher.go | 10 + 14 files changed, 1749 insertions(+), 50 deletions(-) create mode 100644 sync2/atxs.go create mode 100644 sync2/atxs_test.go create mode 100644 sync2/interface.go create mode 100644 sync2/mocks_test.go diff --git a/config/mainnet.go b/config/mainnet.go index 65312d2877..58d0a77cac 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -24,6 +24,7 @@ import ( "github.com/spacemeshos/go-spacemesh/hare4" "github.com/spacemeshos/go-spacemesh/miner" "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/syncer" "github.com/spacemeshos/go-spacemesh/syncer/atxsync" "github.com/spacemeshos/go-spacemesh/syncer/malsync" @@ -77,6 +78,14 @@ func MainnetConfig() Config { hare4conf := hare4.DefaultConfig() hare4conf.Enable = false + + oldAtxSyncCfg := sync2.DefaultConfig() + oldAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = time.Hour + oldAtxSyncCfg.MaxDepth = 16 + newAtxSyncCfg := sync2.DefaultConfig() + newAtxSyncCfg.MaxDepth = 21 + newAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = 5 * time.Minute + return Config{ BaseConfig: BaseConfig{ DataDirParent: defaultDataDir, @@ -212,6 +221,11 @@ func MainnetConfig() Config { DisableMeshAgreement: true, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), + V2: syncer.SyncV2Config{ + OldAtxSyncCfg: oldAtxSyncCfg, + NewAtxSyncCfg: newAtxSyncCfg, + ParallelLoadLimit: 10, + }, }, Recovery: checkpoint.DefaultConfig(), Cache: datastore.DefaultConfig(), diff --git a/config/presets/testnet.go b/config/presets/testnet.go index bb066c352e..713cea25d8 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -25,6 +25,7 @@ import ( "github.com/spacemeshos/go-spacemesh/hare4" "github.com/spacemeshos/go-spacemesh/miner" "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/syncer" "github.com/spacemeshos/go-spacemesh/syncer/atxsync" "github.com/spacemeshos/go-spacemesh/syncer/malsync" @@ -65,6 +66,13 @@ func testnet() config.Config { hare4conf := hare4.DefaultConfig() hare4conf.Enable = false defaultdir := filepath.Join(home, "spacemesh-testnet", "/") + + oldAtxSyncCfg := sync2.DefaultConfig() + oldAtxSyncCfg.MaxDepth = 16 + newAtxSyncCfg := sync2.DefaultConfig() + newAtxSyncCfg.MaxDepth = 21 + newAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = 5 * time.Minute + return config.Config{ Preset: "testnet", BaseConfig: config.BaseConfig{ @@ -163,6 +171,11 @@ func testnet() config.Config { OutOfSyncThresholdLayers: 10, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), + V2: syncer.SyncV2Config{ + OldAtxSyncCfg: oldAtxSyncCfg, + NewAtxSyncCfg: newAtxSyncCfg, + ParallelLoadLimit: 10, + }, }, Recovery: checkpoint.DefaultConfig(), Cache: datastore.DefaultConfig(), diff --git a/fetch/fetch.go b/fetch/fetch.go index 396342a4bb..abee69cbdb 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -11,6 +11,7 @@ import ( "sync" "time" + corehost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -1013,3 +1014,11 @@ func (f *Fetch) SelectBestShuffled(n int) []p2p.Peer { }) return peers } + +func (f *Fetch) Host() corehost.Host { + return f.host.(corehost.Host) +} + +func (f *Fetch) Peers() *peers.Peers { + return f.peers +} diff --git a/fetch/mesh_data.go b/fetch/mesh_data.go index ad271f307b..28382808ba 100644 --- a/fetch/mesh_data.go +++ b/fetch/mesh_data.go @@ -30,7 +30,7 @@ func (f *Fetch) GetAtxs(ctx context.Context, ids []types.ATXID, opts ...system.G return nil } - options := system.GetAtxOpts{} + var options system.GetAtxOpts for _, opt := range opts { opt(&options) } @@ -41,10 +41,20 @@ func (f *Fetch) GetAtxs(ctx context.Context, ids []types.ATXID, opts ...system.G zap.Bool("limiting", !options.LimitingOff), ) hashes := types.ATXIDsToHashes(ids) + handler := f.validators.atx.HandleMessage + if options.RecvChannel != nil { + handler = func(ctx context.Context, id types.Hash32, p p2p.Peer, data []byte) error { + if err := f.validators.atx.HandleMessage(ctx, id, p, data); err != nil { + return err + } + options.RecvChannel <- types.ATXID(id) + return nil + } + } if options.LimitingOff { - return f.getHashes(ctx, hashes, datastore.ATXDB, f.validators.atx.HandleMessage) + return f.getHashes(ctx, hashes, datastore.ATXDB, handler) } - return f.getHashes(ctx, hashes, datastore.ATXDB, f.validators.atx.HandleMessage, withLimiter(f.getAtxsLimiter)) + return f.getHashes(ctx, hashes, datastore.ATXDB, handler, withLimiter(f.getAtxsLimiter)) } type dataReceiver func(context.Context, types.Hash32, p2p.Peer, []byte) error diff --git a/fetch/mesh_data_test.go b/fetch/mesh_data_test.go index 713c96a7b7..56b1b83e91 100644 --- a/fetch/mesh_data_test.go +++ b/fetch/mesh_data_test.go @@ -86,7 +86,7 @@ func startTestLoop(tb testing.TB, f *Fetch, eg *errgroup.Group, stop chan struct default: f.mu.Lock() for h, req := range f.unprocessed { - require.NoError(tb, req.validator(req.ctx, types.Hash32{}, p2p.NoPeer, []byte{})) + require.NoError(tb, req.validator(req.ctx, h, p2p.NoPeer, []byte{})) close(req.promise.completed) delete(f.unprocessed, h) } @@ -591,7 +591,7 @@ func genATXs(tb testing.TB, num uint32) []*types.ActivationTx { } func TestGetATXs(t *testing.T) { - atxs := genATXs(t, 2) + atxs := genATXs(t, 4) f := createFetch(t) f.mAtxH.EXPECT(). HandleMessage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). @@ -602,10 +602,23 @@ func TestGetATXs(t *testing.T) { var eg errgroup.Group startTestLoop(t, f.Fetch, &eg, stop) - atxIDs := types.ToATXIDs(atxs) - require.NoError(t, f.GetAtxs(context.Background(), atxIDs)) + atxIDs1 := types.ToATXIDs(atxs[:2]) + require.NoError(t, f.GetAtxs(context.Background(), atxIDs1)) + + recvCh := make(chan types.ATXID) + atxIDs2 := types.ToATXIDs(atxs[2:]) + var recvIDs []types.ATXID + eg.Go(func() error { + for id := range recvCh { + recvIDs = append(recvIDs, id) + } + return nil + }) + require.NoError(t, f.GetAtxs(context.Background(), atxIDs2, system.WithRecvChannel(recvCh))) + close(recvCh) close(stop) require.NoError(t, eg.Wait()) + require.ElementsMatch(t, atxIDs2, recvIDs) } func TestGetActiveSet(t *testing.T) { diff --git a/sync2/atxs.go b/sync2/atxs.go new file mode 100644 index 0000000000..11feba1399 --- /dev/null +++ b/sync2/atxs.go @@ -0,0 +1,336 @@ +package sync2 + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jonboulle/clockwork" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/p2p/pubsub" + "github.com/spacemeshos/go-spacemesh/p2p/server" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/expr" + "github.com/spacemeshos/go-spacemesh/sync2/dbset" + "github.com/spacemeshos/go-spacemesh/sync2/multipeer" + "github.com/spacemeshos/go-spacemesh/sync2/rangesync" + "github.com/spacemeshos/go-spacemesh/sync2/sqlstore" + "github.com/spacemeshos/go-spacemesh/system" +) + +const ( + proto = "sync/2" +) + +type ATXHandler struct { + logger *zap.Logger + f Fetcher + clock clockwork.Clock + batchSize int + maxAttempts int + maxBatchRetries int + failedBatchDelay time.Duration +} + +var _ multipeer.SyncKeyHandler = &ATXHandler{} + +func NewATXHandler( + logger *zap.Logger, + f Fetcher, + batchSize, maxAttempts, maxBatchRetries int, + failedBatchDelay time.Duration, + clock clockwork.Clock, +) *ATXHandler { + if clock == nil { + clock = clockwork.NewRealClock() + } + return &ATXHandler{ + f: f, + logger: logger, + clock: clock, + batchSize: batchSize, + maxAttempts: maxAttempts, + maxBatchRetries: maxBatchRetries, + failedBatchDelay: failedBatchDelay, + } +} + +func (h *ATXHandler) Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) { + var id types.ATXID + copy(id[:], k) + h.f.RegisterPeerHash(peer, id.Hash32()) + return false, nil +} + +func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { + h.logger.Debug("begin atx commit") + defer h.logger.Debug("end atx commit") + sr := new.Received() + var firstK rangesync.KeyBytes + numDownloaded := 0 + state := make(map[types.ATXID]int) + for k := range sr.Seq { + if firstK == nil { + firstK = k + } else if firstK.Compare(k) == 0 { + break + } + found, err := base.Has(k) + if err != nil { + return fmt.Errorf("check if ATX exists: %w", err) + } + if found { + continue + } + state[types.BytesToATXID(k)] = 0 + } + if err := sr.Error(); err != nil { + return fmt.Errorf("get item: %w", err) + } + total := len(state) + items := make([]types.ATXID, 0, h.batchSize) + startTime := time.Now() + batchAttemptsRemaining := h.maxBatchRetries + for len(state) > 0 { + items = items[:0] + for id, n := range state { + if n >= h.maxAttempts { + h.logger.Debug("failed to download ATX: max attempts reached", + zap.String("atx", id.ShortString())) + delete(state, id) + continue + } + items = append(items, id) + if len(items) == h.batchSize { + break + } + } + if len(items) == 0 { + break + } + + var eg errgroup.Group + recvCh := make(chan types.ATXID) + someSucceeded := false + eg.Go(func() error { + for id := range recvCh { + numDownloaded++ + someSucceeded = true + delete(state, id) + } + return nil + }) + err := h.f.GetAtxs(ctx, items, system.WithRecvChannel(recvCh)) + close(recvCh) + eg.Wait() + if err != nil { + if errors.Is(err, context.Canceled) { + return err + } + batchError := &fetch.BatchError{} + if errors.As(err, &batchError) { + h.logger.Debug("QQQQQ: batch error", zap.Error(err)) + for hash, err := range batchError.Errors { + if _, exists := state[types.ATXID(hash)]; !exists { + continue + } + if errors.Is(err, pubsub.ErrValidationReject) { + // if the atx invalid there's no point downloading it again + state[types.ATXID(hash)] = h.maxAttempts + } else { + state[types.ATXID(hash)]++ + } + } + } else { + h.logger.Debug("failed to download ATXs", zap.Error(err)) + } + } + if !someSucceeded { + if batchAttemptsRemaining == 0 { + return errors.New("failed to download ATXs: max batch retries reached") + } + batchAttemptsRemaining-- + h.logger.Debug("failed to download any ATXs: will retry batch", + zap.Int("remaining", batchAttemptsRemaining), + zap.Duration("delay", h.failedBatchDelay)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-h.clock.After(h.failedBatchDelay): + } + } else { + batchAttemptsRemaining = h.maxBatchRetries + h.logger.Debug("fetched atxs", + zap.Int("total", total), + zap.Int("downloaded", numDownloaded), + zap.Float64("rate per sec", float64(numDownloaded)/time.Since(startTime).Seconds())) + } + } + return nil +} + +type MultiEpochATXSyncer struct { + logger *zap.Logger + oldCfg Config + newCfg Config + parallelLoadLimit int + hss HashSyncSource + newEpoch types.EpochID + atxSyncers []HashSync +} + +func NewMultiEpochATXSyncer( + logger *zap.Logger, + hss HashSyncSource, + oldCfg, newCfg Config, + parallelLoadLimit int, +) *MultiEpochATXSyncer { + return &MultiEpochATXSyncer{ + logger: logger, + oldCfg: oldCfg, + newCfg: newCfg, + parallelLoadLimit: parallelLoadLimit, + hss: hss, + } +} + +func (s *MultiEpochATXSyncer) load(newEpoch types.EpochID) error { + if len(s.atxSyncers) < int(newEpoch) { + s.atxSyncers = append(s.atxSyncers, make([]HashSync, int(newEpoch)-len(s.atxSyncers))...) + } + s.newEpoch = newEpoch + var eg errgroup.Group + if s.parallelLoadLimit > 0 { + eg.SetLimit(s.parallelLoadLimit) + } + for epoch := types.EpochID(1); epoch <= newEpoch; epoch++ { + if s.atxSyncers[epoch-1] != nil { + continue + } + eg.Go(func() error { + name := fmt.Sprintf("atx-sync-%d", epoch) + cfg := s.oldCfg + if epoch == newEpoch { + cfg = s.newCfg + } + hs := s.hss.CreateHashSync(name, cfg, epoch) + if err := hs.Load(); err != nil { + return fmt.Errorf("load ATX syncer for epoch %d: %w", epoch, err) + } + s.atxSyncers[epoch-1] = hs + return nil + }) + } + return eg.Wait() +} + +// EnsureSync ensures that ATX sync is active for all the epochs up to and including +// currentEpoch, and that all ATXs are +// synced up to and including lastWaitEpoch. +// If newEpoch argument is non-zero, faster but less memory efficient sync is used for +// that epoch, based on the newCfg (larger maxDepth). +// For other epochs, oldCfg is used which corresponds to slower but more memory efficient +// sync (smaller maxDepth). +// It returns the last epoch that was synced synchronously. +func (s *MultiEpochATXSyncer) EnsureSync( + ctx context.Context, + lastWaitEpoch, newEpoch types.EpochID, +) (lastSynced types.EpochID, err error) { + if newEpoch != s.newEpoch && int(s.newEpoch) <= len(s.atxSyncers) && s.newEpoch > 0 { + s.atxSyncers[s.newEpoch-1].Stop() + s.atxSyncers[s.newEpoch-1] = nil + } + if err := s.load(newEpoch); err != nil { + return lastSynced, err + } + for epoch := types.EpochID(1); epoch <= newEpoch; epoch++ { + syncer := s.atxSyncers[epoch-1] + if epoch <= lastWaitEpoch { + s.logger.Info("waiting for epoch to sync", zap.Uint32("epoch", epoch.Uint32())) + if err := syncer.StartAndSync(ctx); err != nil { + return lastSynced, fmt.Errorf("error syncing old ATXs: %w", err) + } + lastSynced = epoch + } else { + syncer.Start() + } + } + return lastSynced, nil +} + +// Stop stops all ATX syncers. +func (s *MultiEpochATXSyncer) Stop() { + for _, hs := range s.atxSyncers { + hs.Stop() + } + s.atxSyncers = nil + s.newEpoch = 0 +} + +func atxsTable(epoch types.EpochID) *sqlstore.SyncedTable { + return &sqlstore.SyncedTable{ + TableName: "atxs", + IDColumn: "id", + TimestampColumn: "received", + Filter: expr.MustParse("epoch = ?"), + Binder: func(s *sql.Statement) { + s.BindInt64(1, int64(epoch)) + }, + } +} + +func NewATXSyncer( + logger *zap.Logger, + d *rangesync.Dispatcher, + name string, + cfg Config, + db sql.StateDatabase, + f Fetcher, + epoch types.EpochID, + enableActiveSync bool, +) *P2PHashSync { + curSet := dbset.NewDBSet(db, atxsTable(epoch), 32, cfg.MaxDepth) + return NewP2PHashSync( + logger, d, name, curSet, 32, f.Peers(), + NewATXHandler( + logger, f, cfg.BatchSize, cfg.MaxAttempts, + cfg.MaxBatchRetries, cfg.FailedBatchDelay, nil), + cfg, enableActiveSync) +} + +func NewDispatcher(logger *zap.Logger, f Fetcher) *rangesync.Dispatcher { + d := rangesync.NewDispatcher(logger) + d.SetupServer(f.Host(), proto, server.WithHardTimeout(20*time.Minute)) + return d +} + +type ATXSyncSource struct { + logger *zap.Logger + d *rangesync.Dispatcher + db sql.StateDatabase + f Fetcher + enableActiveSync bool +} + +var _ HashSyncSource = &ATXSyncSource{} + +func NewATXSyncSource( + logger *zap.Logger, + d *rangesync.Dispatcher, + db sql.StateDatabase, + f Fetcher, + enableActiveSync bool, +) *ATXSyncSource { + return &ATXSyncSource{logger: logger, d: d, db: db, f: f, enableActiveSync: enableActiveSync} +} + +// CreateHashSync implements HashSyncSource. +func (as *ATXSyncSource) CreateHashSync(name string, cfg Config, epoch types.EpochID) HashSync { + return NewATXSyncer(as.logger.Named(name), as.d, name, cfg, as.db, as.f, epoch, as.enableActiveSync) +} diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go new file mode 100644 index 0000000000..7dcc780ebe --- /dev/null +++ b/sync2/atxs_test.go @@ -0,0 +1,414 @@ +package sync2_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + gomock "go.uber.org/mock/gomock" + "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/p2p/pubsub" + "github.com/spacemeshos/go-spacemesh/sync2" + "github.com/spacemeshos/go-spacemesh/sync2/rangesync" + "github.com/spacemeshos/go-spacemesh/sync2/rangesync/mocks" + "github.com/spacemeshos/go-spacemesh/system" +) + +func TestAtxHandler_Success(t *testing.T) { + const ( + batchSize = 4 + maxAttempts = 3 + maxBatchRetries = 2 + batchRetryDelay = 10 * time.Second + ) + ctrl := gomock.NewController(t) + allAtxs := make([]types.ATXID, 10) + logger := zaptest.NewLogger(t) + peer := p2p.Peer("foobar") + for i := range allAtxs { + allAtxs[i] = types.RandomATXID() + } + f := NewMockFetcher(ctrl) + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + baseSet := mocks.NewMockOrderedSet(ctrl) + newSet := mocks.NewMockOrderedSet(ctrl) + for _, id := range allAtxs { + f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) + add, err := h.Receive(id.Bytes(), peer) + require.False(t, add) + require.NoError(t, err) + } + toFetch := make(map[types.ATXID]bool) + for _, id := range allAtxs { + toFetch[id] = true + } + var batches []int + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + batches = append(batches, len(atxs)) + var atxOpts system.GetAtxOpts + for _, opt := range opts { + opt(&atxOpts) + } + require.NotNil(t, atxOpts.RecvChannel) + for _, id := range atxs { + require.True(t, toFetch[id], "already fetched or bad ID") + delete(toFetch, id) + select { + case <-time.After(100 * time.Millisecond): + t.Error("timeout sending recvd id") + case atxOpts.RecvChannel <- id: + } + } + return nil + }).Times(3) + newSet.EXPECT().Received().Return(rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + // Received sequence may be cyclic and the handler should stop + // when it sees the first key again. + for { + for _, atx := range allAtxs { + if !yield(atx.Bytes()) { + return + } + } + } + }, + Error: rangesync.NoSeqError, + }) + require.NoError(t, h.Commit(context.Background(), peer, baseSet, newSet)) + require.Empty(t, toFetch) + require.Equal(t, []int{4, 4, 2}, batches) +} + +func TestAtxHandler_Retry(t *testing.T) { + const ( + batchSize = 4 + maxAttempts = 3 + maxBatchRetries = 2 + batchRetryDelay = 10 * time.Second + ) + ctrl := gomock.NewController(t) + allAtxs := make([]types.ATXID, 10) + logger := zaptest.NewLogger(t) + peer := p2p.Peer("foobar") + for i := range allAtxs { + allAtxs[i] = types.RandomATXID() + } + f := NewMockFetcher(ctrl) + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + baseSet := mocks.NewMockOrderedSet(ctrl) + newSet := mocks.NewMockOrderedSet(ctrl) + for _, id := range allAtxs { + f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) + add, err := h.Receive(id.Bytes(), peer) + require.False(t, add) + require.NoError(t, err) + } + failCount := 0 + var fetched []types.ATXID + validationFailed := false + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + errs := make(map[types.Hash32]error) + var atxOpts system.GetAtxOpts + for _, opt := range opts { + opt(&atxOpts) + } + require.NotNil(t, atxOpts.RecvChannel) + for _, id := range atxs { + switch { + case id == allAtxs[0]: + require.False(t, validationFailed, "retried after validation error") + errs[id.Hash32()] = pubsub.ErrValidationReject + validationFailed = true + case id == allAtxs[1] && failCount < 2: + errs[id.Hash32()] = errors.New("fetch failed") + failCount++ + default: + fetched = append(fetched, id) + select { + case <-time.After(100 * time.Millisecond): + t.Error("timeout sending recvd id") + case atxOpts.RecvChannel <- id: + } + } + } + if len(errs) > 0 { + var bErr fetch.BatchError + for h, err := range errs { + bErr.Add(h, err) + } + return &bErr + } + return nil + }).AnyTimes() + newSet.EXPECT().Received().Return(rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + for _, atx := range allAtxs { + if !yield(atx.Bytes()) { + return + } + } + }, + Error: rangesync.NoSeqError, + }) + require.NoError(t, h.Commit(context.Background(), peer, baseSet, newSet)) + require.ElementsMatch(t, allAtxs[1:], fetched) +} + +func TestAtxHandler_Cancel(t *testing.T) { + const ( + batchSize = 4 + maxAttempts = 3 + maxBatchRetries = 2 + batchRetryDelay = 10 * time.Second + ) + atxID := types.RandomATXID() + ctrl := gomock.NewController(t) + logger := zaptest.NewLogger(t) + peer := p2p.Peer("foobar") + f := NewMockFetcher(ctrl) + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + baseSet := mocks.NewMockOrderedSet(ctrl) + newSet := mocks.NewMockOrderedSet(ctrl) + baseSet.EXPECT().Has(rangesync.KeyBytes(atxID[:])).Return(false, nil) + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + return context.Canceled + }) + newSet.EXPECT().Received().Return(rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + yield(atxID.Bytes()) + }, + Error: rangesync.NoSeqError, + }) + require.ErrorIs(t, h.Commit(context.Background(), peer, baseSet, newSet), context.Canceled) +} + +func TestAtxHandler_BatchRetry(t *testing.T) { + const ( + batchSize = 4 + maxAttempts = 3 + maxBatchRetries = 2 + batchRetryDelay = 10 * time.Second + ) + ctrl := gomock.NewController(t) + allAtxs := make([]types.ATXID, 10) + logger := zaptest.NewLogger(t) + peer := p2p.Peer("foobar") + for i := range allAtxs { + allAtxs[i] = types.RandomATXID() + } + clock := clockwork.NewFakeClock() + f := NewMockFetcher(ctrl) + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) + baseSet := mocks.NewMockOrderedSet(ctrl) + newSet := mocks.NewMockOrderedSet(ctrl) + for _, id := range allAtxs { + f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) + add, err := h.Receive(id.Bytes(), peer) + require.False(t, add) + require.NoError(t, err) + } + newSet.EXPECT().Received().Return(rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + // Received sequence may be cyclic and the handler should stop + // when it sees the first key again. + for { + for _, atx := range allAtxs { + if !yield(atx.Bytes()) { + return + } + } + } + }, + Error: rangesync.NoSeqError, + }) + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + return errors.New("fetch failed") + }) + var eg errgroup.Group + eg.Go(func() error { + return h.Commit(context.Background(), peer, baseSet, newSet) + }) + // wait for delay after 1st batch failure + clock.BlockUntil(1) + toFetch := make(map[types.ATXID]bool) + for _, id := range allAtxs { + toFetch[id] = true + } + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + var atxOpts system.GetAtxOpts + for _, opt := range opts { + opt(&atxOpts) + } + require.NotNil(t, atxOpts.RecvChannel) + for _, id := range atxs { + require.True(t, toFetch[id], "already fetched or bad ID") + delete(toFetch, id) + select { + case <-time.After(100 * time.Millisecond): + t.Error("timeout sending recvd id") + case atxOpts.RecvChannel <- id: + } + } + return nil + }).Times(3) + clock.Advance(batchRetryDelay) + require.NoError(t, eg.Wait()) + require.Empty(t, toFetch) +} + +func TestAtxHandler_BatchRetry_Fail(t *testing.T) { + const ( + batchSize = 4 + maxAttempts = 3 + maxBatchRetries = 2 + batchRetryDelay = 10 * time.Second + ) + ctrl := gomock.NewController(t) + allAtxs := make([]types.ATXID, 10) + logger := zaptest.NewLogger(t) + peer := p2p.Peer("foobar") + for i := range allAtxs { + allAtxs[i] = types.RandomATXID() + } + clock := clockwork.NewFakeClock() + f := NewMockFetcher(ctrl) + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) + baseSet := mocks.NewMockOrderedSet(ctrl) + newSet := mocks.NewMockOrderedSet(ctrl) + for _, id := range allAtxs { + f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) + add, err := h.Receive(id.Bytes(), peer) + require.False(t, add) + require.NoError(t, err) + } + newSet.EXPECT().Received().Return(rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + // Received sequence may be cyclic and the handler should stop + // when it sees the first key again. + for { + for _, atx := range allAtxs { + if !yield(atx.Bytes()) { + return + } + } + } + }, + Error: rangesync.NoSeqError, + }) + f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { + return errors.New("fetch failed") + }).Times(3) + var eg errgroup.Group + eg.Go(func() error { + return h.Commit(context.Background(), peer, baseSet, newSet) + }) + for range 2 { + clock.BlockUntil(1) + clock.Advance(batchRetryDelay) + } + require.Error(t, eg.Wait()) +} + +func TestMultiEpochATXSyncer(t *testing.T) { + ctrl := gomock.NewController(t) + logger := zaptest.NewLogger(t) + oldCfg := sync2.DefaultConfig() + oldCfg.MaxDepth = 16 + newCfg := sync2.DefaultConfig() + newCfg.MaxDepth = 24 + hss := NewMockHashSyncSource(ctrl) + mhs := sync2.NewMultiEpochATXSyncer(logger, hss, oldCfg, newCfg, 1) + ctx := context.Background() + + lastSynced, err := mhs.EnsureSync(ctx, 0, 0) + require.NoError(t, err) + require.Zero(t, lastSynced) + + var syncActions []string + curIdx := 0 + hss.EXPECT().CreateHashSync(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(name string, cfg sync2.Config, epoch types.EpochID) sync2.HashSync { + idx := curIdx + curIdx++ + syncActions = append(syncActions, + fmt.Sprintf("new %s epoch %d maxDepth %d -> %d", name, epoch, cfg.MaxDepth, idx)) + hs := NewMockHashSync(ctrl) + hs.EXPECT().Load().DoAndReturn(func() error { + syncActions = append(syncActions, fmt.Sprintf("load %d %s", idx, name)) + return nil + }).AnyTimes() + hs.EXPECT().StartAndSync(ctx).DoAndReturn(func(_ context.Context) error { + syncActions = append(syncActions, fmt.Sprintf("start+sync %d %s", idx, name)) + return nil + }).AnyTimes() + hs.EXPECT().Start().DoAndReturn(func() { + syncActions = append(syncActions, fmt.Sprintf("start %d %s", idx, name)) + }).AnyTimes() + hs.EXPECT().Stop().DoAndReturn(func() { + syncActions = append(syncActions, fmt.Sprintf("stop %d %s", idx, name)) + }).AnyTimes() + return hs + }).AnyTimes() + + // Last wait epoch 3, new epoch 3 + lastSynced, err = mhs.EnsureSync(ctx, 3, 3) + require.NoError(t, err) + require.Equal(t, []string{ + "new atx-sync-1 epoch 1 maxDepth 16 -> 0", + "load 0 atx-sync-1", + "new atx-sync-2 epoch 2 maxDepth 16 -> 1", + "load 1 atx-sync-2", + "new atx-sync-3 epoch 3 maxDepth 24 -> 2", + "load 2 atx-sync-3", + "start+sync 0 atx-sync-1", + "start+sync 1 atx-sync-2", + "start+sync 2 atx-sync-3", + }, syncActions) + syncActions = nil + require.Equal(t, types.EpochID(3), lastSynced) + + // Advance to epoch 4 w/o wait + lastSynced, err = mhs.EnsureSync(ctx, 0, 4) + require.NoError(t, err) + require.Equal(t, []string{ + "stop 2 atx-sync-3", + "new atx-sync-3 epoch 3 maxDepth 16 -> 3", + "load 3 atx-sync-3", + "new atx-sync-4 epoch 4 maxDepth 24 -> 4", + "load 4 atx-sync-4", + "start 0 atx-sync-1", + "start 1 atx-sync-2", + "start 3 atx-sync-3", + "start 4 atx-sync-4", + }, syncActions) + syncActions = nil + require.Equal(t, types.EpochID(0), lastSynced) + + mhs.Stop() + require.Equal(t, []string{ + "stop 0 atx-sync-1", + "stop 1 atx-sync-2", + "stop 3 atx-sync-3", + "stop 4 atx-sync-4", + }, syncActions) +} diff --git a/sync2/interface.go b/sync2/interface.go new file mode 100644 index 0000000000..624c373d2d --- /dev/null +++ b/sync2/interface.go @@ -0,0 +1,36 @@ +package sync2 + +import ( + "context" + + "github.com/libp2p/go-libp2p/core/host" + + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/fetch/peers" + "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/system" +) + +//go:generate mockgen -typed -package=sync2_test -destination=./mocks_test.go -source=./interface.go + +type Fetcher interface { + system.AtxFetcher + Host() host.Host + Peers() *peers.Peers + RegisterPeerHash(peer p2p.Peer, hash types.Hash32) +} + +type HashSync interface { + Load() error + Start() + Stop() + StartAndSync(ctx context.Context) error +} + +type HashSyncSource interface { + CreateHashSync(name string, cfg Config, epoch types.EpochID) HashSync +} + +type LayerTicker interface { + CurrentLayer() types.LayerID +} diff --git a/sync2/mocks_test.go b/sync2/mocks_test.go new file mode 100644 index 0000000000..640dde1143 --- /dev/null +++ b/sync2/mocks_test.go @@ -0,0 +1,498 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./interface.go +// +// Generated by this command: +// +// mockgen -typed -package=sync2_test -destination=./mocks_test.go -source=./interface.go +// + +// Package sync2_test is a generated GoMock package. +package sync2_test + +import ( + context "context" + reflect "reflect" + + host "github.com/libp2p/go-libp2p/core/host" + types "github.com/spacemeshos/go-spacemesh/common/types" + peers "github.com/spacemeshos/go-spacemesh/fetch/peers" + p2p "github.com/spacemeshos/go-spacemesh/p2p" + sync2 "github.com/spacemeshos/go-spacemesh/sync2" + system "github.com/spacemeshos/go-spacemesh/system" + gomock "go.uber.org/mock/gomock" +) + +// MockFetcher is a mock of Fetcher interface. +type MockFetcher struct { + ctrl *gomock.Controller + recorder *MockFetcherMockRecorder + isgomock struct{} +} + +// MockFetcherMockRecorder is the mock recorder for MockFetcher. +type MockFetcherMockRecorder struct { + mock *MockFetcher +} + +// NewMockFetcher creates a new mock instance. +func NewMockFetcher(ctrl *gomock.Controller) *MockFetcher { + mock := &MockFetcher{ctrl: ctrl} + mock.recorder = &MockFetcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFetcher) EXPECT() *MockFetcherMockRecorder { + return m.recorder +} + +// GetAtxs mocks base method. +func (m *MockFetcher) GetAtxs(arg0 context.Context, arg1 []types.ATXID, arg2 ...system.GetAtxOpt) error { + m.ctrl.T.Helper() + varargs := []any{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetAtxs", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetAtxs indicates an expected call of GetAtxs. +func (mr *MockFetcherMockRecorder) GetAtxs(arg0, arg1 any, arg2 ...any) *MockFetcherGetAtxsCall { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0, arg1}, arg2...) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAtxs", reflect.TypeOf((*MockFetcher)(nil).GetAtxs), varargs...) + return &MockFetcherGetAtxsCall{Call: call} +} + +// MockFetcherGetAtxsCall wrap *gomock.Call +type MockFetcherGetAtxsCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockFetcherGetAtxsCall) Return(arg0 error) *MockFetcherGetAtxsCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockFetcherGetAtxsCall) Do(f func(context.Context, []types.ATXID, ...system.GetAtxOpt) error) *MockFetcherGetAtxsCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockFetcherGetAtxsCall) DoAndReturn(f func(context.Context, []types.ATXID, ...system.GetAtxOpt) error) *MockFetcherGetAtxsCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Host mocks base method. +func (m *MockFetcher) Host() host.Host { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Host") + ret0, _ := ret[0].(host.Host) + return ret0 +} + +// Host indicates an expected call of Host. +func (mr *MockFetcherMockRecorder) Host() *MockFetcherHostCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*MockFetcher)(nil).Host)) + return &MockFetcherHostCall{Call: call} +} + +// MockFetcherHostCall wrap *gomock.Call +type MockFetcherHostCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockFetcherHostCall) Return(arg0 host.Host) *MockFetcherHostCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockFetcherHostCall) Do(f func() host.Host) *MockFetcherHostCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockFetcherHostCall) DoAndReturn(f func() host.Host) *MockFetcherHostCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Peers mocks base method. +func (m *MockFetcher) Peers() *peers.Peers { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Peers") + ret0, _ := ret[0].(*peers.Peers) + return ret0 +} + +// Peers indicates an expected call of Peers. +func (mr *MockFetcherMockRecorder) Peers() *MockFetcherPeersCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*MockFetcher)(nil).Peers)) + return &MockFetcherPeersCall{Call: call} +} + +// MockFetcherPeersCall wrap *gomock.Call +type MockFetcherPeersCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockFetcherPeersCall) Return(arg0 *peers.Peers) *MockFetcherPeersCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockFetcherPeersCall) Do(f func() *peers.Peers) *MockFetcherPeersCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockFetcherPeersCall) DoAndReturn(f func() *peers.Peers) *MockFetcherPeersCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// RegisterPeerHash mocks base method. +func (m *MockFetcher) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RegisterPeerHash", peer, hash) +} + +// RegisterPeerHash indicates an expected call of RegisterPeerHash. +func (mr *MockFetcherMockRecorder) RegisterPeerHash(peer, hash any) *MockFetcherRegisterPeerHashCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*MockFetcher)(nil).RegisterPeerHash), peer, hash) + return &MockFetcherRegisterPeerHashCall{Call: call} +} + +// MockFetcherRegisterPeerHashCall wrap *gomock.Call +type MockFetcherRegisterPeerHashCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockFetcherRegisterPeerHashCall) Return() *MockFetcherRegisterPeerHashCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockFetcherRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockFetcherRegisterPeerHashCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockFetcherRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockFetcherRegisterPeerHashCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockHashSync is a mock of HashSync interface. +type MockHashSync struct { + ctrl *gomock.Controller + recorder *MockHashSyncMockRecorder + isgomock struct{} +} + +// MockHashSyncMockRecorder is the mock recorder for MockHashSync. +type MockHashSyncMockRecorder struct { + mock *MockHashSync +} + +// NewMockHashSync creates a new mock instance. +func NewMockHashSync(ctrl *gomock.Controller) *MockHashSync { + mock := &MockHashSync{ctrl: ctrl} + mock.recorder = &MockHashSyncMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockHashSync) EXPECT() *MockHashSyncMockRecorder { + return m.recorder +} + +// Load mocks base method. +func (m *MockHashSync) Load() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load") + ret0, _ := ret[0].(error) + return ret0 +} + +// Load indicates an expected call of Load. +func (mr *MockHashSyncMockRecorder) Load() *MockHashSyncLoadCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockHashSync)(nil).Load)) + return &MockHashSyncLoadCall{Call: call} +} + +// MockHashSyncLoadCall wrap *gomock.Call +type MockHashSyncLoadCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockHashSyncLoadCall) Return(arg0 error) *MockHashSyncLoadCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockHashSyncLoadCall) Do(f func() error) *MockHashSyncLoadCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockHashSyncLoadCall) DoAndReturn(f func() error) *MockHashSyncLoadCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Start mocks base method. +func (m *MockHashSync) Start() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start") +} + +// Start indicates an expected call of Start. +func (mr *MockHashSyncMockRecorder) Start() *MockHashSyncStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockHashSync)(nil).Start)) + return &MockHashSyncStartCall{Call: call} +} + +// MockHashSyncStartCall wrap *gomock.Call +type MockHashSyncStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockHashSyncStartCall) Return() *MockHashSyncStartCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockHashSyncStartCall) Do(f func()) *MockHashSyncStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockHashSyncStartCall) DoAndReturn(f func()) *MockHashSyncStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// StartAndSync mocks base method. +func (m *MockHashSync) StartAndSync(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartAndSync", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartAndSync indicates an expected call of StartAndSync. +func (mr *MockHashSyncMockRecorder) StartAndSync(ctx any) *MockHashSyncStartAndSyncCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartAndSync", reflect.TypeOf((*MockHashSync)(nil).StartAndSync), ctx) + return &MockHashSyncStartAndSyncCall{Call: call} +} + +// MockHashSyncStartAndSyncCall wrap *gomock.Call +type MockHashSyncStartAndSyncCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockHashSyncStartAndSyncCall) Return(arg0 error) *MockHashSyncStartAndSyncCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockHashSyncStartAndSyncCall) Do(f func(context.Context) error) *MockHashSyncStartAndSyncCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockHashSyncStartAndSyncCall) DoAndReturn(f func(context.Context) error) *MockHashSyncStartAndSyncCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *MockHashSync) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop. +func (mr *MockHashSyncMockRecorder) Stop() *MockHashSyncStopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockHashSync)(nil).Stop)) + return &MockHashSyncStopCall{Call: call} +} + +// MockHashSyncStopCall wrap *gomock.Call +type MockHashSyncStopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockHashSyncStopCall) Return() *MockHashSyncStopCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockHashSyncStopCall) Do(f func()) *MockHashSyncStopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockHashSyncStopCall) DoAndReturn(f func()) *MockHashSyncStopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockHashSyncSource is a mock of HashSyncSource interface. +type MockHashSyncSource struct { + ctrl *gomock.Controller + recorder *MockHashSyncSourceMockRecorder + isgomock struct{} +} + +// MockHashSyncSourceMockRecorder is the mock recorder for MockHashSyncSource. +type MockHashSyncSourceMockRecorder struct { + mock *MockHashSyncSource +} + +// NewMockHashSyncSource creates a new mock instance. +func NewMockHashSyncSource(ctrl *gomock.Controller) *MockHashSyncSource { + mock := &MockHashSyncSource{ctrl: ctrl} + mock.recorder = &MockHashSyncSourceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockHashSyncSource) EXPECT() *MockHashSyncSourceMockRecorder { + return m.recorder +} + +// CreateHashSync mocks base method. +func (m *MockHashSyncSource) CreateHashSync(name string, cfg sync2.Config, epoch types.EpochID) sync2.HashSync { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHashSync", name, cfg, epoch) + ret0, _ := ret[0].(sync2.HashSync) + return ret0 +} + +// CreateHashSync indicates an expected call of CreateHashSync. +func (mr *MockHashSyncSourceMockRecorder) CreateHashSync(name, cfg, epoch any) *MockHashSyncSourceCreateHashSyncCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHashSync", reflect.TypeOf((*MockHashSyncSource)(nil).CreateHashSync), name, cfg, epoch) + return &MockHashSyncSourceCreateHashSyncCall{Call: call} +} + +// MockHashSyncSourceCreateHashSyncCall wrap *gomock.Call +type MockHashSyncSourceCreateHashSyncCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockHashSyncSourceCreateHashSyncCall) Return(arg0 sync2.HashSync) *MockHashSyncSourceCreateHashSyncCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockHashSyncSourceCreateHashSyncCall) Do(f func(string, sync2.Config, types.EpochID) sync2.HashSync) *MockHashSyncSourceCreateHashSyncCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockHashSyncSourceCreateHashSyncCall) DoAndReturn(f func(string, sync2.Config, types.EpochID) sync2.HashSync) *MockHashSyncSourceCreateHashSyncCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockLayerTicker is a mock of LayerTicker interface. +type MockLayerTicker struct { + ctrl *gomock.Controller + recorder *MockLayerTickerMockRecorder + isgomock struct{} +} + +// MockLayerTickerMockRecorder is the mock recorder for MockLayerTicker. +type MockLayerTickerMockRecorder struct { + mock *MockLayerTicker +} + +// NewMockLayerTicker creates a new mock instance. +func NewMockLayerTicker(ctrl *gomock.Controller) *MockLayerTicker { + mock := &MockLayerTicker{ctrl: ctrl} + mock.recorder = &MockLayerTickerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLayerTicker) EXPECT() *MockLayerTickerMockRecorder { + return m.recorder +} + +// CurrentLayer mocks base method. +func (m *MockLayerTicker) CurrentLayer() types.LayerID { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CurrentLayer") + ret0, _ := ret[0].(types.LayerID) + return ret0 +} + +// CurrentLayer indicates an expected call of CurrentLayer. +func (mr *MockLayerTickerMockRecorder) CurrentLayer() *MockLayerTickerCurrentLayerCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CurrentLayer", reflect.TypeOf((*MockLayerTicker)(nil).CurrentLayer)) + return &MockLayerTickerCurrentLayerCall{Call: call} +} + +// MockLayerTickerCurrentLayerCall wrap *gomock.Call +type MockLayerTickerCurrentLayerCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockLayerTickerCurrentLayerCall) Return(arg0 types.LayerID) *MockLayerTickerCurrentLayerCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockLayerTickerCurrentLayerCall) Do(f func() types.LayerID) *MockLayerTickerCurrentLayerCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockLayerTickerCurrentLayerCall) DoAndReturn(f func() types.LayerID) *MockLayerTickerCurrentLayerCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/syncer/interface.go b/syncer/interface.go index 6c9d2244d9..e1d9c5205c 100644 --- a/syncer/interface.go +++ b/syncer/interface.go @@ -73,3 +73,8 @@ type forkFinder interface { FindFork(context.Context, p2p.Peer, types.LayerID, types.Hash32) (types.LayerID, error) Purge(bool, ...p2p.Peer) } + +type multiEpochAtxSyncerV2 interface { + EnsureSync(ctx context.Context, lastWaitEpoch, newEpoch types.EpochID) (lastSynced types.EpochID, err error) + Stop() +} diff --git a/syncer/mocks/mocks.go b/syncer/mocks/mocks.go index 5cf1f95dd6..7980900806 100644 --- a/syncer/mocks/mocks.go +++ b/syncer/mocks/mocks.go @@ -1681,3 +1681,102 @@ func (c *MockforkFinderUpdateAgreementCall) DoAndReturn(f func(p2p.Peer, types.L c.Call = c.Call.DoAndReturn(f) return c } + +// MockmultiEpochAtxSyncerV2 is a mock of multiEpochAtxSyncerV2 interface. +type MockmultiEpochAtxSyncerV2 struct { + ctrl *gomock.Controller + recorder *MockmultiEpochAtxSyncerV2MockRecorder + isgomock struct{} +} + +// MockmultiEpochAtxSyncerV2MockRecorder is the mock recorder for MockmultiEpochAtxSyncerV2. +type MockmultiEpochAtxSyncerV2MockRecorder struct { + mock *MockmultiEpochAtxSyncerV2 +} + +// NewMockmultiEpochAtxSyncerV2 creates a new mock instance. +func NewMockmultiEpochAtxSyncerV2(ctrl *gomock.Controller) *MockmultiEpochAtxSyncerV2 { + mock := &MockmultiEpochAtxSyncerV2{ctrl: ctrl} + mock.recorder = &MockmultiEpochAtxSyncerV2MockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockmultiEpochAtxSyncerV2) EXPECT() *MockmultiEpochAtxSyncerV2MockRecorder { + return m.recorder +} + +// EnsureSync mocks base method. +func (m *MockmultiEpochAtxSyncerV2) EnsureSync(ctx context.Context, lastWaitEpoch, newEpoch types.EpochID) (types.EpochID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureSync", ctx, lastWaitEpoch, newEpoch) + ret0, _ := ret[0].(types.EpochID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnsureSync indicates an expected call of EnsureSync. +func (mr *MockmultiEpochAtxSyncerV2MockRecorder) EnsureSync(ctx, lastWaitEpoch, newEpoch any) *MockmultiEpochAtxSyncerV2EnsureSyncCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureSync", reflect.TypeOf((*MockmultiEpochAtxSyncerV2)(nil).EnsureSync), ctx, lastWaitEpoch, newEpoch) + return &MockmultiEpochAtxSyncerV2EnsureSyncCall{Call: call} +} + +// MockmultiEpochAtxSyncerV2EnsureSyncCall wrap *gomock.Call +type MockmultiEpochAtxSyncerV2EnsureSyncCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockmultiEpochAtxSyncerV2EnsureSyncCall) Return(lastSynced types.EpochID, err error) *MockmultiEpochAtxSyncerV2EnsureSyncCall { + c.Call = c.Call.Return(lastSynced, err) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockmultiEpochAtxSyncerV2EnsureSyncCall) Do(f func(context.Context, types.EpochID, types.EpochID) (types.EpochID, error)) *MockmultiEpochAtxSyncerV2EnsureSyncCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockmultiEpochAtxSyncerV2EnsureSyncCall) DoAndReturn(f func(context.Context, types.EpochID, types.EpochID) (types.EpochID, error)) *MockmultiEpochAtxSyncerV2EnsureSyncCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *MockmultiEpochAtxSyncerV2) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop. +func (mr *MockmultiEpochAtxSyncerV2MockRecorder) Stop() *MockmultiEpochAtxSyncerV2StopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockmultiEpochAtxSyncerV2)(nil).Stop)) + return &MockmultiEpochAtxSyncerV2StopCall{Call: call} +} + +// MockmultiEpochAtxSyncerV2StopCall wrap *gomock.Call +type MockmultiEpochAtxSyncerV2StopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockmultiEpochAtxSyncerV2StopCall) Return() *MockmultiEpochAtxSyncerV2StopCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockmultiEpochAtxSyncerV2StopCall) Do(f func()) *MockmultiEpochAtxSyncerV2StopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockmultiEpochAtxSyncerV2StopCall) DoAndReturn(f func()) *MockmultiEpochAtxSyncerV2StopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/syncer/syncer.go b/syncer/syncer.go index ff52123950..6f4e80a979 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -18,6 +18,9 @@ import ( "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/mesh" "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sync2" + "github.com/spacemeshos/go-spacemesh/sync2/rangesync" "github.com/spacemeshos/go-spacemesh/syncer/atxsync" "github.com/spacemeshos/go-spacemesh/syncer/malsync" "github.com/spacemeshos/go-spacemesh/system" @@ -39,10 +42,25 @@ type Config struct { OutOfSyncThresholdLayers uint32 `mapstructure:"out-of-sync-threshold"` AtxSync atxsync.Config `mapstructure:"atx-sync"` MalSync malsync.Config `mapstructure:"malfeasance-sync"` + V2 SyncV2Config `mapstructure:"v2"` +} + +type SyncV2Config struct { + Enable bool `mapstructure:"enable"` + EnableActiveSync bool `mapstructure:"enable-active-sync"` + OldAtxSyncCfg sync2.Config `mapstructure:"old-atx-sync"` + NewAtxSyncCfg sync2.Config `mapstructure:"new-atx-sync"` + ParallelLoadLimit int `mapstructure:"parallel-load-limit"` } // DefaultConfig for the syncer. func DefaultConfig() Config { + oldAtxSyncCfg := sync2.DefaultConfig() + oldAtxSyncCfg.MaxDepth = 16 + oldAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = time.Hour + newAtxSyncCfg := sync2.DefaultConfig() + newAtxSyncCfg.MaxDepth = 21 + newAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = 5 * time.Minute return Config{ Interval: 10 * time.Second, EpochEndFraction: 0.5, @@ -54,6 +72,13 @@ func DefaultConfig() Config { OutOfSyncThresholdLayers: 3, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), + V2: SyncV2Config{ + Enable: false, + EnableActiveSync: false, + OldAtxSyncCfg: oldAtxSyncCfg, + NewAtxSyncCfg: newAtxSyncCfg, + ParallelLoadLimit: 10, + }, } } @@ -119,6 +144,12 @@ func withForkFinder(f forkFinder) Option { } } +func withAtxSyncerV2(asv2 multiEpochAtxSyncerV2) Option { + return func(s *Syncer) { + s.asv2 = asv2 + } +} + // Syncer is responsible to keep the node in sync with the network. type Syncer struct { logger *zap.Logger @@ -162,6 +193,9 @@ type Syncer struct { eg errgroup.Group stop context.CancelFunc + + asv2 multiEpochAtxSyncerV2 + dispatcher *rangesync.Dispatcher } // NewSyncer creates a new Syncer instance. @@ -207,6 +241,15 @@ func NewSyncer( s.isBusy.Store(false) s.lastLayerSynced.Store(s.mesh.LatestLayer().Uint32()) s.lastEpochSynced.Store(types.GetEffectiveGenesis().GetEpoch().Uint32() - 1) + if s.cfg.V2.Enable && s.asv2 == nil { + s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher)) + hss := sync2.NewATXSyncSource( + s.logger, s.dispatcher, cdb.Database.(sql.StateDatabase), + fetcher.(sync2.Fetcher), s.cfg.V2.EnableActiveSync) + s.asv2 = sync2.NewMultiEpochATXSyncer( + s.logger, hss, s.cfg.V2.OldAtxSyncCfg, s.cfg.V2.NewAtxSyncCfg, + s.cfg.V2.ParallelLoadLimit) + } return s } @@ -218,6 +261,9 @@ func (s *Syncer) Close() { s.stop() s.logger.Debug("waiting for syncer goroutines to finish") err := s.eg.Wait() + if s.asv2 != nil { + s.asv2.Stop() + } s.logger.Debug("all syncer goroutines finished", zap.Error(err)) } @@ -251,7 +297,13 @@ func (s *Syncer) Start() { s.syncOnce.Do(func() { ctx, cancel := context.WithCancel(context.Background()) s.stop = cancel + s.logger.Info("starting syncer loop", log.ZContext(ctx)) + if s.dispatcher != nil { + s.eg.Go(func() error { + return s.dispatcher.Server.Run(ctx) + }) + } s.eg.Go(func() error { if s.ticker.CurrentLayer() <= types.GetEffectiveGenesis() { s.setSyncState(ctx, synced) @@ -413,7 +465,7 @@ func (s *Syncer) synchronize(ctx context.Context) bool { return false } - if err := s.syncAtx(ctx); err != nil { + if err := s.syncAtxAndMalfeasance(ctx); err != nil { if !errors.Is(err, context.Canceled) { s.logger.Error("failed to sync atxs", log.ZContext(ctx), zap.Error(err)) } @@ -423,6 +475,7 @@ func (s *Syncer) synchronize(ctx context.Context) bool { if s.ticker.CurrentLayer() <= types.GetEffectiveGenesis() { return true } + // always sync to currentLayer-1 to reduce race with gossip and hare/tortoise for layer := s.getLastSyncedLayer().Add(1); layer.Before(s.ticker.CurrentLayer()); layer = layer.Add(1) { if err := s.syncLayer(ctx, layer); err != nil { @@ -471,8 +524,18 @@ func (s *Syncer) synchronize(ctx context.Context) bool { return success } -func (s *Syncer) syncAtx(ctx context.Context) error { +func (s *Syncer) ensureATXsInSync(ctx context.Context) error { current := s.ticker.CurrentLayer() + publish := current.GetEpoch() + if publish == 0 { + return nil // nothing to sync in epoch 0 + } + + // if we are not advanced enough sync previous epoch, otherwise start syncing activations published in this epoch + if current.OrdinalInEpoch() <= uint32(float64(types.GetLayersPerEpoch())*s.cfg.EpochEndFraction) { + publish -= 1 + } + // on startup always download all activations that were published before current epoch if !s.ListenToATXGossip() { s.logger.Debug("syncing atx from genesis", @@ -486,31 +549,15 @@ func (s *Syncer) syncAtx(ctx context.Context) error { } } s.logger.Debug("atxs synced to epoch", log.ZContext(ctx), zap.Stringer("last epoch", s.lastAtxEpoch())) - - // FIXME https://github.com/spacemeshos/go-spacemesh/issues/3987 - s.logger.Info("syncing malicious proofs", log.ZContext(ctx)) - if err := s.syncMalfeasance(ctx, current.GetEpoch()); err != nil { - return err - } - s.logger.Info("malicious IDs synced", log.ZContext(ctx)) - s.setATXSynced() - } - - publish := current.GetEpoch() - if publish == 0 { - return nil // nothing to sync in epoch 0 } - // if we are not advanced enough sync previous epoch, otherwise start syncing activations published in this epoch - if current.OrdinalInEpoch() <= uint32(float64(types.GetLayersPerEpoch())*s.cfg.EpochEndFraction) { - publish -= 1 - } if epoch := s.backgroundSync.epoch.Load(); epoch != 0 && epoch != publish.Uint32() { s.backgroundSync.cancel() s.backgroundSync.eg.Wait() s.backgroundSync.epoch.Store(0) } if s.backgroundSync.epoch.Load() == 0 && publish.Uint32() != 0 { + // TODO: syncv2 s.logger.Debug("download atx for epoch in background", zap.Stringer("publish", publish), log.ZContext(ctx)) s.backgroundSync.epoch.Store(publish.Uint32()) ctx, cancel := context.WithCancel(ctx) @@ -533,7 +580,75 @@ func (s *Syncer) syncAtx(ctx context.Context) error { return err }) } - if !s.malSync.started { + return nil +} + +// ensureATXsInSyncV2 ensures that the ATXs are in sync and being synchronized +// continuously using syncv2. +func (s *Syncer) ensureATXsInSyncV2(ctx context.Context) error { + current := s.ticker.CurrentLayer() + currentEpoch := current.GetEpoch() + if currentEpoch == 0 { + return nil // nothing to sync in epoch 0 + } + publish := currentEpoch + if current.OrdinalInEpoch() <= uint32(float64(types.GetLayersPerEpoch())*s.cfg.EpochEndFraction) { + publish-- + } + + if !s.ListenToATXGossip() && s.cfg.V2.EnableActiveSync { + // ATXs are not in sync yet, to we need to sync them synchronously + lastWaitEpoch := types.EpochID(0) + if currentEpoch > 1 { + lastWaitEpoch = currentEpoch - 1 + } + s.logger.Debug("syncing atx from genesis", + log.ZContext(ctx), + zap.Stringer("current layer", current), + zap.Stringer("last synced epoch", s.lastAtxEpoch()), + zap.Stringer("lastWaitEpoch", lastWaitEpoch), + zap.Stringer("publish", publish), + ) + lastAtxEpoch, err := s.asv2.EnsureSync(ctx, lastWaitEpoch, publish) + if lastAtxEpoch > 0 { + s.setLastAtxEpoch(lastAtxEpoch) + } + if err != nil { + return fmt.Errorf("syncing atxs: %w", err) + } + s.logger.Debug("atxs synced to epoch", + log.ZContext(ctx), zap.Stringer("last epoch", s.lastAtxEpoch())) + return nil + } + + // When active syncv2 is not enabled, this will only cause the per-epoch sync + // servers (multiplexed via dispatcher) to be activated, without attempting to + // initiate sync against the peers + s.logger.Debug("activating sync2", zap.Uint32("new epoch", publish.Uint32())) + if _, err := s.asv2.EnsureSync(ctx, 0, publish); err != nil { + return fmt.Errorf("activating sync: %w", err) + } + + return nil +} + +func (s *Syncer) ensureMalfeasanceInSync(ctx context.Context) error { + // TODO: use syncv2 for malfeasance proofs: + // https://github.com/spacemeshos/go-spacemesh/issues/3987 + current := s.ticker.CurrentLayer() + if !s.ListenToATXGossip() { + s.logger.Info("syncing malicious proofs", log.ZContext(ctx)) + if err := s.syncMalfeasance(ctx, current.GetEpoch()); err != nil { + return err + } + s.logger.Info("malicious IDs synced", log.ZContext(ctx)) + // Malfeasance proofs are synced after the actual ATXs. + // We set ATX synced status after both ATXs and malfeascance proofs + // are in sync. + s.setATXSynced() + } + + if current.GetEpoch() > 0 && !s.malSync.started { s.malSync.started = true s.malSync.eg.Go(func() error { select { @@ -548,9 +663,26 @@ func (s *Syncer) syncAtx(ctx context.Context) error { } }) } + return nil } +func (s *Syncer) syncAtxAndMalfeasance(ctx context.Context) error { + if s.cfg.V2.Enable { + if err := s.ensureATXsInSyncV2(ctx); err != nil { + return err + } + } + if !s.cfg.V2.Enable || !s.cfg.V2.EnableActiveSync { + // If syncv2 is being used in server-only mode, we still need to run + // active syncv1. + if err := s.ensureATXsInSync(ctx); err != nil { + return err + } + } + return s.ensureMalfeasanceInSync(ctx) +} + func isTooFarBehind( ctx context.Context, logger *zap.Logger, diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index cb74d04344..1b979ab89a 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -82,6 +82,7 @@ type testSyncer struct { mTortoise *smocks.MockTortoise mCertHdr *mocks.MockcertHandler mForkFinder *mocks.MockforkFinder + mASV2 *mocks.MockmultiEpochAtxSyncerV2 } func (ts *testSyncer) expectMalEnsureInSync(current types.LayerID) { @@ -92,7 +93,7 @@ func (ts *testSyncer) expectMalEnsureInSync(current types.LayerID) { ) } -func (ts *testSyncer) expectDownloadLoop() chan struct{} { +func (ts *testSyncer) expectMalDownloadLoop() chan struct{} { ch := make(chan struct{}) ts.mMalSyncer.EXPECT().DownloadLoop(gomock.Any()). DoAndReturn(func(context.Context) error { @@ -109,7 +110,7 @@ func (ts *testSyncer) expectDownloadLoop() chan struct{} { return ch } -func newTestSyncer(tb testing.TB, interval time.Duration) *testSyncer { +func newTestSyncerWithConfig(tb testing.TB, cfg Config) *testSyncer { lg := zaptest.NewLogger(tb) mt := newMockLayerTicker() ctrl := gomock.NewController(tb) @@ -127,6 +128,7 @@ func newTestSyncer(tb testing.TB, interval time.Duration) *testSyncer { mTortoise: smocks.NewMockTortoise(ctrl), mCertHdr: mocks.NewMockcertHandler(ctrl), mForkFinder: mocks.NewMockforkFinder(ctrl), + mASV2: mocks.NewMockmultiEpochAtxSyncerV2(ctrl), } db := statesql.InMemoryTest(tb) ts.cdb = datastore.NewCachedDB(db, lg) @@ -137,14 +139,6 @@ func newTestSyncer(tb testing.TB, interval time.Duration) *testSyncer { ts.msh, err = mesh.NewMesh(db, atxsdata, ts.mTortoise, exec, ts.mConState, lg) require.NoError(tb, err) - cfg := Config{ - Interval: interval, - GossipDuration: 5 * time.Millisecond, - EpochEndFraction: 0.66, - SyncCertDistance: 4, - HareDelayLayers: 5, - OutOfSyncThresholdLayers: outOfSyncThreshold, - } ts.syncer = NewSyncer( ts.cdb, ts.mTicker, @@ -160,16 +154,39 @@ func newTestSyncer(tb testing.TB, interval time.Duration) *testSyncer { WithLogger(lg), withDataFetcher(ts.mDataFetcher), withForkFinder(ts.mForkFinder), + withAtxSyncerV2(ts.mASV2), ) return ts } +func defaultTestConfig(interval time.Duration) Config { + return Config{ + Interval: interval, + GossipDuration: 5 * time.Millisecond, + EpochEndFraction: 0.66, + SyncCertDistance: 4, + HareDelayLayers: 5, + OutOfSyncThresholdLayers: outOfSyncThreshold, + } +} + +func newTestSyncer(tb testing.TB, interval time.Duration) *testSyncer { + return newTestSyncerWithConfig(tb, defaultTestConfig(interval)) +} + func newSyncerWithoutPeriodicRuns(tb testing.TB) *testSyncer { ts := newTestSyncer(tb, never) ts.mDataFetcher.EXPECT().SelectBestShuffled(gomock.Any()).Return([]p2p.Peer{"non-empty"}).AnyTimes() return ts } +func newSyncerWithoutPeriodicRunsWithConfig(tb testing.TB, cfg Config) *testSyncer { + cfg.Interval = never + ts := newTestSyncerWithConfig(tb, cfg) + ts.mDataFetcher.EXPECT().SelectBestShuffled(gomock.Any()).Return([]p2p.Peer{"non-empty"}).AnyTimes() + return ts +} + func newTestSyncerForState(tb testing.TB) *testSyncer { ts := newTestSyncer(tb, never) return ts @@ -194,6 +211,7 @@ func TestStartAndShutdown(t *testing.T) { ts.syncer.IsSynced(ctx) }, time.Second, 10*time.Millisecond) + ts.mASV2.EXPECT().Stop() cancel() require.False(t, ts.syncer.synchronize(ctx)) ts.syncer.Close() @@ -210,7 +228,7 @@ func TestSynchronize_OnlyOneSynchronize(t *testing.T) { ts.mTicker.advanceToLayer(current) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - dlCh := ts.expectDownloadLoop() + dlCh := ts.expectMalDownloadLoop() ts.syncer.Start() ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() @@ -243,6 +261,7 @@ func TestSynchronize_OnlyOneSynchronize(t *testing.T) { require.NoError(t, eg.Wait()) <-dlCh + ts.mASV2.EXPECT().Stop() cancel() ts.syncer.Close() } @@ -271,7 +290,7 @@ func advanceState(tb testing.TB, ts *testSyncer, from, to types.LayerID) { func TestSynchronize_AllGood(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() gLayer := types.GetEffectiveGenesis() current1 := gLayer.Add(10) ts.mTicker.advanceToLayer(current1) @@ -346,7 +365,7 @@ func TestSynchronize_AllGood(t *testing.T) { func TestSynchronize_FetchLayerDataFailed(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() gLayer := types.GetEffectiveGenesis() current := gLayer.Add(2) ts.mTicker.advanceToLayer(current) @@ -459,7 +478,7 @@ func TestSyncAtxs_Genesis(t *testing.T) { }) t.Run("first atx epoch", func(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() epoch := types.EpochID(1) current := epoch.FirstLayer() + 2 ts.mTicker.advanceToLayer(current) // to pass epoch end fraction threshold @@ -487,6 +506,33 @@ func TestSyncAtxs_Genesis(t *testing.T) { }) } +func TestSyncAtxs_Genesis_SyncV2(t *testing.T) { + cfg := defaultTestConfig(never) + cfg.V2.Enable = true + cfg.V2.EnableActiveSync = true + + t.Run("no atx expected", func(t *testing.T) { + ts := newSyncerWithoutPeriodicRunsWithConfig(t, cfg) + ts.mTicker.advanceToLayer(1) + require.True(t, ts.syncer.synchronize(context.Background())) + require.True(t, ts.syncer.ListenToATXGossip()) + require.Equal(t, types.EpochID(0), ts.syncer.lastAtxEpoch()) + }) + + t.Run("first atx epoch", func(t *testing.T) { + ts := newSyncerWithoutPeriodicRunsWithConfig(t, cfg) + ts.expectMalDownloadLoop() + epoch := types.EpochID(1) + current := epoch.FirstLayer() + 2 + ts.mTicker.advanceToLayer(current) // to pass epoch end fraction threshold + require.False(t, ts.syncer.ListenToATXGossip()) + ts.mASV2.EXPECT().EnsureSync(gomock.Any(), types.EpochID(0), epoch) + ts.expectMalEnsureInSync(current) + require.True(t, ts.syncer.synchronize(context.Background())) + require.True(t, ts.syncer.ListenToATXGossip()) + }) +} + func TestSyncAtxs(t *testing.T) { tcs := []struct { desc string @@ -507,7 +553,7 @@ func TestSyncAtxs(t *testing.T) { for _, tc := range tcs { t.Run(tc.desc, func(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() lyr := startWithSyncedState(t, ts) require.LessOrEqual(t, lyr, tc.current) ts.mTicker.advanceToLayer(tc.current) @@ -523,9 +569,72 @@ func TestSyncAtxs(t *testing.T) { } } +func startWithSyncedState_SyncV2(tb testing.TB, ts *testSyncer) types.LayerID { + tb.Helper() + + gLayer := types.GetEffectiveGenesis() + ts.mTicker.advanceToLayer(gLayer) + ts.expectMalEnsureInSync(gLayer) + ts.mASV2.EXPECT().EnsureSync(gomock.Any(), types.EpochID(0), types.EpochID(1)).MinTimes(1) + // ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gLayer.GetEpoch(), gomock.Any()) + require.True(tb, ts.syncer.synchronize(context.Background())) + ts.syncer.waitBackgroundSync() + require.True(tb, ts.syncer.ListenToATXGossip()) + require.True(tb, ts.syncer.ListenToGossip()) + require.True(tb, ts.syncer.IsSynced(context.Background())) + + current := gLayer.Add(2) + ts.mTicker.advanceToLayer(current) + lyr := current.Sub(1) + ts.mDataFetcher.EXPECT().PollLayerData(gomock.Any(), lyr) + + require.True(tb, ts.syncer.synchronize(context.Background())) + require.True(tb, ts.syncer.ListenToATXGossip()) + require.True(tb, ts.syncer.ListenToGossip()) + require.True(tb, ts.syncer.IsSynced(context.Background())) + return current +} + +func TestSyncAtxs_SyncV2(t *testing.T) { + cfg := defaultTestConfig(never) + cfg.V2.Enable = true + cfg.V2.EnableActiveSync = true + tcs := []struct { + desc string + current types.LayerID + downloaded types.EpochID + }{ + { + desc: "start of epoch", + current: 13, + downloaded: 3, + }, + { + desc: "end of epoch", + current: 14, + downloaded: 4, + }, + } + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + ts := newSyncerWithoutPeriodicRunsWithConfig(t, cfg) + ts.expectMalDownloadLoop() + lyr := startWithSyncedState_SyncV2(t, ts) + require.LessOrEqual(t, lyr, tc.current) + ts.mTicker.advanceToLayer(tc.current) + + ts.mASV2.EXPECT().EnsureSync(gomock.Any(), types.EpochID(0), tc.downloaded) + for lid := lyr; lid < tc.current; lid++ { + ts.mDataFetcher.EXPECT().PollLayerData(gomock.Any(), lid) + } + require.True(t, ts.syncer.synchronize(context.Background())) + }) + } +} + func TestSynchronize_StaySyncedUponFailure(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() lyr := startWithSyncedState(t, ts) current := lyr.Add(1) ts.mTicker.advanceToLayer(current) @@ -542,7 +651,7 @@ func TestSynchronize_StaySyncedUponFailure(t *testing.T) { func TestSynchronize_BecomeNotSyncedUponFailureIfNoGossip(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() lyr := startWithSyncedState(t, ts) current := lyr.Add(outOfSyncThreshold) ts.mTicker.advanceToLayer(current) @@ -561,7 +670,7 @@ func TestSynchronize_BecomeNotSyncedUponFailureIfNoGossip(t *testing.T) { // test the case where the node originally starts from notSynced and eventually becomes synced. func TestFromNotSyncedToSynced(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() lyr := types.GetEffectiveGenesis().Add(1) current := lyr.Add(5) @@ -596,7 +705,7 @@ func TestFromNotSyncedToSynced(t *testing.T) { // to notSynced. func TestFromGossipSyncToNotSynced(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() lyr := types.GetEffectiveGenesis().Add(1) current := lyr.Add(1) @@ -628,7 +737,7 @@ func TestFromGossipSyncToNotSynced(t *testing.T) { func TestNetworkHasNoData(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() lyr := startWithSyncedState(t, ts) require.True(t, ts.syncer.IsSynced(context.Background())) @@ -654,7 +763,7 @@ func TestNetworkHasNoData(t *testing.T) { // eventually become synced again. func TestFromSyncedToNotSynced(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() require.True(t, ts.syncer.synchronize(context.Background())) @@ -705,7 +814,7 @@ func waitOutGossipSync(tb testing.TB, ts *testSyncer) { func TestSync_AlsoSyncProcessedLayer(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() lyr := types.GetEffectiveGenesis().Add(1) current := lyr.Add(1) @@ -766,7 +875,7 @@ func TestSyncer_IsBeaconSynced(t *testing.T) { func TestSynchronize_RecoverFromCheckpoint(t *testing.T) { ts := newSyncerWithoutPeriodicRuns(t) - ts.expectDownloadLoop() + ts.expectMalDownloadLoop() current := types.GetEffectiveGenesis().Add(types.GetLayersPerEpoch() * 5) // recover from a checkpoint types.SetEffectiveGenesis(current.Uint32()) @@ -786,6 +895,7 @@ func TestSynchronize_RecoverFromCheckpoint(t *testing.T) { WithLogger(ts.syncer.logger), withDataFetcher(ts.mDataFetcher), withForkFinder(ts.mForkFinder), + withAtxSyncerV2(ts.mASV2), ) // should not sync any atxs before current epoch ts.mAtxSyncer.EXPECT().Download(gomock.Any(), current.GetEpoch(), gomock.Any()) diff --git a/system/fetcher.go b/system/fetcher.go index dec2655835..61f2d5c451 100644 --- a/system/fetcher.go +++ b/system/fetcher.go @@ -28,16 +28,26 @@ type BlockFetcher interface { type GetAtxOpts struct { LimitingOff bool + RecvChannel chan<- types.ATXID } type GetAtxOpt func(*GetAtxOpts) +// WithoutLimiting disables rate limiting when downloading ATXs. func WithoutLimiting() GetAtxOpt { return func(opts *GetAtxOpts) { opts.LimitingOff = true } } +// WithRecvChannel sets the channel to receive successfully downloaded and validated ATXs +// IDs on. +func WithRecvChannel(ch chan<- types.ATXID) GetAtxOpt { + return func(opts *GetAtxOpts) { + opts.RecvChannel = ch + } +} + // AtxFetcher defines an interface for fetching ATXs from remote peers. type AtxFetcher interface { GetAtxs(context.Context, []types.ATXID, ...GetAtxOpt) error From eff6963965f7e364efb3e56d7f279f362e016e70 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 11 Nov 2024 13:36:11 +0400 Subject: [PATCH 04/22] sync2: multipeer: fix edge cases Split sync could become blocked when there were slow peers. Their subranges are assigned to other peers, and there were bugs causing indefinite blocking and panics in these cases. Moreover, after other peers managed to sync the slow peers' subranges ahead of them, we need to interrupt syncing against the slow peers as it's no longer needed. In multipeer sync, when every peer has failed to sync, e.g. due to temporary connection interruption, we don't need to wait for the full sync interval, using shorter wait time between retries. --- sync2/multipeer/multipeer.go | 25 +++++- sync2/multipeer/multipeer_test.go | 64 +++++++++++++- sync2/multipeer/split_sync.go | 30 ++++++- sync2/multipeer/split_sync_test.go | 130 ++++++++++++++++++++--------- sync2/p2p.go | 106 +++++++++++++++-------- sync2/p2p_test.go | 4 +- 6 files changed, 273 insertions(+), 86 deletions(-) diff --git a/sync2/multipeer/multipeer.go b/sync2/multipeer/multipeer.go index 46b1d7cf48..9dde1cdcc7 100644 --- a/sync2/multipeer/multipeer.go +++ b/sync2/multipeer/multipeer.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "math/rand/v2" "time" "github.com/jonboulle/clockwork" @@ -64,6 +65,9 @@ type MultiPeerReconcilerConfig struct { MinCompleteFraction float64 `mapstructure:"min-complete-fraction"` // Interval between syncs. SyncInterval time.Duration `mapstructure:"sync-interval"` + // Interval spread factor for split sync. + // The actual interval will be SyncInterval * (1 + (random[0..2]*SplitSyncIntervalSpread-1)). + SyncIntervalSpread float64 `mapstructure:"sync-interval-spread"` // Interval between retries after a failed sync. RetryInterval time.Duration `mapstructure:"retry-interval"` // Interval between rechecking for peers after no synchronization peers were @@ -91,6 +95,7 @@ func DefaultConfig() MultiPeerReconcilerConfig { MaxFullDiff: 10000, MaxSyncDiff: 100, SyncInterval: 5 * time.Minute, + SyncIntervalSpread: 0.5, RetryInterval: 1 * time.Minute, NoPeersRecheckInterval: 30 * time.Second, SplitSyncGracePeriod: time.Minute, @@ -259,7 +264,11 @@ func (mpr *MultiPeerReconciler) needSplitSync(s syncability) bool { } func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Peer) error { + if len(syncPeers) == 0 { + return errors.New("no peers to sync against") + } var eg errgroup.Group + numSucceeded := 0 for _, p := range syncPeers { syncer, err := mpr.syncBase.Derive(ctx, p) if err != nil { @@ -270,6 +279,7 @@ func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Pe err := syncer.Sync(ctx, nil, nil) switch { case err == nil: + numSucceeded++ mpr.sl.NoteSync() case errors.Is(err, context.Canceled): return err @@ -281,7 +291,13 @@ func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Pe return nil }) } - return eg.Wait() + if err := eg.Wait(); err != nil { + return err + } + if numSucceeded == 0 { + return errors.New("all syncs failed") + } + return nil } func (mpr *MultiPeerReconciler) syncOnce(ctx context.Context, lastWasSplit bool) (full bool, err error) { @@ -341,7 +357,7 @@ func (mpr *MultiPeerReconciler) syncOnce(ctx context.Context, lastWasSplit bool) } // Run runs the MultiPeerReconciler. -func (mpr *MultiPeerReconciler) Run(ctx context.Context) error { +func (mpr *MultiPeerReconciler) Run(ctx context.Context, kickCh chan struct{}) error { // The point of using split sync, which syncs different key ranges against // different peers, vs full sync which syncs the full key range against different // peers, is: @@ -379,7 +395,9 @@ func (mpr *MultiPeerReconciler) Run(ctx context.Context) error { lastWasSplit := false LOOP: for { - interval := mpr.cfg.SyncInterval + interval := time.Duration( + float64(mpr.cfg.SyncInterval) * + (1 + mpr.cfg.SyncIntervalSpread*(rand.Float64()*2-1))) full, err = mpr.syncOnce(ctx, lastWasSplit) if err != nil { if errors.Is(err, context.Canceled) { @@ -402,6 +420,7 @@ LOOP: err = ctx.Err() break LOOP case <-mpr.clock.After(interval): + case <-kickCh: } } // The loop is only exited upon context cancellation. diff --git a/sync2/multipeer/multipeer_test.go b/sync2/multipeer/multipeer_test.go index 353fa705fd..f0369f48ae 100644 --- a/sync2/multipeer/multipeer_test.go +++ b/sync2/multipeer/multipeer_test.go @@ -66,6 +66,7 @@ type multiPeerSyncTester struct { reconciler *multipeer.MultiPeerReconciler cancel context.CancelFunc eg errgroup.Group + kickCh chan struct{} // EXPECT() calls should not be done concurrently // https://github.com/golang/mock/issues/533#issuecomment-821537840 mtx sync.Mutex @@ -80,10 +81,13 @@ func newMultiPeerSyncTester(t *testing.T, addPeers int) *multiPeerSyncTester { syncRunner: NewMocksyncRunner(ctrl), peers: peers.New(), clock: clockwork.NewFakeClock().(fakeClock), + kickCh: make(chan struct{}, 1), } cfg := multipeer.DefaultConfig() - cfg.SyncInterval = time.Minute + cfg.SyncInterval = 40 * time.Second + cfg.SyncIntervalSpread = 0.1 cfg.SyncPeerCount = numSyncPeers + cfg.RetryInterval = 5 * time.Second cfg.MinSplitSyncPeers = 2 cfg.MinSplitSyncCount = 90 cfg.MaxFullDiff = 20 @@ -110,7 +114,7 @@ func (mt *multiPeerSyncTester) addPeers(n int) []p2p.Peer { func (mt *multiPeerSyncTester) start() context.Context { var ctx context.Context ctx, mt.cancel = context.WithTimeout(context.Background(), 10*time.Second) - mt.eg.Go(func() error { return mt.reconciler.Run(ctx) }) + mt.eg.Go(func() error { return mt.reconciler.Run(ctx, mt.kickCh) }) mt.Cleanup(func() { mt.cancel() if err := mt.eg.Wait(); err != nil { @@ -120,6 +124,10 @@ func (mt *multiPeerSyncTester) start() context.Context { return ctx } +func (mt *multiPeerSyncTester) kick() { + mt.kickCh <- struct{}{} +} + func (mt *multiPeerSyncTester) expectProbe(times int, pr rangesync.ProbeResult) *peerList { var pl peerList mt.syncBase.EXPECT().Probe(gomock.Any(), gomock.Any()).DoAndReturn( @@ -180,7 +188,7 @@ func TestMultiPeerSync(t *testing.T) { mt := newMultiPeerSyncTester(t, 0) ctx := mt.start() mt.clock.BlockUntilContext(ctx, 1) - // Advance by sync interval. No peers yet + // Advance by sync interval (incl. spread). No peers yet mt.clock.Advance(time.Minute) mt.clock.BlockUntilContext(ctx, 1) // It is safe to do EXPECT() calls while the MultiPeerReconciler is blocked @@ -246,6 +254,34 @@ func TestMultiPeerSync(t *testing.T) { mt.syncBase.EXPECT().Wait() }) + t.Run("sync after kick", func(t *testing.T) { + mt := newMultiPeerSyncTester(t, 10) + mt.syncBase.EXPECT().Count().Return(100, nil).AnyTimes() + require.False(t, mt.reconciler.Synced()) + expect := func() { + pl := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{ + FP: "foo", + Count: 100, + Sim: 0.99, // high enough for full sync + }) + mt.expectFullSync(pl, numSyncPeers, 0) + mt.syncBase.EXPECT().Wait() + } + expect() + // first full sync happens immediately + ctx := mt.start() + mt.clock.BlockUntilContext(ctx, 1) + mt.satisfy() + for i := 0; i < numSyncs; i++ { + expect() + mt.kick() + mt.clock.BlockUntilContext(ctx, 1) + mt.satisfy() + } + require.True(t, mt.reconciler.Synced()) + mt.syncBase.EXPECT().Wait() + }) + t.Run("full sync, peers with low count ignored", func(t *testing.T) { mt := newMultiPeerSyncTester(t, 0) addedPeers := mt.addPeers(numSyncPeers) @@ -346,6 +382,28 @@ func TestMultiPeerSync(t *testing.T) { mt.syncBase.EXPECT().Wait() }) + t.Run("all peers failed during full sync", func(t *testing.T) { + mt := newMultiPeerSyncTester(t, 10) + mt.syncBase.EXPECT().Count().Return(100, nil).AnyTimes() + + pl := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) + mt.expectFullSync(pl, numSyncPeers, numSyncPeers) + mt.syncBase.EXPECT().Wait().AnyTimes() + + ctx := mt.start() + mt.clock.BlockUntilContext(ctx, 1) + mt.satisfy() + + pl = mt.expectProbe(numSyncPeers, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) + mt.expectFullSync(pl, numSyncPeers, 0) + // Retry should happen after mere 5 seconds as no peers have succeeded, no + // need to wait full sync interval. + mt.clock.Advance(5 * time.Second) + mt.satisfy() + + require.True(t, mt.reconciler.Synced()) + }) + t.Run("failed synced key handling during full sync", func(t *testing.T) { mt := newMultiPeerSyncTester(t, 10) mt.syncBase.EXPECT().Count().Return(100, nil).AnyTimes() diff --git a/sync2/multipeer/split_sync.go b/sync2/multipeer/split_sync.go index b084c30e0d..cbce8bd79a 100644 --- a/sync2/multipeer/split_sync.go +++ b/sync2/multipeer/split_sync.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/spacemeshos/go-spacemesh/fetch/peers" + "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/p2p" ) @@ -150,6 +151,8 @@ func (s *splitSync) handleSyncResult(r syncResult) error { s.syncPeers = append(s.syncPeers, r.s.Peer()) s.numRemaining-- s.logger.Debug("peer synced successfully", + log.ZShortStringer("x", sr.X), + log.ZShortStringer("y", sr.Y), zap.Stringer("peer", r.s.Peer()), zap.Int("numPeers", s.numPeers), zap.Int("numRemaining", s.numRemaining), @@ -199,8 +202,18 @@ func (s *splitSync) Sync(ctx context.Context) error { } select { case sr = <-s.slowRangeCh: - // push this syncRange to the back of the queue - s.sq.Update(sr, s.clock.Now()) + // Push this syncRange to the back of the queue. + // There's some chance that the peer managed to complete + // the sync while the range was still sitting in the + // channel, so we double-check if it's done. + if !sr.Done { + s.logger.Debug("slow peer, reassigning the range", + log.ZShortStringer("x", sr.X), log.ZShortStringer("y", sr.Y)) + s.sq.Update(sr, s.clock.Now()) + } else { + s.logger.Debug("slow peer, NOT reassigning the range: DONE", + log.ZShortStringer("x", sr.X), log.ZShortStringer("y", sr.Y)) + } case <-syncCtx.Done(): return syncCtx.Err() case r := <-s.resCh: @@ -210,5 +223,16 @@ func (s *splitSync) Sync(ctx context.Context) error { } } } - return s.eg.Wait() + // Stop late peers that didn't manage to sync their ranges in time. + // The ranges were already reassigned to other peers and successfully + // synced by this point. + cancel() + err := s.eg.Wait() + if s.numRemaining == 0 { + // If all the ranges are synced, the split sync is considered successful + // even if some peers failed to sync their ranges, so that these ranges + // got synced by other peers. + return nil + } + return err } diff --git a/sync2/multipeer/split_sync_test.go b/sync2/multipeer/split_sync_test.go index 195cf45561..2ddb092010 100644 --- a/sync2/multipeer/split_sync_test.go +++ b/sync2/multipeer/split_sync_test.go @@ -3,6 +3,7 @@ package multipeer_test import ( "context" "errors" + "fmt" "sync" "testing" "time" @@ -13,7 +14,6 @@ import ( "go.uber.org/zap/zaptest" "golang.org/x/sync/errgroup" - "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/sync2/multipeer" @@ -22,9 +22,9 @@ import ( type splitSyncTester struct { testing.TB - + ctrl *gomock.Controller syncPeers []p2p.Peer - clock clockwork.Clock + clock clockwork.FakeClock mtx sync.Mutex fail map[hexRange]bool expPeerRanges map[hexRange]int @@ -56,6 +56,8 @@ var tstRanges = []hexRange{ func newTestSplitSync(t testing.TB) *splitSyncTester { ctrl := gomock.NewController(t) tst := &splitSyncTester{ + TB: t, + ctrl: ctrl, syncPeers: make([]p2p.Peer, 4), clock: clockwork.NewFakeClock(), fail: make(map[hexRange]bool), @@ -70,42 +72,7 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { peers: peers.New(), } for n := range tst.syncPeers { - tst.syncPeers[n] = p2p.Peer(types.RandomBytes(20)) - } - for index, p := range tst.syncPeers { - tst.syncBase.EXPECT(). - Derive(gomock.Any(), p). - DoAndReturn(func(_ context.Context, peer p2p.Peer) (multipeer.PeerSyncer, error) { - s := NewMockPeerSyncer(ctrl) - s.EXPECT().Peer().Return(p).AnyTimes() - // TODO: do better job at tracking Release() calls - s.EXPECT().Release().AnyTimes() - s.EXPECT(). - Sync(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, x, y rangesync.KeyBytes) error { - tst.mtx.Lock() - defer tst.mtx.Unlock() - require.NotNil(t, x) - require.NotNil(t, y) - k := hexRange{x.String(), y.String()} - tst.peerRanges[k] = append(tst.peerRanges[k], peer) - count, found := tst.expPeerRanges[k] - require.True(t, found, "peer range not found: x %s y %s", x, y) - if tst.fail[k] { - t.Logf("ERR: peer %d x %s y %s", - index, x.String(), y.String()) - tst.fail[k] = false - return errors.New("injected fault") - } else { - t.Logf("OK: peer %d x %s y %s", - index, x.String(), y.String()) - tst.expPeerRanges[k] = count + 1 - } - return nil - }) - return s, nil - }). - AnyTimes() + tst.syncPeers[n] = p2p.Peer(fmt.Sprintf("peer%d", n)) } for _, p := range tst.syncPeers { tst.peers.Add(p) @@ -122,8 +89,45 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { return tst } +func (tst *splitSyncTester) expectPeerSync(p p2p.Peer) { + tst.syncBase.EXPECT(). + Derive(gomock.Any(), p). + DoAndReturn(func(_ context.Context, peer p2p.Peer) (multipeer.PeerSyncer, error) { + s := NewMockPeerSyncer(tst.ctrl) + s.EXPECT().Peer().Return(p).AnyTimes() + s.EXPECT(). + Sync(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, x, y rangesync.KeyBytes) error { + tst.mtx.Lock() + defer tst.mtx.Unlock() + require.NotNil(tst, x) + require.NotNil(tst, y) + k := hexRange{x.String(), y.String()} + tst.peerRanges[k] = append(tst.peerRanges[k], peer) + count, found := tst.expPeerRanges[k] + require.True(tst, found, "peer range not found: x %s y %s", x, y) + if tst.fail[k] { + tst.Logf("ERR: peer %s x %s y %s", + string(p), x.String(), y.String()) + tst.fail[k] = false + return errors.New("injected fault") + } else { + tst.Logf("OK: peer %s x %s y %s", + string(p), x.String(), y.String()) + tst.expPeerRanges[k] = count + 1 + } + return nil + }) + s.EXPECT().Release() + return s, nil + }).AnyTimes() +} + func TestSplitSync(t *testing.T) { tst := newTestSplitSync(t) + for _, p := range tst.syncPeers { + tst.expectPeerSync(p) + } var eg errgroup.Group eg.Go(func() error { return tst.splitSync.Sync(context.Background()) @@ -134,8 +138,11 @@ func TestSplitSync(t *testing.T) { } } -func TestSplitSyncRetry(t *testing.T) { +func TestSplitSync_Retry(t *testing.T) { tst := newTestSplitSync(t) + for _, p := range tst.syncPeers { + tst.expectPeerSync(p) + } tst.fail[tstRanges[1]] = true tst.fail[tstRanges[2]] = true var eg errgroup.Group @@ -148,3 +155,46 @@ func TestSplitSyncRetry(t *testing.T) { require.Equal(t, 1, count, "peer range not synced: x %s y %s", pr[0], pr[1]) } } + +func TestSplitSync_SlowPeers(t *testing.T) { + tst := newTestSplitSync(t) + + for _, p := range tst.syncPeers[:2] { + tst.expectPeerSync(p) + } + + for _, p := range tst.syncPeers[2:] { + tst.syncBase.EXPECT(). + Derive(gomock.Any(), p). + DoAndReturn(func(_ context.Context, peer p2p.Peer) (multipeer.PeerSyncer, error) { + s := NewMockPeerSyncer(tst.ctrl) + s.EXPECT().Peer().Return(p).AnyTimes() + s.EXPECT(). + Sync(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, x, y rangesync.KeyBytes) error { + <-ctx.Done() + return nil + }) + s.EXPECT().Release() + return s, nil + }) + } + + var eg errgroup.Group + eg.Go(func() error { + return tst.splitSync.Sync(context.Background()) + }) + + require.Eventually(t, func() bool { + tst.mtx.Lock() + defer tst.mtx.Unlock() + return len(tst.peerRanges) == 2 + }, 10*time.Millisecond, time.Millisecond) + // Make sure all 4 grace period timers are started. + tst.clock.BlockUntil(4) + tst.clock.Advance(time.Minute) + require.NoError(t, eg.Wait()) + for pr, count := range tst.expPeerRanges { + require.Equal(t, 1, count, "bad sync count: x %s y %s", pr[0], pr[1]) + } +} diff --git a/sync2/p2p.go b/sync2/p2p.go index 4310adff2c..ce3d16f7c4 100644 --- a/sync2/p2p.go +++ b/sync2/p2p.go @@ -22,9 +22,13 @@ import ( type Config struct { rangesync.RangeSetReconcilerConfig `mapstructure:",squash"` multipeer.MultiPeerReconcilerConfig `mapstructure:",squash"` - EnableActiveSync bool `mapstructure:"enable-active-sync"` - TrafficLimit int `mapstructure:"traffic-limit"` - MessageLimit int `mapstructure:"message-limit"` + TrafficLimit int `mapstructure:"traffic-limit"` + MessageLimit int `mapstructure:"message-limit"` + MaxDepth int `mapstructure:"max-depth"` + BatchSize int `mapstructure:"batch-size"` + MaxAttempts int `mapstructure:"max-attempts"` + MaxBatchRetries int `mapstructure:"max-batch-retries"` + FailedBatchDelay time.Duration `mapstructure:"failed-batch-delay"` } // DefaultConfig returns the default configuration for the P2PHashSync. @@ -34,20 +38,27 @@ func DefaultConfig() Config { MultiPeerReconcilerConfig: multipeer.DefaultConfig(), TrafficLimit: 200_000_000, MessageLimit: 20_000_000, + MaxDepth: 24, + BatchSize: 1000, + MaxAttempts: 3, + MaxBatchRetries: 3, + FailedBatchDelay: 10 * time.Second, } } // P2PHashSync is handles the synchronization of a local OrderedSet against other peers. type P2PHashSync struct { - logger *zap.Logger - cfg Config - os rangesync.OrderedSet - syncBase multipeer.SyncBase - reconciler *multipeer.MultiPeerReconciler - cancel context.CancelFunc - eg errgroup.Group - start sync.Once - running atomic.Bool + logger *zap.Logger + cfg Config + enableActiveSync bool + os rangesync.OrderedSet + syncBase multipeer.SyncBase + reconciler *multipeer.MultiPeerReconciler + cancel context.CancelFunc + eg errgroup.Group + startOnce sync.Once + running atomic.Bool + kickCh chan struct{} } // NewP2PHashSync creates a new P2PHashSync. @@ -56,23 +67,24 @@ func NewP2PHashSync( d *rangesync.Dispatcher, name string, os rangesync.OrderedSet, - keyLen, maxDepth int, + keyLen int, peers *peers.Peers, handler multipeer.SyncKeyHandler, cfg Config, - requester rangesync.Requester, + enableActiveSync bool, ) *P2PHashSync { s := &P2PHashSync{ - logger: logger, - os: os, - cfg: cfg, + logger: logger, + os: os, + cfg: cfg, + kickCh: make(chan struct{}, 1), + enableActiveSync: enableActiveSync, } - // var ps multipeer.PairwiseSyncer - ps := rangesync.NewPairwiseSetSyncer(logger, requester, name, cfg.RangeSetReconcilerConfig) + ps := rangesync.NewPairwiseSetSyncer(logger, d, name, cfg.RangeSetReconcilerConfig) s.syncBase = multipeer.NewSetSyncBase(logger, ps, s.os, handler) s.reconciler = multipeer.NewMultiPeerReconciler( logger, cfg.MultiPeerReconcilerConfig, - s.syncBase, peers, keyLen, maxDepth) + s.syncBase, peers, keyLen, cfg.MaxDepth) d.Register(name, s.serve) return s } @@ -99,6 +111,9 @@ func (s *P2PHashSync) Set() rangesync.OrderedSet { // Load loads the OrderedSet from the underlying storage. func (s *P2PHashSync) Load() error { + if s.os.Loaded() { + return nil + } s.logger.Info("loading the set") start := time.Now() // We pre-load the set to avoid waiting for it to load during a @@ -113,30 +128,51 @@ func (s *P2PHashSync) Load() error { s.logger.Info("done loading the set", zap.Duration("elapsed", time.Since(start)), zap.Int("count", info.Count), - zap.Stringer("fingerprint", info.Fingerprint)) + zap.Stringer("fingerprint", info.Fingerprint), + zap.Int("maxDepth", s.cfg.MaxDepth)) return nil } -// Start starts the multi-peer reconciler. -func (s *P2PHashSync) Start() { - if !s.cfg.EnableActiveSync { - s.logger.Info("active sync is disabled") - return - } +func (s *P2PHashSync) start() (isWaiting bool) { s.running.Store(true) - s.start.Do(func() { - s.eg.Go(func() error { - defer s.running.Store(false) - var ctx context.Context - ctx, s.cancel = context.WithCancel(context.Background()) - return s.reconciler.Run(ctx) - }) + isWaiting = true + s.startOnce.Do(func() { + isWaiting = false + if s.enableActiveSync { + s.eg.Go(func() error { + defer s.running.Store(false) + var ctx context.Context + ctx, s.cancel = context.WithCancel(context.Background()) + return s.reconciler.Run(ctx, s.kickCh) + }) + return + } else { + s.logger.Info("active syncv2 is disabled") + return + } }) + return isWaiting +} + +// Start starts the multi-peer reconciler if it is not already running. +func (s *P2PHashSync) Start() { + s.start() +} + +// StartAndSync starts the multi-peer reconciler if it is not already running, and waits +// until the local OrderedSet is in sync with the peers. +func (s *P2PHashSync) StartAndSync(ctx context.Context) error { + if s.start() { + // If the multipeer reconciler is waiting for sync, we kick it to start + // the sync so as not to wait for the next scheduled sync interval. + s.kickCh <- struct{}{} + } + return s.WaitForSync(ctx) } // Stop stops the multi-peer reconciler. func (s *P2PHashSync) Stop() { - if !s.cfg.EnableActiveSync || !s.running.Load() { + if !s.enableActiveSync || !s.running.Load() { return } if s.cancel != nil { diff --git a/sync2/p2p_test.go b/sync2/p2p_test.go index 54b63c4a60..a0a21571d1 100644 --- a/sync2/p2p_test.go +++ b/sync2/p2p_test.go @@ -92,8 +92,8 @@ func TestP2P(t *testing.T) { } } cfg := sync2.DefaultConfig() - cfg.EnableActiveSync = true cfg.SyncInterval = 100 * time.Millisecond + cfg.MaxDepth = maxDepth host := mesh.Hosts()[n] handlers[n] = &fakeHandler{ mtx: &mtx, @@ -109,7 +109,7 @@ func TestP2P(t *testing.T) { eg.Go(func() error { return srv.Run(ctx) }) hs[n] = sync2.NewP2PHashSync( logger.Named(fmt.Sprintf("node%d", n)), - d, "test", &os, keyLen, maxDepth, ps, handlers[n], cfg, srv) + d, "test", &os, keyLen, ps, handlers[n], cfg, true) require.NoError(t, hs[n].Load()) is := hs[n].Set().(*rangesync.DumbSet) is.SetAllowMultiReceive(true) From 4984889a50f329373235e83176f57959c4769f57 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 11 Nov 2024 23:46:08 +0400 Subject: [PATCH 05/22] sync2: dbset: fix connection leak in non-loaded DBSets --- sync2/dbset/dbset.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index 564c6afb7a..bf015b5b1b 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -291,12 +291,14 @@ func (d *DBSet) Recent(since time.Time) (rangesync.SeqResult, int) { func (d *DBSet) Release() { d.loadMtx.Lock() defer d.loadMtx.Unlock() - if d.ft == nil { - return + if d.ft != nil { + d.ft.Release() + d.ft = nil } - d.ft.Release() - d.ft = nil - if c, ok := d.db.(sql.Connection); ok { - c.Release() + if d.db != nil { + if c, ok := d.db.(sql.Connection); ok { + c.Release() + } + d.db = nil } } From bb31cc63e54593ebae7d82b64148cb769d1f5c46 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 13 Nov 2024 19:20:26 +0400 Subject: [PATCH 06/22] sql: revert removing Rollback method of the Migration interface --- sql/database_test.go | 3 ++ sql/interface.go | 1 + sql/migrations.go | 5 +++ sql/mocks.go | 38 +++++++++++++++++++ sql/schema.go | 10 ++++- .../migrations/state_0021_migration.go | 4 ++ .../migrations/state_0025_migration.go | 4 ++ 7 files changed, 64 insertions(+), 1 deletion(-) diff --git a/sql/database_test.go b/sql/database_test.go index 899ef2b493..caf15fff25 100644 --- a/sql/database_test.go +++ b/sql/database_test.go @@ -93,6 +93,8 @@ func Test_Migration_Rollback(t *testing.T) { migration1.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(nil) migration2.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(errors.New("migration 2 failed")) + migration2.EXPECT().Rollback().Return(nil) + dbFile := filepath.Join(t.TempDir(), "test.sql") _, err := Open("file:"+dbFile, WithDatabaseSchema(&Schema{ @@ -127,6 +129,7 @@ func Test_Migration_Rollback_Only_NewMigrations(t *testing.T) { migration2.EXPECT().Name().Return("test").AnyTimes() migration2.EXPECT().Order().Return(2).AnyTimes() migration2.EXPECT().Apply(gomock.Any(), gomock.Any()).Return(errors.New("migration 2 failed")) + migration2.EXPECT().Rollback().Return(nil) _, err = Open("file:"+dbFile, WithLogger(logger), diff --git a/sql/interface.go b/sql/interface.go index 14efae19c0..6728a6b388 100644 --- a/sql/interface.go +++ b/sql/interface.go @@ -15,6 +15,7 @@ type Migration interface { // Apply applies the migration. Apply(db Executor, logger *zap.Logger) error // Name returns the name of the migration. + Rollback() error Name() string // Order returns the sequential number of the migration. Order() int diff --git a/sql/migrations.go b/sql/migrations.go index b601f30685..92d1a1d516 100644 --- a/sql/migrations.go +++ b/sql/migrations.go @@ -89,6 +89,11 @@ func (m *sqlMigration) Order() int { return m.order } +func (sqlMigration) Rollback() error { + // handled by the DB itself + return nil +} + func version(db Executor) (int, error) { var current int if _, err := db.Exec("PRAGMA user_version;", nil, func(stmt *Statement) bool { diff --git a/sql/mocks.go b/sql/mocks.go index ae5e3413e2..2be336b646 100644 --- a/sql/mocks.go +++ b/sql/mocks.go @@ -216,3 +216,41 @@ func (c *MockMigrationOrderCall) DoAndReturn(f func() int) *MockMigrationOrderCa c.Call = c.Call.DoAndReturn(f) return c } + +// Rollback mocks base method. +func (m *MockMigration) Rollback() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Rollback") + ret0, _ := ret[0].(error) + return ret0 +} + +// Rollback indicates an expected call of Rollback. +func (mr *MockMigrationMockRecorder) Rollback() *MockMigrationRollbackCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rollback", reflect.TypeOf((*MockMigration)(nil).Rollback)) + return &MockMigrationRollbackCall{Call: call} +} + +// MockMigrationRollbackCall wrap *gomock.Call +type MockMigrationRollbackCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockMigrationRollbackCall) Return(arg0 error) *MockMigrationRollbackCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockMigrationRollbackCall) Do(f func() error) *MockMigrationRollbackCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockMigrationRollbackCall) DoAndReturn(f func() error) *MockMigrationRollbackCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/sql/schema.go b/sql/schema.go index 144ad43eff..aa05416206 100644 --- a/sql/schema.go +++ b/sql/schema.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" "os" @@ -145,13 +146,20 @@ func (s *Schema) Migrate(logger *zap.Logger, db Database, before, vacuumState in db.Intercept("logQueries", logQueryInterceptor(logger)) defer db.RemoveInterceptor("logQueries") } - for _, m := range s.Migrations { + for i, m := range s.Migrations { if m.Order() <= before { continue } if err := db.WithTxImmediate(context.Background(), func(tx Transaction) error { if _, ok := s.skipMigration[m.Order()]; !ok { if err := m.Apply(tx, logger); err != nil { + for j := i; j >= 0 && s.Migrations[j].Order() > before; j-- { + if e := s.Migrations[j].Rollback(); e != nil { + err = errors.Join(err, fmt.Errorf("rollback %s: %w", m.Name(), e)) + break + } + } + return fmt.Errorf("apply %s: %w", m.Name(), err) } } diff --git a/sql/statesql/migrations/state_0021_migration.go b/sql/statesql/migrations/state_0021_migration.go index 874e558d8a..b88471fbeb 100644 --- a/sql/statesql/migrations/state_0021_migration.go +++ b/sql/statesql/migrations/state_0021_migration.go @@ -32,6 +32,10 @@ func (*migration0021) Order() int { return 21 } +func (*migration0021) Rollback() error { + return nil +} + func (m *migration0021) Apply(db sql.Executor, logger *zap.Logger) error { if err := m.applySql(db); err != nil { return err diff --git a/sql/statesql/migrations/state_0025_migration.go b/sql/statesql/migrations/state_0025_migration.go index c3869d74c6..71fee844fa 100644 --- a/sql/statesql/migrations/state_0025_migration.go +++ b/sql/statesql/migrations/state_0025_migration.go @@ -40,6 +40,10 @@ func (*migration0025) Order() int { return 25 } +func (*migration0025) Rollback() error { + return nil +} + func (m *migration0025) Apply(db sql.Executor, logger *zap.Logger) error { updates := map[types.NodeID][]byte{} From 3e5a4014c74296c294bc87f95854d45ce25f56c0 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 13 Nov 2024 19:33:10 +0400 Subject: [PATCH 07/22] sql: remove Database.Connection() method, keep WithConnection() --- sql/database.go | 55 +++++++++++--------------------------------- sql/database_test.go | 19 +++------------ 2 files changed, 17 insertions(+), 57 deletions(-) diff --git a/sql/database.go b/sql/database.go index 1620f633fc..c12ab1a87a 100644 --- a/sql/database.go +++ b/sql/database.go @@ -608,20 +608,15 @@ type Database interface { // and rolls it back otherwise. // If the context is canceled, the currently running SQL statement is interrupted. WithTxImmediate(ctx context.Context, exec func(Transaction) error) error - // Connection returns a connection from the database pool. + // WithConnection executes the provided function with a connection from the + // database pool. // If many queries are to be executed in a row, but there's no need for an // explicit transaction which may be long-running and thus block // WAL checkpointing, it may be preferable to use a single connection for // it to avoid database pool overhead. - // The connection needs to be always returned to the pool by calling its Release - // method. - // If the context is canceled, the currently running SQL statement is interrupted. - Connection(ctx context.Context) (Connection, error) - // WithConnection executes the provided function with a connection from the - // database pool. // The connection is released back to the pool after the function returns. // If the context is canceled, the currently running SQL statement is interrupted. - WithConnection(ctx context.Context, exec func(Connection) error) error + WithConnection(ctx context.Context, exec func(Executor) error) error // Intercept adds an interceptor function to the database. The interceptor // functions are invoked upon each query on the database, including queries // executed within transactions. @@ -642,13 +637,6 @@ type Transaction interface { Release() error } -// Connection represents a database connection. -type Connection interface { - Executor - // Release releases the connection back to the connection pool. - Release() -} - type sqliteDatabase struct { *queryCache pool *sqlitex.Pool @@ -819,28 +807,21 @@ func (db *sqliteDatabase) Close() error { return nil } -// Connection implements Database. -func (db *sqliteDatabase) Connection(ctx context.Context) (Connection, error) { +// WithConnection implements Database. +func (db *sqliteDatabase) WithConnection(ctx context.Context, exec func(Executor) error) error { if db.closed { - return nil, ErrClosed + return ErrClosed } conCtx, cancel := context.WithCancel(ctx) conn := db.getConn(conCtx) - if conn == nil { + defer func() { cancel() - return nil, ErrNoConnection - } - return &sqliteConn{queryCache: db.queryCache, db: db, conn: conn, freeConn: cancel}, nil -} - -// WithConnection implements Database. -func (db *sqliteDatabase) WithConnection(ctx context.Context, exec func(Connection) error) error { - conn, err := db.Connection(ctx) - if err != nil { - return err + db.pool.Put(conn) + }() + if conn == nil { + return ErrNoConnection } - defer conn.Release() - return exec(conn) + return exec(&sqliteConn{queryCache: db.queryCache, db: db, conn: conn}) } // Intercept adds an interceptor function to the database. The interceptor functions @@ -1166,16 +1147,8 @@ func (tx *sqliteTx) Exec(query string, encoder Encoder, decoder Decoder) (int, e type sqliteConn struct { *queryCache - db *sqliteDatabase - conn *sqlite.Conn - freeConn func() -} - -var _ Connection = &sqliteConn{} - -func (c *sqliteConn) Release() { - c.freeConn() - c.db.pool.Put(c.conn) + db *sqliteDatabase + conn *sqlite.Conn } func (c *sqliteConn) Exec(query string, encoder Encoder, decoder Decoder) (int, error) { diff --git a/sql/database_test.go b/sql/database_test.go index caf15fff25..60b81ff9cf 100644 --- a/sql/database_test.go +++ b/sql/database_test.go @@ -641,22 +641,9 @@ func TestExclusive(t *testing.T) { func TestConnection(t *testing.T) { db := InMemoryTest(t) - c, err := db.Connection(context.Background()) - require.NoError(t, err) var r int - n, err := c.Exec("select ?", func(stmt *Statement) { - stmt.BindInt64(1, 42) - }, func(stmt *Statement) bool { - r = stmt.ColumnInt(0) - return true - }) - require.NoError(t, err) - require.Equal(t, 1, n) - require.Equal(t, 42, r) - c.Release() - - require.NoError(t, db.WithConnection(context.Background(), func(c Connection) error { - n, err := c.Exec("select ?", func(stmt *Statement) { + require.NoError(t, db.WithConnection(context.Background(), func(ex Executor) error { + n, err := ex.Exec("select ?", func(stmt *Statement) { stmt.BindInt64(1, 42) }, func(stmt *Statement) bool { r = stmt.ColumnInt(0) @@ -668,7 +655,7 @@ func TestConnection(t *testing.T) { return nil })) - require.Error(t, db.WithConnection(context.Background(), func(c Connection) error { + require.Error(t, db.WithConnection(context.Background(), func(Executor) error { return errors.New("error") })) } From f5cae06b7e8101ae00aee72f2c1c6f88ca8ebafd Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 13 Nov 2024 23:22:36 +0400 Subject: [PATCH 08/22] sql: allow multiple connections to in-memory database --- sql/database.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/database.go b/sql/database.go index c12ab1a87a..90931ff611 100644 --- a/sql/database.go +++ b/sql/database.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "math/rand/v2" "net/url" "os" "strings" @@ -223,8 +224,12 @@ type Opt func(c *conf) // OpenInMemory creates an in-memory database. func OpenInMemory(opts ...Opt) (*sqliteDatabase, error) { - opts = append(opts, WithConnections(1), withForceFresh()) - return Open("file::memory:?mode=memory", opts...) + opts = append(opts, withForceFresh()) + // Unique uri is needed to avoid sharing the same in-memory database, + // while allowing multiple connections to the same database. + uri := fmt.Sprintf("file:mem-%d-%d?mode=memory&cache=shared", + rand.Uint64(), rand.Uint64()) + return Open(uri, opts...) } // InMemory creates an in-memory database for testing and panics if From 9e585e508263d06dabb2e3a3bac2dfb6602e9a9b Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 13 Nov 2024 23:23:10 +0400 Subject: [PATCH 09/22] sync2: fixup for temporary OrderedSet copies --- sync2/dbset/dbset.go | 48 +++----- sync2/dbset/dbset_test.go | 166 ++++++++++++++-------------- sync2/dbset/p2p_test.go | 123 +++++++++++---------- sync2/multipeer/interface.go | 8 +- sync2/multipeer/mocks_test.go | 113 +++++++------------ sync2/multipeer/multipeer.go | 30 ++--- sync2/multipeer/multipeer_test.go | 12 +- sync2/multipeer/setsyncbase.go | 44 ++++---- sync2/multipeer/setsyncbase_test.go | 161 ++++++++++++++------------- sync2/multipeer/split_sync.go | 38 +++---- sync2/multipeer/split_sync_test.go | 11 +- sync2/p2p.go | 15 +-- sync2/p2p_test.go | 57 ++++++---- sync2/rangesync/dumbset.go | 8 +- sync2/rangesync/interface.go | 16 +-- sync2/rangesync/mocks/mocks.go | 99 ++++++----------- 16 files changed, 438 insertions(+), 511 deletions(-) diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index bf015b5b1b..795ca24f1f 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -227,29 +227,16 @@ func (d *DBSet) Advance() error { return d.snapshot.LoadSinceSnapshot(d.db, oldSnapshot, d.handleIDfromDB) } -// Copy creates a copy of the DBSet. +// WithCopy invokes the specified function, passing it a temporary copy of the DBSet. // Implements rangesync.OrderedSet. -func (d *DBSet) Copy(ctx context.Context, syncScope bool) (rangesync.OrderedSet, error) { +func (d *DBSet) WithCopy(ctx context.Context, toCall func(rangesync.OrderedSet) error) error { if err := d.EnsureLoaded(); err != nil { - return nil, fmt.Errorf("loading DBSet: %w", err) + return fmt.Errorf("loading DBSet: %w", err) } d.loadMtx.Lock() - defer d.loadMtx.Unlock() ft := d.ft.Clone().(*fptree.FPTree) - ex := d.db - if syncScope { - db, ok := d.db.(sql.Database) - if ok { - // We might want to pass a real context here, but FPTree relies on - var err error - ex, err = db.Connection(context.Background()) - if err != nil { - return nil, fmt.Errorf("get connection: %w", err) - } - } - } - return &DBSet{ - db: ex, + ds := &DBSet{ + db: d.db, ft: ft, st: d.st, snapshot: d.snapshot, @@ -257,7 +244,18 @@ func (d *DBSet) Copy(ctx context.Context, syncScope bool) (rangesync.OrderedSet, maxDepth: d.maxDepth, dbStore: d.dbStore, received: maps.Clone(d.received), - }, nil + } + d.loadMtx.Unlock() + defer ds.release() + db, ok := d.db.(sql.Database) + if ok { + return db.WithConnection(ctx, func(ex sql.Executor) error { + ds.db = ex + return toCall(ds) + }) + } else { + return toCall(ds) + } } // Has returns true if the DBSet contains the given item. @@ -286,19 +284,9 @@ func (d *DBSet) Recent(since time.Time) (rangesync.SeqResult, int) { return d.dbStore.Since(make(rangesync.KeyBytes, d.keyLen), since.UnixNano()) } -// Release releases resources associated with the DBSet. -// Implements rangesync.OrderedSet. -func (d *DBSet) Release() { - d.loadMtx.Lock() - defer d.loadMtx.Unlock() +func (d *DBSet) release() { if d.ft != nil { d.ft.Release() d.ft = nil } - if d.db != nil { - if c, ok := d.db.(sql.Connection); ok { - c.Release() - } - d.db = nil - } } diff --git a/sync2/dbset/dbset_test.go b/sync2/dbset/dbset_test.go index 6dd60582be..7970cb4830 100644 --- a/sync2/dbset/dbset_test.go +++ b/sync2/dbset/dbset_test.go @@ -38,7 +38,6 @@ func TestDBSet_Empty(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() empty, err := s.Empty() require.NoError(t, err) require.True(t, empty) @@ -82,7 +81,6 @@ func TestDBSet(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() require.Equal(t, "0000000000000000000000000000000000000000000000000000000000000000", firstKey(t, s.Items()).String()) has, err := s.Has( @@ -186,7 +184,6 @@ func TestDBSet_Receive(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() require.Equal(t, "0000000000000000000000000000000000000000000000000000000000000000", firstKey(t, s.Items()).String()) @@ -218,41 +215,42 @@ func TestDBSet_Copy(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() require.Equal(t, "0000000000000000000000000000000000000000000000000000000000000000", firstKey(t, s.Items()).String()) - copy, err := s.Copy(context.Background(), false) - require.NoError(t, err) + require.NoError(t, s.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { + info, err := copy.GetRangeInfo(ids[2], ids[0]) + require.NoError(t, err) + require.Equal(t, 2, info.Count) + require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[2], firstKey(t, info.Items)) - info, err := copy.GetRangeInfo(ids[2], ids[0]) - require.NoError(t, err) - require.Equal(t, 2, info.Count) - require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[2], firstKey(t, info.Items)) + newID := rangesync.MustParseHexKeyBytes( + "abcdef1234567890000000000000000000000000000000000000000000000000") + require.NoError(t, copy.Receive(newID)) - newID := rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000") - require.NoError(t, copy.Receive(newID)) + info, err = s.GetRangeInfo(ids[2], ids[0]) + require.NoError(t, err) + require.Equal(t, 2, info.Count) + require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[2], firstKey(t, info.Items)) - info, err = s.GetRangeInfo(ids[2], ids[0]) - require.NoError(t, err) - require.Equal(t, 2, info.Count) - require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[2], firstKey(t, info.Items)) + items, err := s.Received().FirstN(100) + require.NoError(t, err) + require.Empty(t, items) - items, err := s.Received().FirstN(100) - require.NoError(t, err) - require.Empty(t, items) + info, err = s.GetRangeInfo(ids[2], ids[0]) + require.NoError(t, err) + require.Equal(t, 2, info.Count) + require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[2], firstKey(t, info.Items)) - info, err = s.GetRangeInfo(ids[2], ids[0]) - require.NoError(t, err) - require.Equal(t, 2, info.Count) - require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[2], firstKey(t, info.Items)) + items, err = copy.(*dbset.DBSet).Received().FirstN(100) + require.NoError(t, err) + require.Equal(t, []rangesync.KeyBytes{newID}, items) - items, err = copy.(*dbset.DBSet).Received().FirstN(100) - require.NoError(t, err) - require.Equal(t, []rangesync.KeyBytes{newID}, items) + return nil + })) } func TestDBItemStore_Advance(t *testing.T) { @@ -271,74 +269,76 @@ func TestDBItemStore_Advance(t *testing.T) { verifyDS := func(db sql.Database, os rangesync.OrderedSet) { require.NoError(t, os.EnsureLoaded()) - copy, err := os.Copy(context.Background(), false) - require.NoError(t, err) + require.NoError(t, os.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { + info, err := os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err := os.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + sqlstore.InsertDBItems(t, db, []rangesync.KeyBytes{ + rangesync.MustParseHexKeyBytes( + "abcdef1234567890000000000000000000000000000000000000000000000000"), + }) - sqlstore.InsertDBItems(t, db, []rangesync.KeyBytes{ - rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000"), - }) + info, err = os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = os.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + require.NoError(t, os.Advance()) - require.NoError(t, os.Advance()) + info, err = os.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 5, info.Count) + require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = os.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 5, info.Count) - require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + info, err = copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 4, info.Count) + require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) - info, err = copy.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 4, info.Count) - require.Equal(t, "cfe98ba54761032ddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + return nil + })) - copy1, err := os.Copy(context.Background(), false) - require.NoError(t, err) - info, err = copy1.GetRangeInfo(ids[0], ids[0]) - require.NoError(t, err) - require.Equal(t, 5, info.Count) - require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) - require.Equal(t, ids[0], firstKey(t, info.Items)) + require.NoError(t, os.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { + info, err := copy.GetRangeInfo(ids[0], ids[0]) + require.NoError(t, err) + require.Equal(t, 5, info.Count) + require.Equal(t, "642464b773377bbddddddddd", info.Fingerprint.String()) + require.Equal(t, ids[0], firstKey(t, info.Items)) + return nil + })) } t.Run("original DBSet", func(t *testing.T) { db := sqlstore.PopulateDB(t, testKeyLen, ids) dbSet := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer dbSet.Release() verifyDS(db, dbSet) }) t.Run("DBSet copy", func(t *testing.T) { db := sqlstore.PopulateDB(t, testKeyLen, ids) origSet := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer origSet.Release() - os, err := origSet.Copy(context.Background(), false) - require.NoError(t, err) - verifyDS(db, os) + require.NoError(t, origSet.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { + verifyDS(db, copy) + return nil + })) }) } @@ -356,7 +356,6 @@ func TestDBSet_Added(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - defer s.Release() requireEmpty(t, s.Received()) add := []rangesync.KeyBytes{ @@ -376,9 +375,10 @@ func TestDBSet_Added(t *testing.T) { rangesync.MustParseHexKeyBytes("4444444444444444444444444444444444444444444444444444444444444444"), }, added) - copy, err := s.Copy(context.Background(), false) - require.NoError(t, err) - added1, err := copy.(*dbset.DBSet).Received().FirstN(3) - require.NoError(t, err) - require.ElementsMatch(t, added, added1) + require.NoError(t, s.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { + added1, err := copy.(*dbset.DBSet).Received().FirstN(3) + require.NoError(t, err) + require.ElementsMatch(t, added, added1) + return nil + })) } diff --git a/sync2/dbset/p2p_test.go b/sync2/dbset/p2p_test.go index fc5f9842b0..c636c461ba 100644 --- a/sync2/dbset/p2p_test.go +++ b/sync2/dbset/p2p_test.go @@ -185,74 +185,81 @@ func runSync( cfg.MaxReconcDiff = 1 // always reconcile pssA := rangesync.NewPairwiseSetSyncerInternal(syncLogger.Named("sideA"), nil, "test", cfg, &tr, clock) d := rangesync.NewDispatcher(log) - copyA, err := setA.Copy(context.Background(), false) - require.NoError(t, err) - syncSetA := copyA.(*dbset.DBSet) - pssA.Register(d, syncSetA) - srv := server.New(mesh.Hosts()[0], proto, - d.Dispatch, - server.WithTimeout(time.Minute), - server.WithLog(log)) - - var eg errgroup.Group - - client := server.New(mesh.Hosts()[1], proto, - func(_ context.Context, _ p2p.Peer, _ []byte, _ io.ReadWriter) error { - return errors.New("client should not receive requests") - }, - server.WithTimeout(time.Minute), - server.WithLog(log)) - - defer func() { - cancel() - eg.Wait() - }() - eg.Go(func() error { - return srv.Run(ctx) - }) + require.NoError(t, setA.WithCopy(context.Background(), func(copyA rangesync.OrderedSet) error { + syncSetA := copyA.(*dbset.DBSet) + pssA.Register(d, syncSetA) + srv := server.New(mesh.Hosts()[0], proto, + d.Dispatch, + server.WithTimeout(time.Minute), + server.WithLog(log)) + + var eg errgroup.Group + + client := server.New(mesh.Hosts()[1], proto, + func(_ context.Context, _ p2p.Peer, _ []byte, _ io.ReadWriter) error { + return errors.New("client should not receive requests") + }, + server.WithTimeout(time.Minute), + server.WithLog(log)) + + defer func() { + cancel() + eg.Wait() + }() + eg.Go(func() error { + return srv.Run(ctx) + }) - // Wait for the server to activate - require.Eventually(t, func() bool { - for _, h := range mesh.Hosts() { - if len(h.Mux().Protocols()) == 0 { - return false + // Wait for the server to activate + require.Eventually(t, func() bool { + for _, h := range mesh.Hosts() { + if len(h.Mux().Protocols()) == 0 { + return false + } + } + return true + }, time.Second, 10*time.Millisecond) + + startTimer(t) + pssB := rangesync.NewPairwiseSetSyncerInternal( + syncLogger.Named("sideB"), client, "test", cfg, &tr, clock) + + tStart := time.Now() + require.NoError(t, setB.WithCopy(context.Background(), func(copyB rangesync.OrderedSet) error { + syncSetB := copyB.(*dbset.DBSet) + require.NoError(t, pssB.Sync(ctx, srvPeerID, syncSetB, x, x)) + stopTimer(t) + t.Logf("synced in %v, sent %d, recv %d", time.Since(tStart), pssB.Sent(), pssB.Received()) + + if !verify { + return nil } - } - return true - }, time.Second, 10*time.Millisecond) - startTimer(t) - pssB := rangesync.NewPairwiseSetSyncerInternal(syncLogger.Named("sideB"), client, "test", cfg, &tr, clock) + // Check that the sets are equal after we add the received items + addReceived(t, dbA, setA, syncSetA) + addReceived(t, dbB, setB, syncSetB) - tStart := time.Now() - copyB, err := setB.Copy(context.Background(), false) - require.NoError(t, err) - syncSetB := copyB.(*dbset.DBSet) - require.NoError(t, pssB.Sync(ctx, srvPeerID, syncSetB, x, x)) - stopTimer(t) - t.Logf("synced in %v, sent %d, recv %d", time.Since(tStart), pssB.Sent(), pssB.Received()) + require.Equal(t, receivedRecent, tr.receivedItems > 0) + require.Equal(t, sentRecent, tr.sentItems > 0) - if verify { - // Check that the sets are equal after we add the received items - addReceived(t, dbA, setA, syncSetA) - addReceived(t, dbB, setB, syncSetB) + if len(combined) == 0 { + return nil + } - require.Equal(t, receivedRecent, tr.receivedItems > 0) - require.Equal(t, sentRecent, tr.sentItems > 0) + actItemsA, err := setA.Items().Collect() + require.NoError(t, err) - if len(combined) == 0 { - return - } + actItemsB, err := setB.Items().Collect() + require.NoError(t, err) - actItemsA, err := setA.Items().Collect() - require.NoError(t, err) + assert.Equal(t, combined, actItemsA) + assert.Equal(t, actItemsA, actItemsB) - actItemsB, err := setB.Items().Collect() - require.NoError(t, err) + return nil + })) - assert.Equal(t, combined, actItemsA) - assert.Equal(t, actItemsA, actItemsB) - } + return nil + })) } func fooR(id string, seconds int) fooRow { diff --git a/sync2/multipeer/interface.go b/sync2/multipeer/interface.go index 6ecbed2135..b3a5236171 100644 --- a/sync2/multipeer/interface.go +++ b/sync2/multipeer/interface.go @@ -17,8 +17,9 @@ import ( type SyncBase interface { // Count returns the number of items in the set. Count() (int, error) - // Derive creates a Syncer for the specified peer. - Derive(ctx context.Context, p p2p.Peer) (PeerSyncer, error) + // WithPeerSyncer creates a Syncer for the specified peer and passes it to the specified function. + // When the function returns, the syncer is discarded, releasing the resources associated with it. + WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(PeerSyncer) error) error // Probe probes the specified peer, obtaining its set fingerprint, // the number of items and the similarity value. Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) @@ -34,9 +35,6 @@ type PeerSyncer interface { Sync(ctx context.Context, x, y rangesync.KeyBytes) error // Serve serves a synchronization request on the specified stream. Serve(ctx context.Context, stream io.ReadWriter) error - // Release releases the resources associated with the syncer. - // Calling Release on a syncer that is already released is a no-op. - Release() } // SyncKeyHandler is a handler for keys that are received from peers. diff --git a/sync2/multipeer/mocks_test.go b/sync2/multipeer/mocks_test.go index b9e3c79fd9..48c0401b5b 100644 --- a/sync2/multipeer/mocks_test.go +++ b/sync2/multipeer/mocks_test.go @@ -83,45 +83,6 @@ func (c *MockSyncBaseCountCall) DoAndReturn(f func() (int, error)) *MockSyncBase return c } -// Derive mocks base method. -func (m *MockSyncBase) Derive(ctx context.Context, p p2p.Peer) (multipeer.PeerSyncer, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Derive", ctx, p) - ret0, _ := ret[0].(multipeer.PeerSyncer) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Derive indicates an expected call of Derive. -func (mr *MockSyncBaseMockRecorder) Derive(ctx, p any) *MockSyncBaseDeriveCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Derive", reflect.TypeOf((*MockSyncBase)(nil).Derive), ctx, p) - return &MockSyncBaseDeriveCall{Call: call} -} - -// MockSyncBaseDeriveCall wrap *gomock.Call -type MockSyncBaseDeriveCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSyncBaseDeriveCall) Return(arg0 multipeer.PeerSyncer, arg1 error) *MockSyncBaseDeriveCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSyncBaseDeriveCall) Do(f func(context.Context, p2p.Peer) (multipeer.PeerSyncer, error)) *MockSyncBaseDeriveCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncBaseDeriveCall) DoAndReturn(f func(context.Context, p2p.Peer) (multipeer.PeerSyncer, error)) *MockSyncBaseDeriveCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Probe mocks base method. func (m *MockSyncBase) Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) { m.ctrl.T.Helper() @@ -199,6 +160,44 @@ func (c *MockSyncBaseWaitCall) DoAndReturn(f func() error) *MockSyncBaseWaitCall return c } +// WithPeerSyncer mocks base method. +func (m *MockSyncBase) WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(multipeer.PeerSyncer) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithPeerSyncer", ctx, p, toCall) + ret0, _ := ret[0].(error) + return ret0 +} + +// WithPeerSyncer indicates an expected call of WithPeerSyncer. +func (mr *MockSyncBaseMockRecorder) WithPeerSyncer(ctx, p, toCall any) *MockSyncBaseWithPeerSyncerCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithPeerSyncer", reflect.TypeOf((*MockSyncBase)(nil).WithPeerSyncer), ctx, p, toCall) + return &MockSyncBaseWithPeerSyncerCall{Call: call} +} + +// MockSyncBaseWithPeerSyncerCall wrap *gomock.Call +type MockSyncBaseWithPeerSyncerCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSyncBaseWithPeerSyncerCall) Return(arg0 error) *MockSyncBaseWithPeerSyncerCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSyncBaseWithPeerSyncerCall) Do(f func(context.Context, p2p.Peer, func(multipeer.PeerSyncer) error) error) *MockSyncBaseWithPeerSyncerCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSyncBaseWithPeerSyncerCall) DoAndReturn(f func(context.Context, p2p.Peer, func(multipeer.PeerSyncer) error) error) *MockSyncBaseWithPeerSyncerCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // MockPeerSyncer is a mock of PeerSyncer interface. type MockPeerSyncer struct { ctrl *gomock.Controller @@ -261,42 +260,6 @@ func (c *MockPeerSyncerPeerCall) DoAndReturn(f func() p2p.Peer) *MockPeerSyncerP return c } -// Release mocks base method. -func (m *MockPeerSyncer) Release() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Release") -} - -// Release indicates an expected call of Release. -func (mr *MockPeerSyncerMockRecorder) Release() *MockPeerSyncerReleaseCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockPeerSyncer)(nil).Release)) - return &MockPeerSyncerReleaseCall{Call: call} -} - -// MockPeerSyncerReleaseCall wrap *gomock.Call -type MockPeerSyncerReleaseCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockPeerSyncerReleaseCall) Return() *MockPeerSyncerReleaseCall { - c.Call = c.Call.Return() - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockPeerSyncerReleaseCall) Do(f func()) *MockPeerSyncerReleaseCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockPeerSyncerReleaseCall) DoAndReturn(f func()) *MockPeerSyncerReleaseCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Serve mocks base method. func (m *MockPeerSyncer) Serve(ctx context.Context, stream io.ReadWriter) error { m.ctrl.T.Helper() diff --git a/sync2/multipeer/multipeer.go b/sync2/multipeer/multipeer.go index 46b1d7cf48..f4487eb65e 100644 --- a/sync2/multipeer/multipeer.go +++ b/sync2/multipeer/multipeer.go @@ -261,22 +261,22 @@ func (mpr *MultiPeerReconciler) needSplitSync(s syncability) bool { func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Peer) error { var eg errgroup.Group for _, p := range syncPeers { - syncer, err := mpr.syncBase.Derive(ctx, p) - if err != nil { - return fmt.Errorf("derive syncer: %w", err) - } eg.Go(func() error { - defer syncer.Release() - err := syncer.Sync(ctx, nil, nil) - switch { - case err == nil: - mpr.sl.NoteSync() - case errors.Is(err, context.Canceled): - return err - default: - // failing to sync against a particular peer is not considered - // a fatal sync failure, so we just log the error - mpr.logger.Error("error syncing peer", zap.Stringer("peer", p), zap.Error(err)) + if err := mpr.syncBase.WithPeerSyncer(ctx, p, func(ps PeerSyncer) error { + err := ps.Sync(ctx, nil, nil) + switch { + case err == nil: + mpr.sl.NoteSync() + case errors.Is(err, context.Canceled): + return err + default: + // failing to sync against a particular peer is not considered + // a fatal sync failure, so we just log the error + mpr.logger.Error("error syncing peer", zap.Stringer("peer", p), zap.Error(err)) + } + return nil + }); err != nil { + return fmt.Errorf("sync %s: %w", p, err) } return nil }) diff --git a/sync2/multipeer/multipeer_test.go b/sync2/multipeer/multipeer_test.go index 353fa705fd..9e11733add 100644 --- a/sync2/multipeer/multipeer_test.go +++ b/sync2/multipeer/multipeer_test.go @@ -148,21 +148,23 @@ func (mt *multiPeerSyncTester) expectFullSync(pl *peerList, times, numFails int) // delegate to the real fullsync return mt.reconciler.FullSync(ctx, peers) }) - mt.syncBase.EXPECT().Derive(gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, p p2p.Peer) (multipeer.PeerSyncer, error) { + mt.syncBase.EXPECT().WithPeerSyncer(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, + p p2p.Peer, + toCall func(multipeer.PeerSyncer) error, + ) error { mt.mtx.Lock() defer mt.mtx.Unlock() require.Contains(mt, pl.get(), p) s := NewMockPeerSyncer(mt.ctrl) s.EXPECT().Peer().Return(p).AnyTimes() - // TODO: do better job at tracking Release() calls - s.EXPECT().Release().AnyTimes() expSync := s.EXPECT().Sync(gomock.Any(), gomock.Nil(), gomock.Nil()) if numFails != 0 { expSync.Return(errors.New("sync failed")) numFails-- } - return s, nil + return toCall(s) }).Times(times) } diff --git a/sync2/multipeer/setsyncbase.go b/sync2/multipeer/setsyncbase.go index c20e4f1b86..4c4b95a261 100644 --- a/sync2/multipeer/setsyncbase.go +++ b/sync2/multipeer/setsyncbase.go @@ -66,37 +66,31 @@ func (ssb *SetSyncBase) Count() (int, error) { return info.Count, nil } -// Derive implements SyncBase. -func (ssb *SetSyncBase) Derive(ctx context.Context, p p2p.Peer) (PeerSyncer, error) { - ssb.mtx.Lock() - defer ssb.mtx.Unlock() - os, err := ssb.os.Copy(ctx, true) - if err != nil { - return nil, fmt.Errorf("copy set: %w", err) - } - return &peerSetSyncer{ - SetSyncBase: ssb, - OrderedSet: os, - p: p, - handler: ssb.handler, - }, nil +// WithPeerSyncer implements SyncBase. +func (ssb *SetSyncBase) WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(PeerSyncer) error) error { + return ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { + return toCall(&peerSetSyncer{ + SetSyncBase: ssb, + OrderedSet: os, + p: p, + handler: ssb.handler, + }) + }) } // Probe implements SyncBase. -func (ssb *SetSyncBase) Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) { +func (ssb *SetSyncBase) Probe(ctx context.Context, p p2p.Peer) (pr rangesync.ProbeResult, err error) { // Use a snapshot of the store to avoid holding the mutex for a long time - ssb.mtx.Lock() - os, err := ssb.os.Copy(ctx, true) - defer os.Release() - ssb.mtx.Unlock() - if err != nil { - return rangesync.ProbeResult{}, fmt.Errorf("copy set: %w", err) + if err := ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { + pr, err = ssb.ps.Probe(ctx, p, os, nil, nil) + if err != nil { + return fmt.Errorf("probing peer %s: %w", p, err) + } + return nil + }); err != nil { + return rangesync.ProbeResult{}, fmt.Errorf("using set copy for probe: %w", err) } - pr, err := ssb.ps.Probe(ctx, p, os, nil, nil) - if err != nil { - return rangesync.ProbeResult{}, fmt.Errorf("probing peer %s: %w", p, err) - } return pr, nil } diff --git a/sync2/multipeer/setsyncbase_test.go b/sync2/multipeer/setsyncbase_test.go index 5b00aa8eba..234f454cc9 100644 --- a/sync2/multipeer/setsyncbase_test.go +++ b/sync2/multipeer/setsyncbase_test.go @@ -69,17 +69,15 @@ func (st *setSyncBaseTester) getWaitCh(k rangesync.KeyBytes) chan error { func (st *setSyncBaseTester) expectCopy(addedKeys ...rangesync.KeyBytes) *mocks.MockOrderedSet { copy := mocks.NewMockOrderedSet(st.ctrl) - st.os.EXPECT().Copy(gomock.Any(), true).DoAndReturn( - func(context.Context, bool) (rangesync.OrderedSet, error) { + st.os.EXPECT().WithCopy(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, toCall func(rangesync.OrderedSet) error) error { copy.EXPECT().Items().DoAndReturn(func() rangesync.SeqResult { return rangesync.EmptySeqResult() }).AnyTimes() for _, k := range addedKeys { copy.EXPECT().Receive(k) } - // TODO: do better job at tracking Release() calls - copy.EXPECT().Release().AnyTimes() - return copy, nil + return toCall(copy) }) return copy } @@ -139,24 +137,27 @@ func TestSetSyncBase(t *testing.T) { addedKey := rangesync.RandomKeyBytes(32) st.expectCopy(addedKey) - ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) - require.NoError(t, err) - require.Equal(t, p2p.Peer("p1"), ss.Peer()) + require.NoError(t, st.ssb.WithPeerSyncer( + context.Background(), p2p.Peer("p1"), + func(ps multipeer.PeerSyncer) error { + require.Equal(t, p2p.Peer("p1"), ps.Peer()) - x := rangesync.RandomKeyBytes(32) - y := rangesync.RandomKeyBytes(32) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), ss, x, y) - require.NoError(t, ss.Sync(context.Background(), x, y)) + x := rangesync.RandomKeyBytes(32) + y := rangesync.RandomKeyBytes(32) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + st.os.EXPECT().Advance() + st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), ps, x, y) + require.NoError(t, ps.Sync(context.Background(), x, y)) - st.os.EXPECT().Has(addedKey) - st.os.EXPECT().Receive(addedKey) - st.expectSync(p2p.Peer("p1"), ss, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ss.Sync(context.Background(), nil, nil)) - close(st.getWaitCh(addedKey)) + st.os.EXPECT().Has(addedKey) + st.os.EXPECT().Receive(addedKey) + st.expectSync(p2p.Peer("p1"), ps, addedKey) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + st.os.EXPECT().Advance() + require.NoError(t, ps.Sync(context.Background(), nil, nil)) + close(st.getWaitCh(addedKey)) + return nil + })) handledKeys, err := st.wait(1) require.NoError(t, err) @@ -169,20 +170,22 @@ func TestSetSyncBase(t *testing.T) { addedKey := rangesync.RandomKeyBytes(32) st.expectCopy(addedKey, addedKey, addedKey) - ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) - require.NoError(t, err) - require.Equal(t, p2p.Peer("p1"), ss.Peer()) - - // added just once - st.os.EXPECT().Receive(addedKey) - for i := 0; i < 3; i++ { - st.os.EXPECT().Has(addedKey) - st.expectSync(p2p.Peer("p1"), ss, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ss.Sync(context.Background(), nil, nil)) - } - close(st.getWaitCh(addedKey)) + require.NoError(t, st.ssb.WithPeerSyncer( + context.Background(), p2p.Peer("p1"), + func(ps multipeer.PeerSyncer) error { + require.Equal(t, p2p.Peer("p1"), ps.Peer()) + // added just once + st.os.EXPECT().Receive(addedKey) + for i := 0; i < 3; i++ { + st.os.EXPECT().Has(addedKey) + st.expectSync(p2p.Peer("p1"), ps, addedKey) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + st.os.EXPECT().Advance() + require.NoError(t, ps.Sync(context.Background(), nil, nil)) + } + close(st.getWaitCh(addedKey)) + return nil + })) handledKeys, err := st.wait(1) require.NoError(t, err) @@ -196,21 +199,23 @@ func TestSetSyncBase(t *testing.T) { k1 := rangesync.RandomKeyBytes(32) k2 := rangesync.RandomKeyBytes(32) st.expectCopy(k1, k2) - ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) - require.NoError(t, err) - require.Equal(t, p2p.Peer("p1"), ss.Peer()) - - st.os.EXPECT().Has(k1) - st.os.EXPECT().Has(k2) - st.os.EXPECT().Receive(k1) - st.os.EXPECT().Receive(k2) - st.expectSync(p2p.Peer("p1"), ss, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ss.Sync(context.Background(), nil, nil)) - close(st.getWaitCh(k1)) - close(st.getWaitCh(k2)) + require.NoError(t, st.ssb.WithPeerSyncer( + context.Background(), p2p.Peer("p1"), + func(ps multipeer.PeerSyncer) error { + require.Equal(t, p2p.Peer("p1"), ps.Peer()) + st.os.EXPECT().Has(k1) + st.os.EXPECT().Has(k2) + st.os.EXPECT().Receive(k1) + st.os.EXPECT().Receive(k2) + st.expectSync(p2p.Peer("p1"), ps, k1, k2) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + st.os.EXPECT().Advance() + require.NoError(t, ps.Sync(context.Background(), nil, nil)) + close(st.getWaitCh(k1)) + close(st.getWaitCh(k2)) + return nil + })) handledKeys, err := st.wait(2) require.NoError(t, err) require.ElementsMatch(t, []rangesync.KeyBytes{k1, k2}, handledKeys) @@ -223,20 +228,23 @@ func TestSetSyncBase(t *testing.T) { k1 := rangesync.RandomKeyBytes(32) k2 := rangesync.RandomKeyBytes(32) st.expectCopy(k1, k2) - ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) - require.NoError(t, err) - require.Equal(t, p2p.Peer("p1"), ss.Peer()) + require.NoError(t, st.ssb.WithPeerSyncer( + context.Background(), p2p.Peer("p1"), + func(ps multipeer.PeerSyncer) error { + require.Equal(t, p2p.Peer("p1"), ps.Peer()) - st.os.EXPECT().Has(k1) - st.os.EXPECT().Has(k2) - // k1 is not propagated to syncBase due to the handler failure - st.os.EXPECT().Receive(k2) - st.expectSync(p2p.Peer("p1"), ss, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ss.Sync(context.Background(), nil, nil)) - st.getWaitCh(k1) <- errors.New("fail") - close(st.getWaitCh(k2)) + st.os.EXPECT().Has(k1) + st.os.EXPECT().Has(k2) + // k1 is not propagated to syncBase due to the handler failure + st.os.EXPECT().Receive(k2) + st.expectSync(p2p.Peer("p1"), ps, k1, k2) + st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + st.os.EXPECT().Advance() + require.NoError(t, ps.Sync(context.Background(), nil, nil)) + st.getWaitCh(k1) <- errors.New("fail") + close(st.getWaitCh(k2)) + return nil + })) handledKeys, err := st.wait(2) require.ErrorContains(t, err, "some key handlers failed") @@ -253,19 +261,22 @@ func TestSetSyncBase(t *testing.T) { os.AddUnchecked(hs[0]) os.AddUnchecked(hs[1]) st := newSetSyncBaseTester(t, &os) - ss, err := st.ssb.Derive(context.Background(), p2p.Peer("p1")) - require.NoError(t, err) - ss.(rangesync.OrderedSet).Receive(hs[2]) - ss.(rangesync.OrderedSet).Add(hs[2]) - ss.(rangesync.OrderedSet).Receive(hs[3]) - ss.(rangesync.OrderedSet).Add(hs[3]) - // syncer's cloned set has new key immediately - has, err := ss.(rangesync.OrderedSet).Has(hs[2]) - require.NoError(t, err) - require.True(t, has) - has, err = ss.(rangesync.OrderedSet).Has(hs[3]) - require.NoError(t, err) - require.True(t, has) + require.NoError(t, st.ssb.WithPeerSyncer( + context.Background(), p2p.Peer("p1"), + func(ps multipeer.PeerSyncer) error { + ps.(rangesync.OrderedSet).Receive(hs[2]) + ps.(rangesync.OrderedSet).Add(hs[2]) + ps.(rangesync.OrderedSet).Receive(hs[3]) + ps.(rangesync.OrderedSet).Add(hs[3]) + // syncer's cloned set has new key immediately + has, err := ps.(rangesync.OrderedSet).Has(hs[2]) + require.NoError(t, err) + require.True(t, has) + has, err = ps.(rangesync.OrderedSet).Has(hs[3]) + require.NoError(t, err) + require.True(t, has) + return nil + })) st.getWaitCh(hs[2]) <- errors.New("fail") close(st.getWaitCh(hs[3])) handledKeys, err := st.wait(2) diff --git a/sync2/multipeer/split_sync.go b/sync2/multipeer/split_sync.go index b084c30e0d..6d69293451 100644 --- a/sync2/multipeer/split_sync.go +++ b/sync2/multipeer/split_sync.go @@ -16,7 +16,7 @@ import ( ) type syncResult struct { - s PeerSyncer + ps PeerSyncer err error } @@ -82,23 +82,23 @@ func (s *splitSync) nextPeer() p2p.Peer { } func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange) error { - syncer, err := s.syncBase.Derive(ctx, p) - if err != nil { - return fmt.Errorf("derive syncer: %w", err) - } sr.NumSyncers++ s.numRunning++ doneCh := make(chan struct{}) s.eg.Go(func() error { - defer syncer.Release() - err := syncer.Sync(ctx, sr.X, sr.Y) - close(doneCh) - select { - case <-ctx.Done(): - return ctx.Err() - case s.resCh <- syncResult{s: syncer, err: err}: - return nil + if err := s.syncBase.WithPeerSyncer(ctx, p, func(ps PeerSyncer) error { + err := ps.Sync(ctx, sr.X, sr.Y) + close(doneCh) + select { + case <-ctx.Done(): + return ctx.Err() + case s.resCh <- syncResult{ps: ps, err: err}: + return nil + } + }); err != nil { + return fmt.Errorf("sync peer %s: %w", p, err) } + return nil }) gpTimer := s.clock.After(s.gracePeriod) s.eg.Go(func() error { @@ -121,18 +121,18 @@ func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange } func (s *splitSync) handleSyncResult(r syncResult) error { - sr, found := s.syncMap[r.s.Peer()] + sr, found := s.syncMap[r.ps.Peer()] if !found { panic("BUG: error in split sync syncMap handling") } s.numRunning-- - delete(s.syncMap, r.s.Peer()) + delete(s.syncMap, r.ps.Peer()) sr.NumSyncers-- if r.err != nil { s.numPeers-- - s.failedPeers[r.s.Peer()] = struct{}{} + s.failedPeers[r.ps.Peer()] = struct{}{} s.logger.Debug("remove failed peer", - zap.Stringer("peer", r.s.Peer()), + zap.Stringer("peer", r.ps.Peer()), zap.Int("numPeers", s.numPeers), zap.Int("numRemaining", s.numRemaining), zap.Int("numRunning", s.numRunning), @@ -147,10 +147,10 @@ func (s *splitSync) handleSyncResult(r syncResult) error { } } else { sr.Done = true - s.syncPeers = append(s.syncPeers, r.s.Peer()) + s.syncPeers = append(s.syncPeers, r.ps.Peer()) s.numRemaining-- s.logger.Debug("peer synced successfully", - zap.Stringer("peer", r.s.Peer()), + zap.Stringer("peer", r.ps.Peer()), zap.Int("numPeers", s.numPeers), zap.Int("numRemaining", s.numRemaining), zap.Int("numRunning", s.numRunning), diff --git a/sync2/multipeer/split_sync_test.go b/sync2/multipeer/split_sync_test.go index 195cf45561..68c6224fdf 100644 --- a/sync2/multipeer/split_sync_test.go +++ b/sync2/multipeer/split_sync_test.go @@ -74,12 +74,15 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { } for index, p := range tst.syncPeers { tst.syncBase.EXPECT(). - Derive(gomock.Any(), p). - DoAndReturn(func(_ context.Context, peer p2p.Peer) (multipeer.PeerSyncer, error) { + WithPeerSyncer(gomock.Any(), p, gomock.Any()). + DoAndReturn(func( + _ context.Context, + peer p2p.Peer, + toCall func(multipeer.PeerSyncer) error, + ) error { s := NewMockPeerSyncer(ctrl) s.EXPECT().Peer().Return(p).AnyTimes() // TODO: do better job at tracking Release() calls - s.EXPECT().Release().AnyTimes() s.EXPECT(). Sync(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, x, y rangesync.KeyBytes) error { @@ -103,7 +106,7 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { } return nil }) - return s, nil + return toCall(s) }). AnyTimes() } diff --git a/sync2/p2p.go b/sync2/p2p.go index 4310adff2c..ba3d098eef 100644 --- a/sync2/p2p.go +++ b/sync2/p2p.go @@ -79,17 +79,10 @@ func NewP2PHashSync( func (s *P2PHashSync) serve(ctx context.Context, peer p2p.Peer, stream io.ReadWriter) error { // We derive a dedicated Syncer for the peer being served to pass all the received - // items through the handler before adding them to the main ItemStore - syncer, err := s.syncBase.Derive(ctx, peer) - if err != nil { - return fmt.Errorf("derive syncer: %w", err) - } - defer syncer.Release() - if err := syncer.Serve(ctx, stream); err != nil { - syncer.Release() - return err - } - return nil + // items through the handler before adding them to the main OrderedSet. + return s.syncBase.WithPeerSyncer(ctx, peer, func(syncer multipeer.PeerSyncer) error { + return syncer.Serve(ctx, stream) + }) } // Set returns the OrderedSet that is being synchronized. diff --git a/sync2/p2p_test.go b/sync2/p2p_test.go index 54b63c4a60..f0207f052a 100644 --- a/sync2/p2p_test.go +++ b/sync2/p2p_test.go @@ -128,21 +128,29 @@ func TestP2P(t *testing.T) { if !hsync.Synced() { return false } - os, err := hsync.Set().Copy(context.Background(), false) - require.NoError(t, err) - for _, k := range handlers[n].committedItems() { - os.(*rangesync.DumbSet).AddUnchecked(k) - } - empty, err := os.Empty() - require.NoError(t, err) - if empty { - return false - } - k, err := os.Items().First() - require.NoError(t, err) - info, err := os.GetRangeInfo(k, k) - require.NoError(t, err) - if info.Count < numHashes { + r := true + require.NoError(t, hsync.Set().WithCopy( + context.Background(), + func(os rangesync.OrderedSet) error { + for _, k := range handlers[n].committedItems() { + os.(*rangesync.DumbSet).AddUnchecked(k) + } + empty, err := os.Empty() + require.NoError(t, err) + if empty { + r = false + } else { + k, err := os.Items().First() + require.NoError(t, err) + info, err := os.GetRangeInfo(k, k) + require.NoError(t, err) + if info.Count < numHashes { + r = false + } + } + return nil + })) + if !r { return false } } @@ -151,13 +159,16 @@ func TestP2P(t *testing.T) { for n, hsync := range hs { hsync.Stop() - os, err := hsync.Set().Copy(context.Background(), false) - require.NoError(t, err) - for _, k := range handlers[n].committedItems() { - os.(*rangesync.DumbSet).AddUnchecked(k) - } - actualItems, err := os.Items().Collect() - require.NoError(t, err) - require.ElementsMatch(t, initialSet, actualItems) + require.NoError(t, hsync.Set().WithCopy( + context.Background(), + func(os rangesync.OrderedSet) error { + for _, k := range handlers[n].committedItems() { + os.(*rangesync.DumbSet).AddUnchecked(k) + } + actualItems, err := os.Items().Collect() + require.NoError(t, err) + require.ElementsMatch(t, initialSet, actualItems) + return nil + })) } } diff --git a/sync2/rangesync/dumbset.go b/sync2/rangesync/dumbset.go index ac4e734312..5c4abfc782 100644 --- a/sync2/rangesync/dumbset.go +++ b/sync2/rangesync/dumbset.go @@ -307,11 +307,9 @@ func (ds *DumbSet) Items() SeqResult { return ds.seq(0) } -// Copy implements OrderedSet. -func (ds *DumbSet) Copy(_ context.Context, syncScope bool) (OrderedSet, error) { - return &DumbSet{ - keys: slices.Clone(ds.keys), - }, nil +// WithCopy implements OrderedSet. +func (ds *DumbSet) WithCopy(_ context.Context, toCall func(OrderedSet) error) error { + return toCall(&DumbSet{keys: slices.Clone(ds.keys)}) } // Recent implements OrderedSet. diff --git a/sync2/rangesync/interface.go b/sync2/rangesync/interface.go index 12d42ae2c7..517efc18ad 100644 --- a/sync2/rangesync/interface.go +++ b/sync2/rangesync/interface.go @@ -30,6 +30,7 @@ type SplitInfo struct { } // OrderedSet represents the set that can be synced against a remote peer. +// OrderedSet methods are non-threadsafe except for WithCopy, Loaded and EnsureLoaded. type OrderedSet interface { // Add adds a new key to the set. // It should not perform any additional actions related to handling @@ -58,13 +59,11 @@ type OrderedSet interface { Items() SeqResult // Empty returns true if the set is empty. Empty() (bool, error) - // Copy makes a shallow copy of the OrderedSet. - // syncScope argument is a hint that can be used to optimize resource usage. - // If syncScope is true, then the copy is intended to be used for the duration of - // a synchronization run. - // If syncScope if false, then the lifetime of the copy is not clearly defined. - // The list of received items as returned by Received is also inherited by the copy. - Copy(ctx context.Context, syncScope bool) (OrderedSet, error) + // WithCopy runs the specified function, passing to it a temporary shallow copy of + // the OrderedSet. The copy is discarded after the function returns, releasing + // any resources associated with it. + // The list of received items as returned by Received is inherited by the copy. + WithCopy(ctx context.Context, toCall func(OrderedSet) error) error // Recent returns an Iterator that yields the items added since the specified // timestamp. Some OrderedSet implementations may not have Recent implemented, in // which case it should return an empty sequence. @@ -80,9 +79,6 @@ type OrderedSet interface { Advance() error // Has returns true if the specified key is present in OrderedSet. Has(KeyBytes) (bool, error) - // Release releases the resources associated with the set. - // Calling Release on a set that is already released is a no-op. - Release() } type Requester interface { diff --git a/sync2/rangesync/mocks/mocks.go b/sync2/rangesync/mocks/mocks.go index 481ea093ec..304aa1d9a4 100644 --- a/sync2/rangesync/mocks/mocks.go +++ b/sync2/rangesync/mocks/mocks.go @@ -118,45 +118,6 @@ func (c *MockOrderedSetAdvanceCall) DoAndReturn(f func() error) *MockOrderedSetA return c } -// Copy mocks base method. -func (m *MockOrderedSet) Copy(ctx context.Context, syncScope bool) (rangesync.OrderedSet, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Copy", ctx, syncScope) - ret0, _ := ret[0].(rangesync.OrderedSet) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Copy indicates an expected call of Copy. -func (mr *MockOrderedSetMockRecorder) Copy(ctx, syncScope any) *MockOrderedSetCopyCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockOrderedSet)(nil).Copy), ctx, syncScope) - return &MockOrderedSetCopyCall{Call: call} -} - -// MockOrderedSetCopyCall wrap *gomock.Call -type MockOrderedSetCopyCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockOrderedSetCopyCall) Return(arg0 rangesync.OrderedSet, arg1 error) *MockOrderedSetCopyCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockOrderedSetCopyCall) Do(f func(context.Context, bool) (rangesync.OrderedSet, error)) *MockOrderedSetCopyCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetCopyCall) DoAndReturn(f func(context.Context, bool) (rangesync.OrderedSet, error)) *MockOrderedSetCopyCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Empty mocks base method. func (m *MockOrderedSet) Empty() (bool, error) { m.ctrl.T.Helper() @@ -503,77 +464,79 @@ func (c *MockOrderedSetRecentCall) DoAndReturn(f func(time.Time) (rangesync.SeqR return c } -// Release mocks base method. -func (m *MockOrderedSet) Release() { +// SplitRange mocks base method. +func (m *MockOrderedSet) SplitRange(x, y rangesync.KeyBytes, count int) (rangesync.SplitInfo, error) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Release") + ret := m.ctrl.Call(m, "SplitRange", x, y, count) + ret0, _ := ret[0].(rangesync.SplitInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// Release indicates an expected call of Release. -func (mr *MockOrderedSetMockRecorder) Release() *MockOrderedSetReleaseCall { +// SplitRange indicates an expected call of SplitRange. +func (mr *MockOrderedSetMockRecorder) SplitRange(x, y, count any) *MockOrderedSetSplitRangeCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockOrderedSet)(nil).Release)) - return &MockOrderedSetReleaseCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SplitRange", reflect.TypeOf((*MockOrderedSet)(nil).SplitRange), x, y, count) + return &MockOrderedSetSplitRangeCall{Call: call} } -// MockOrderedSetReleaseCall wrap *gomock.Call -type MockOrderedSetReleaseCall struct { +// MockOrderedSetSplitRangeCall wrap *gomock.Call +type MockOrderedSetSplitRangeCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetReleaseCall) Return() *MockOrderedSetReleaseCall { - c.Call = c.Call.Return() +func (c *MockOrderedSetSplitRangeCall) Return(arg0 rangesync.SplitInfo, arg1 error) *MockOrderedSetSplitRangeCall { + c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetReleaseCall) Do(f func()) *MockOrderedSetReleaseCall { +func (c *MockOrderedSetSplitRangeCall) Do(f func(rangesync.KeyBytes, rangesync.KeyBytes, int) (rangesync.SplitInfo, error)) *MockOrderedSetSplitRangeCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetReleaseCall) DoAndReturn(f func()) *MockOrderedSetReleaseCall { +func (c *MockOrderedSetSplitRangeCall) DoAndReturn(f func(rangesync.KeyBytes, rangesync.KeyBytes, int) (rangesync.SplitInfo, error)) *MockOrderedSetSplitRangeCall { c.Call = c.Call.DoAndReturn(f) return c } -// SplitRange mocks base method. -func (m *MockOrderedSet) SplitRange(x, y rangesync.KeyBytes, count int) (rangesync.SplitInfo, error) { +// WithCopy mocks base method. +func (m *MockOrderedSet) WithCopy(ctx context.Context, toCall func(rangesync.OrderedSet) error) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SplitRange", x, y, count) - ret0, _ := ret[0].(rangesync.SplitInfo) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "WithCopy", ctx, toCall) + ret0, _ := ret[0].(error) + return ret0 } -// SplitRange indicates an expected call of SplitRange. -func (mr *MockOrderedSetMockRecorder) SplitRange(x, y, count any) *MockOrderedSetSplitRangeCall { +// WithCopy indicates an expected call of WithCopy. +func (mr *MockOrderedSetMockRecorder) WithCopy(ctx, toCall any) *MockOrderedSetWithCopyCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SplitRange", reflect.TypeOf((*MockOrderedSet)(nil).SplitRange), x, y, count) - return &MockOrderedSetSplitRangeCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithCopy", reflect.TypeOf((*MockOrderedSet)(nil).WithCopy), ctx, toCall) + return &MockOrderedSetWithCopyCall{Call: call} } -// MockOrderedSetSplitRangeCall wrap *gomock.Call -type MockOrderedSetSplitRangeCall struct { +// MockOrderedSetWithCopyCall wrap *gomock.Call +type MockOrderedSetWithCopyCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetSplitRangeCall) Return(arg0 rangesync.SplitInfo, arg1 error) *MockOrderedSetSplitRangeCall { - c.Call = c.Call.Return(arg0, arg1) +func (c *MockOrderedSetWithCopyCall) Return(arg0 error) *MockOrderedSetWithCopyCall { + c.Call = c.Call.Return(arg0) return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetSplitRangeCall) Do(f func(rangesync.KeyBytes, rangesync.KeyBytes, int) (rangesync.SplitInfo, error)) *MockOrderedSetSplitRangeCall { +func (c *MockOrderedSetWithCopyCall) Do(f func(context.Context, func(rangesync.OrderedSet) error) error) *MockOrderedSetWithCopyCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetSplitRangeCall) DoAndReturn(f func(rangesync.KeyBytes, rangesync.KeyBytes, int) (rangesync.SplitInfo, error)) *MockOrderedSetSplitRangeCall { +func (c *MockOrderedSetWithCopyCall) DoAndReturn(f func(context.Context, func(rangesync.OrderedSet) error) error) *MockOrderedSetWithCopyCall { c.Call = c.Call.DoAndReturn(f) return c } From ce73f548af3fb06947159d022dfaf53a70710c93 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 20 Nov 2024 19:05:11 +0400 Subject: [PATCH 10/22] sync2: use saner retry scheme for fetched ATXs --- fetch/limiter.go | 5 +++- fetch/mesh_data.go | 40 ++++++++++++++++++--------- fetch/mesh_data_test.go | 16 +++++------ sync2/atxs.go | 43 ++++++++++++----------------- sync2/atxs_test.go | 60 ++++++++++++++++++++++++++--------------- system/fetcher.go | 14 ++++++---- 6 files changed, 104 insertions(+), 74 deletions(-) diff --git a/fetch/limiter.go b/fetch/limiter.go index 3222433d5b..11993ddb38 100644 --- a/fetch/limiter.go +++ b/fetch/limiter.go @@ -2,6 +2,8 @@ package fetch import ( "context" + + "github.com/spacemeshos/go-spacemesh/common/types" ) type limiter interface { @@ -10,7 +12,8 @@ type limiter interface { } type getHashesOpts struct { - limiter limiter + limiter limiter + callback func(types.Hash32, error) } type noLimit struct{} diff --git a/fetch/mesh_data.go b/fetch/mesh_data.go index 28382808ba..0f2760dc89 100644 --- a/fetch/mesh_data.go +++ b/fetch/mesh_data.go @@ -42,19 +42,16 @@ func (f *Fetch) GetAtxs(ctx context.Context, ids []types.ATXID, opts ...system.G ) hashes := types.ATXIDsToHashes(ids) handler := f.validators.atx.HandleMessage - if options.RecvChannel != nil { - handler = func(ctx context.Context, id types.Hash32, p p2p.Peer, data []byte) error { - if err := f.validators.atx.HandleMessage(ctx, id, p, data); err != nil { - return err - } - options.RecvChannel <- types.ATXID(id) - return nil - } + var ghOpts []getHashesOpt + if !options.LimitingOff { + ghOpts = append(ghOpts, withLimiter(f.getAtxsLimiter)) } - if options.LimitingOff { - return f.getHashes(ctx, hashes, datastore.ATXDB, handler) + if options.Callback != nil { + ghOpts = append(ghOpts, withHashCallback(func(hash types.Hash32, err error) { + options.Callback(types.ATXID(hash), err) + })) } - return f.getHashes(ctx, hashes, datastore.ATXDB, handler, withLimiter(f.getAtxsLimiter)) + return f.getHashes(ctx, hashes, datastore.ATXDB, handler, ghOpts...) } type dataReceiver func(context.Context, types.Hash32, p2p.Peer, []byte) error @@ -67,6 +64,12 @@ func withLimiter(l limiter) getHashesOpt { } } +func withHashCallback(callback func(types.Hash32, error)) getHashesOpt { + return func(o *getHashesOpts) { + o.callback = callback + } +} + func (f *Fetch) getHashes( ctx context.Context, hashes []types.Hash32, @@ -75,7 +78,8 @@ func (f *Fetch) getHashes( opts ...getHashesOpt, ) error { options := getHashesOpts{ - limiter: noLimit{}, + limiter: noLimit{}, + callback: func(types.Hash32, error) {}, } for _, opt := range opts { opt(&options) @@ -92,18 +96,26 @@ func (f *Fetch) getHashes( for i, hash := range hashes { if err := options.limiter.Acquire(ctx, 1); err != nil { pendingMetric.Add(float64(i - len(hashes))) - return fmt.Errorf("acquiring slot to get hash: %w", err) + err = fmt.Errorf("acquiring slot to get hash: %w", err) + for _, h := range hashes[i:] { + options.callback(h, err) + } + return err } p, err := f.getHash(ctx, hash, hint, receiver) if err != nil { options.limiter.Release(1) pendingMetric.Add(float64(i - len(hashes))) + for _, h := range hashes[i:] { + options.callback(h, err) + } return err } if p == nil { // data is available locally options.limiter.Release(1) pendingMetric.Add(-1) + options.callback(hash, nil) continue } @@ -112,6 +124,7 @@ func (f *Fetch) getHashes( case <-ctx.Done(): options.limiter.Release(1) pendingMetric.Add(-1) + options.callback(hash, ctx.Err()) return ctx.Err() case <-p.completed: options.limiter.Release(1) @@ -127,6 +140,7 @@ func (f *Fetch) getHashes( bfailure.Add(hash, p.err) mu.Unlock() } + options.callback(hash, p.err) return nil } }) diff --git a/fetch/mesh_data_test.go b/fetch/mesh_data_test.go index 56b1b83e91..d5f277b65d 100644 --- a/fetch/mesh_data_test.go +++ b/fetch/mesh_data_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sync" "testing" p2phost "github.com/libp2p/go-libp2p/core/host" @@ -605,17 +606,16 @@ func TestGetATXs(t *testing.T) { atxIDs1 := types.ToATXIDs(atxs[:2]) require.NoError(t, f.GetAtxs(context.Background(), atxIDs1)) - recvCh := make(chan types.ATXID) atxIDs2 := types.ToATXIDs(atxs[2:]) var recvIDs []types.ATXID - eg.Go(func() error { - for id := range recvCh { + var mtx sync.Mutex + require.NoError(t, f.GetAtxs(context.Background(), atxIDs2, + system.WithATXCallback(func(id types.ATXID, err error) { + mtx.Lock() + defer mtx.Unlock() + require.NoError(t, err) recvIDs = append(recvIDs, id) - } - return nil - }) - require.NoError(t, f.GetAtxs(context.Background(), atxIDs2, system.WithRecvChannel(recvCh))) - close(recvCh) + }))) close(stop) require.NoError(t, eg.Wait()) require.ElementsMatch(t, atxIDs2, recvIDs) diff --git a/sync2/atxs.go b/sync2/atxs.go index 11feba1399..45fec8d9b7 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "github.com/jonboulle/clockwork" @@ -95,7 +96,7 @@ func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new ranges } total := len(state) items := make([]types.ATXID, 0, h.batchSize) - startTime := time.Now() + startTime := h.clock.Now() batchAttemptsRemaining := h.maxBatchRetries for len(state) > 0 { items = items[:0] @@ -115,39 +116,28 @@ func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new ranges break } - var eg errgroup.Group - recvCh := make(chan types.ATXID) someSucceeded := false - eg.Go(func() error { - for id := range recvCh { + var mtx sync.Mutex + err := h.f.GetAtxs(ctx, items, system.WithATXCallback(func(id types.ATXID, err error) { + mtx.Lock() + defer mtx.Unlock() + switch { + case err == nil: numDownloaded++ someSucceeded = true delete(state, id) + case errors.Is(err, pubsub.ErrValidationReject): + // if the atx invalid there's no point downloading it again + state[id] = h.maxAttempts + default: + state[id]++ } - return nil - }) - err := h.f.GetAtxs(ctx, items, system.WithRecvChannel(recvCh)) - close(recvCh) - eg.Wait() + })) if err != nil { if errors.Is(err, context.Canceled) { return err } - batchError := &fetch.BatchError{} - if errors.As(err, &batchError) { - h.logger.Debug("QQQQQ: batch error", zap.Error(err)) - for hash, err := range batchError.Errors { - if _, exists := state[types.ATXID(hash)]; !exists { - continue - } - if errors.Is(err, pubsub.ErrValidationReject) { - // if the atx invalid there's no point downloading it again - state[types.ATXID(hash)] = h.maxAttempts - } else { - state[types.ATXID(hash)]++ - } - } - } else { + if !errors.Is(err, &fetch.BatchError{}) { h.logger.Debug("failed to download ATXs", zap.Error(err)) } } @@ -166,10 +156,11 @@ func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new ranges } } else { batchAttemptsRemaining = h.maxBatchRetries + elapsed := h.clock.Since(startTime) h.logger.Debug("fetched atxs", zap.Int("total", total), zap.Int("downloaded", numDownloaded), - zap.Float64("rate per sec", float64(numDownloaded)/time.Since(startTime).Seconds())) + zap.Float64("rate per sec", float64(numDownloaded)/elapsed.Seconds())) } } return nil diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go index 7dcc780ebe..ebfe3451a3 100644 --- a/sync2/atxs_test.go +++ b/sync2/atxs_test.go @@ -38,7 +38,8 @@ func TestAtxHandler_Success(t *testing.T) { allAtxs[i] = types.RandomATXID() } f := NewMockFetcher(ctrl) - h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + clock := clockwork.NewFakeClock() + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { @@ -60,15 +61,11 @@ func TestAtxHandler_Success(t *testing.T) { for _, opt := range opts { opt(&atxOpts) } - require.NotNil(t, atxOpts.RecvChannel) + require.NotNil(t, atxOpts.Callback) for _, id := range atxs { require.True(t, toFetch[id], "already fetched or bad ID") delete(toFetch, id) - select { - case <-time.After(100 * time.Millisecond): - t.Error("timeout sending recvd id") - case atxOpts.RecvChannel <- id: - } + atxOpts.Callback(id, nil) } return nil }).Times(3) @@ -106,7 +103,8 @@ func TestAtxHandler_Retry(t *testing.T) { allAtxs[i] = types.RandomATXID() } f := NewMockFetcher(ctrl) - h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + clock := clockwork.NewFakeClock() + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { @@ -126,23 +124,21 @@ func TestAtxHandler_Retry(t *testing.T) { for _, opt := range opts { opt(&atxOpts) } - require.NotNil(t, atxOpts.RecvChannel) + require.NotNil(t, atxOpts.Callback) for _, id := range atxs { switch { case id == allAtxs[0]: require.False(t, validationFailed, "retried after validation error") errs[id.Hash32()] = pubsub.ErrValidationReject + atxOpts.Callback(id, errs[id.Hash32()]) validationFailed = true case id == allAtxs[1] && failCount < 2: errs[id.Hash32()] = errors.New("fetch failed") + atxOpts.Callback(id, errs[id.Hash32()]) failCount++ default: fetched = append(fetched, id) - select { - case <-time.After(100 * time.Millisecond): - t.Error("timeout sending recvd id") - case atxOpts.RecvChannel <- id: - } + atxOpts.Callback(id, nil) } } if len(errs) > 0 { @@ -164,6 +160,31 @@ func TestAtxHandler_Retry(t *testing.T) { }, Error: rangesync.NoSeqError, }) + + // If it so happens that a full batch fails, we need to advance the clock to + // trigger the retry. + ctx, cancel := context.WithCancel(context.Background()) + var eg errgroup.Group + eg.Go(func() error { + for { + // FIXME: BlockUntilContext is not included in FakeClock interface. + // This will be fixed in a post-0.4.0 clockwork release, but with a breaking change that + // makes FakeClock a struct instead of an interface. + // See: https://github.com/jonboulle/clockwork/pull/71 + clock.(interface { + BlockUntilContext(ctx context.Context, n int) error + }).BlockUntilContext(ctx, 1) + if ctx.Err() != nil { + return nil + } + clock.Advance(batchRetryDelay) + } + }) + defer func() { + cancel() + eg.Wait() + }() + require.NoError(t, h.Commit(context.Background(), peer, baseSet, newSet)) require.ElementsMatch(t, allAtxs[1:], fetched) } @@ -180,7 +201,8 @@ func TestAtxHandler_Cancel(t *testing.T) { logger := zaptest.NewLogger(t) peer := p2p.Peer("foobar") f := NewMockFetcher(ctrl) - h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, nil) + clock := clockwork.NewFakeClock() + h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) newSet := mocks.NewMockOrderedSet(ctrl) baseSet.EXPECT().Has(rangesync.KeyBytes(atxID[:])).Return(false, nil) @@ -257,15 +279,11 @@ func TestAtxHandler_BatchRetry(t *testing.T) { for _, opt := range opts { opt(&atxOpts) } - require.NotNil(t, atxOpts.RecvChannel) + require.NotNil(t, atxOpts.Callback) for _, id := range atxs { require.True(t, toFetch[id], "already fetched or bad ID") delete(toFetch, id) - select { - case <-time.After(100 * time.Millisecond): - t.Error("timeout sending recvd id") - case atxOpts.RecvChannel <- id: - } + atxOpts.Callback(id, nil) } return nil }).Times(3) diff --git a/system/fetcher.go b/system/fetcher.go index 61f2d5c451..343f874516 100644 --- a/system/fetcher.go +++ b/system/fetcher.go @@ -28,7 +28,7 @@ type BlockFetcher interface { type GetAtxOpts struct { LimitingOff bool - RecvChannel chan<- types.ATXID + Callback func(types.ATXID, error) } type GetAtxOpt func(*GetAtxOpts) @@ -40,11 +40,15 @@ func WithoutLimiting() GetAtxOpt { } } -// WithRecvChannel sets the channel to receive successfully downloaded and validated ATXs -// IDs on. -func WithRecvChannel(ch chan<- types.ATXID) GetAtxOpt { +// WithATXCallback sets a callback function to be called after each ATX is downloaded, +// found locally or failed to download. +// The callback is guaranteed to be called exactly once for each ATX ID passed to GetAtxs. +// The callback is guaranteed not to be invoked after GetAtxs returns. +// The callback may be called concurrently from multiple goroutines. +// A non-nil error is passed in case the ATX cannot be found locally and failed to download. +func WithATXCallback(callback func(types.ATXID, error)) GetAtxOpt { return func(opts *GetAtxOpts) { - opts.RecvChannel = ch + opts.Callback = callback } } From f411f7ef34ef280b481ae31660c945d0d9136313 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Thu, 21 Nov 2024 00:26:41 +0400 Subject: [PATCH 11/22] syncer: rename "v2" field to "reconcSync" in the config --- config/mainnet.go | 2 +- config/presets/testnet.go | 2 +- syncer/syncer.go | 30 +++++++++++++++--------------- syncer/syncer_test.go | 8 ++++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/config/mainnet.go b/config/mainnet.go index 58d0a77cac..bf29839606 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -221,7 +221,7 @@ func MainnetConfig() Config { DisableMeshAgreement: true, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), - V2: syncer.SyncV2Config{ + ReconcSync: syncer.ReconcSyncConfig{ OldAtxSyncCfg: oldAtxSyncCfg, NewAtxSyncCfg: newAtxSyncCfg, ParallelLoadLimit: 10, diff --git a/config/presets/testnet.go b/config/presets/testnet.go index 713cea25d8..e7856e509c 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -171,7 +171,7 @@ func testnet() config.Config { OutOfSyncThresholdLayers: 10, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), - V2: syncer.SyncV2Config{ + ReconcSync: syncer.ReconcSyncConfig{ OldAtxSyncCfg: oldAtxSyncCfg, NewAtxSyncCfg: newAtxSyncCfg, ParallelLoadLimit: 10, diff --git a/syncer/syncer.go b/syncer/syncer.go index 32089d3820..78de264603 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -37,15 +37,15 @@ type Config struct { TallyVotesFrequency float64 MaxStaleDuration time.Duration `mapstructure:"maxstaleduration"` Standalone bool - GossipDuration time.Duration `mapstructure:"gossipduration"` - DisableMeshAgreement bool `mapstructure:"disable-mesh-agreement"` - OutOfSyncThresholdLayers uint32 `mapstructure:"out-of-sync-threshold"` - AtxSync atxsync.Config `mapstructure:"atx-sync"` - MalSync malsync.Config `mapstructure:"malfeasance-sync"` - V2 SyncV2Config `mapstructure:"v2"` + GossipDuration time.Duration `mapstructure:"gossipduration"` + DisableMeshAgreement bool `mapstructure:"disable-mesh-agreement"` + OutOfSyncThresholdLayers uint32 `mapstructure:"out-of-sync-threshold"` + AtxSync atxsync.Config `mapstructure:"atx-sync"` + MalSync malsync.Config `mapstructure:"malfeasance-sync"` + ReconcSync ReconcSyncConfig `mapstructure:"reconc-sync"` } -type SyncV2Config struct { +type ReconcSyncConfig struct { Enable bool `mapstructure:"enable"` EnableActiveSync bool `mapstructure:"enable-active-sync"` OldAtxSyncCfg sync2.Config `mapstructure:"old-atx-sync"` @@ -72,7 +72,7 @@ func DefaultConfig() Config { OutOfSyncThresholdLayers: 3, AtxSync: atxsync.DefaultConfig(), MalSync: malsync.DefaultConfig(), - V2: SyncV2Config{ + ReconcSync: ReconcSyncConfig{ Enable: false, EnableActiveSync: false, OldAtxSyncCfg: oldAtxSyncCfg, @@ -238,14 +238,14 @@ func NewSyncer( s.isBusy.Store(false) s.lastLayerSynced.Store(s.mesh.LatestLayer().Uint32()) s.lastEpochSynced.Store(types.GetEffectiveGenesis().GetEpoch().Uint32() - 1) - if s.cfg.V2.Enable && s.asv2 == nil { + if s.cfg.ReconcSync.Enable && s.asv2 == nil { s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher)) hss := sync2.NewATXSyncSource( s.logger, s.dispatcher, cdb.Database.(sql.StateDatabase), - fetcher.(sync2.Fetcher), s.cfg.V2.EnableActiveSync) + fetcher.(sync2.Fetcher), s.cfg.ReconcSync.EnableActiveSync) s.asv2 = sync2.NewMultiEpochATXSyncer( - s.logger, hss, s.cfg.V2.OldAtxSyncCfg, s.cfg.V2.NewAtxSyncCfg, - s.cfg.V2.ParallelLoadLimit) + s.logger, hss, s.cfg.ReconcSync.OldAtxSyncCfg, s.cfg.ReconcSync.NewAtxSyncCfg, + s.cfg.ReconcSync.ParallelLoadLimit) } return s } @@ -588,7 +588,7 @@ func (s *Syncer) ensureATXsInSyncV2(ctx context.Context) error { publish-- } - if !s.ListenToATXGossip() && s.cfg.V2.EnableActiveSync { + if !s.ListenToATXGossip() && s.cfg.ReconcSync.EnableActiveSync { // ATXs are not in sync yet, to we need to sync them synchronously lastWaitEpoch := types.EpochID(0) if currentEpoch > 1 { @@ -660,12 +660,12 @@ func (s *Syncer) ensureMalfeasanceInSync(ctx context.Context) error { } func (s *Syncer) syncAtxAndMalfeasance(ctx context.Context) error { - if s.cfg.V2.Enable { + if s.cfg.ReconcSync.Enable { if err := s.ensureATXsInSyncV2(ctx); err != nil { return err } } - if !s.cfg.V2.Enable || !s.cfg.V2.EnableActiveSync { + if !s.cfg.ReconcSync.Enable || !s.cfg.ReconcSync.EnableActiveSync { // If syncv2 is being used in server-only mode, we still need to run // active syncv1. if err := s.ensureATXsInSync(ctx); err != nil { diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 3bd479a677..057ac29311 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -505,8 +505,8 @@ func TestSyncAtxs_Genesis(t *testing.T) { func TestSyncAtxs_Genesis_SyncV2(t *testing.T) { cfg := defaultTestConfig(never) - cfg.V2.Enable = true - cfg.V2.EnableActiveSync = true + cfg.ReconcSync.Enable = true + cfg.ReconcSync.EnableActiveSync = true t.Run("no atx expected", func(t *testing.T) { ts := newSyncerWithoutPeriodicRunsWithConfig(t, cfg) @@ -594,8 +594,8 @@ func startWithSyncedState_SyncV2(tb testing.TB, ts *testSyncer) types.LayerID { func TestSyncAtxs_SyncV2(t *testing.T) { cfg := defaultTestConfig(never) - cfg.V2.Enable = true - cfg.V2.EnableActiveSync = true + cfg.ReconcSync.Enable = true + cfg.ReconcSync.EnableActiveSync = true tcs := []struct { desc string current types.LayerID From bb7226fcf659a0cb7fef52f7f1e9aaac7c950784 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Thu, 21 Nov 2024 00:38:47 +0400 Subject: [PATCH 12/22] sync2: add server options and request rate limits --- fetch/fetch.go | 4 ++-- sync2/atxs.go | 4 ++-- syncer/syncer.go | 24 ++++++++++++++++++------ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index 479b4f4106..0aee4ed636 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -116,7 +116,7 @@ type ServerConfig struct { Interval time.Duration `mapstructure:"interval"` } -func (s ServerConfig) toOpts() []server.Opt { +func (s ServerConfig) ToOpts() []server.Opt { opts := []server.Opt{} if s.Queue != 0 { opts = append(opts, server.WithQueueSize(s.Queue)) @@ -366,7 +366,7 @@ func (f *Fetch) registerServer( if f.cfg.EnableServerMetrics { opts = append(opts, server.WithMetrics()) } - opts = append(opts, f.cfg.getServerConfig(protocol).toOpts()...) + opts = append(opts, f.cfg.getServerConfig(protocol).ToOpts()...) f.servers[protocol] = server.New(host, protocol, handler, opts...) } diff --git a/sync2/atxs.go b/sync2/atxs.go index 45fec8d9b7..90663f61e6 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -295,9 +295,9 @@ func NewATXSyncer( cfg, enableActiveSync) } -func NewDispatcher(logger *zap.Logger, f Fetcher) *rangesync.Dispatcher { +func NewDispatcher(logger *zap.Logger, f Fetcher, opts []server.Opt) *rangesync.Dispatcher { d := rangesync.NewDispatcher(logger) - d.SetupServer(f.Host(), proto, server.WithHardTimeout(20*time.Minute)) + d.SetupServer(f.Host(), proto, opts...) return d } diff --git a/syncer/syncer.go b/syncer/syncer.go index 78de264603..65fd3042f8 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -18,6 +18,7 @@ import ( "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/mesh" "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/p2p/server" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/sync2/rangesync" @@ -46,11 +47,13 @@ type Config struct { } type ReconcSyncConfig struct { - Enable bool `mapstructure:"enable"` - EnableActiveSync bool `mapstructure:"enable-active-sync"` - OldAtxSyncCfg sync2.Config `mapstructure:"old-atx-sync"` - NewAtxSyncCfg sync2.Config `mapstructure:"new-atx-sync"` - ParallelLoadLimit int `mapstructure:"parallel-load-limit"` + Enable bool `mapstructure:"enable"` + EnableActiveSync bool `mapstructure:"enable-active-sync"` + OldAtxSyncCfg sync2.Config `mapstructure:"old-atx-sync"` + NewAtxSyncCfg sync2.Config `mapstructure:"new-atx-sync"` + ParallelLoadLimit int `mapstructure:"parallel-load-limit"` + HardTimeout time.Duration `mapstructure:"hard-timeout"` + ServerConfig fetch.ServerConfig `mapstructure:"server-config"` } // DefaultConfig for the syncer. @@ -78,6 +81,12 @@ func DefaultConfig() Config { OldAtxSyncCfg: oldAtxSyncCfg, NewAtxSyncCfg: newAtxSyncCfg, ParallelLoadLimit: 10, + HardTimeout: 10 * time.Minute, + ServerConfig: fetch.ServerConfig{ + Queue: 200, + Requests: 100, + Interval: time.Second, + }, }, } } @@ -239,7 +248,10 @@ func NewSyncer( s.lastLayerSynced.Store(s.mesh.LatestLayer().Uint32()) s.lastEpochSynced.Store(types.GetEffectiveGenesis().GetEpoch().Uint32() - 1) if s.cfg.ReconcSync.Enable && s.asv2 == nil { - s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher)) + serverOpts := append( + s.cfg.ReconcSync.ServerConfig.ToOpts(), + server.WithHardTimeout(s.cfg.ReconcSync.HardTimeout)) + s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher), serverOpts) hss := sync2.NewATXSyncSource( s.logger, s.dispatcher, cdb.Database.(sql.StateDatabase), fetcher.(sync2.Fetcher), s.cfg.ReconcSync.EnableActiveSync) From 284f836bc0275f926a28d815bce7e8390d527732 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 25 Nov 2024 00:37:07 +0400 Subject: [PATCH 13/22] sync2: fix mainnet/testnet configs --- config/mainnet.go | 6 ++++++ config/presets/testnet.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/config/mainnet.go b/config/mainnet.go index bf29839606..2af24825c9 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -225,6 +225,12 @@ func MainnetConfig() Config { OldAtxSyncCfg: oldAtxSyncCfg, NewAtxSyncCfg: newAtxSyncCfg, ParallelLoadLimit: 10, + HardTimeout: 10 * time.Minute, + ServerConfig: fetch.ServerConfig{ + Queue: 200, + Requests: 100, + Interval: time.Second, + }, }, }, Recovery: checkpoint.DefaultConfig(), diff --git a/config/presets/testnet.go b/config/presets/testnet.go index e7856e509c..476b4fe16b 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -175,6 +175,12 @@ func testnet() config.Config { OldAtxSyncCfg: oldAtxSyncCfg, NewAtxSyncCfg: newAtxSyncCfg, ParallelLoadLimit: 10, + HardTimeout: time.Minute, + ServerConfig: fetch.ServerConfig{ + Queue: 200, + Requests: 100, + Interval: time.Second, + }, }, }, Recovery: checkpoint.DefaultConfig(), From a8d87f119965d90580279e3b6e34d50bc6c55c9c Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 25 Nov 2024 16:32:19 +0400 Subject: [PATCH 14/22] sync2: only handle synced keys during commit It turned out that sync interactions are happening rather quickly, and thus it is not really practical to try and begin handling arriving keys (e.g. ATX IDs) during sync itself. Moreover, on-the-fly key handling was actually only used to register peer IDs for each ATX ID to fetch the actual blob from, and that can be done in the handler's Commit() method just as well. --- sync2/atxs.go | 23 ++- sync2/atxs_test.go | 97 ++++------- sync2/dbset/dbset.go | 7 +- sync2/dbset/dbset_test.go | 37 ++-- sync2/dbset/p2p_test.go | 4 +- sync2/multipeer/interface.go | 30 ++-- sync2/multipeer/mocks_test.go | 230 +++---------------------- sync2/multipeer/multipeer.go | 38 ++--- sync2/multipeer/multipeer_test.go | 54 +----- sync2/multipeer/setsyncbase.go | 154 ++++------------- sync2/multipeer/setsyncbase_test.go | 250 +++++----------------------- sync2/multipeer/split_sync.go | 57 +++---- sync2/multipeer/split_sync_test.go | 74 +++----- sync2/p2p.go | 14 +- sync2/p2p_test.go | 44 ++--- sync2/rangesync/dumbset.go | 6 +- sync2/rangesync/interface.go | 12 +- sync2/rangesync/mocks/mocks.go | 13 +- sync2/rangesync/seq.go | 19 ++- 19 files changed, 299 insertions(+), 864 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 90663f61e6..4bb2a57abb 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -62,21 +62,18 @@ func NewATXHandler( } } -func (h *ATXHandler) Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) { - var id types.ATXID - copy(id[:], k) - h.f.RegisterPeerHash(peer, id.Hash32()) - return false, nil -} - -func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { +func (h *ATXHandler) Commit( + ctx context.Context, + peer p2p.Peer, + base rangesync.OrderedSet, + received rangesync.SeqResult, +) error { h.logger.Debug("begin atx commit") defer h.logger.Debug("end atx commit") - sr := new.Received() var firstK rangesync.KeyBytes numDownloaded := 0 state := make(map[types.ATXID]int) - for k := range sr.Seq { + for k := range received.Seq { if firstK == nil { firstK = k } else if firstK.Compare(k) == 0 { @@ -89,9 +86,11 @@ func (h *ATXHandler) Commit(ctx context.Context, peer p2p.Peer, base, new ranges if found { continue } - state[types.BytesToATXID(k)] = 0 + id := types.BytesToATXID(k) + h.f.RegisterPeerHash(peer, id.Hash32()) + state[id] = 0 } - if err := sr.Error(); err != nil { + if err := received.Error(); err != nil { return fmt.Errorf("get item: %w", err) } total := len(state) diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go index ebfe3451a3..46db742898 100644 --- a/sync2/atxs_test.go +++ b/sync2/atxs_test.go @@ -23,6 +23,21 @@ import ( "github.com/spacemeshos/go-spacemesh/system" ) +func atxSeqResult(atxs []types.ATXID) rangesync.SeqResult { + return rangesync.SeqResult{ + Seq: func(yield func(k rangesync.KeyBytes) bool) { + // Received sequence may be cyclic and the handler should stop + // when it sees the first key again. + for _, atx := range atxs { + if !yield(atx.Bytes()) { + return + } + } + }, + Error: rangesync.NoSeqError, + } +} + func TestAtxHandler_Success(t *testing.T) { const ( batchSize = 4 @@ -41,13 +56,9 @@ func TestAtxHandler_Success(t *testing.T) { clock := clockwork.NewFakeClock() h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) f.EXPECT().RegisterPeerHash(peer, id.Hash32()) - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) - add, err := h.Receive(id.Bytes(), peer) - require.False(t, add) - require.NoError(t, err) } toFetch := make(map[types.ATXID]bool) for _, id := range allAtxs { @@ -69,21 +80,7 @@ func TestAtxHandler_Success(t *testing.T) { } return nil }).Times(3) - newSet.EXPECT().Received().Return(rangesync.SeqResult{ - Seq: func(yield func(k rangesync.KeyBytes) bool) { - // Received sequence may be cyclic and the handler should stop - // when it sees the first key again. - for { - for _, atx := range allAtxs { - if !yield(atx.Bytes()) { - return - } - } - } - }, - Error: rangesync.NoSeqError, - }) - require.NoError(t, h.Commit(context.Background(), peer, baseSet, newSet)) + require.NoError(t, h.Commit(context.Background(), peer, baseSet, atxSeqResult(allAtxs))) require.Empty(t, toFetch) require.Equal(t, []int{4, 4, 2}, batches) } @@ -106,13 +103,9 @@ func TestAtxHandler_Retry(t *testing.T) { clock := clockwork.NewFakeClock() h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) f.EXPECT().RegisterPeerHash(peer, id.Hash32()) - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) - add, err := h.Receive(id.Bytes(), peer) - require.False(t, add) - require.NoError(t, err) } failCount := 0 var fetched []types.ATXID @@ -150,16 +143,6 @@ func TestAtxHandler_Retry(t *testing.T) { } return nil }).AnyTimes() - newSet.EXPECT().Received().Return(rangesync.SeqResult{ - Seq: func(yield func(k rangesync.KeyBytes) bool) { - for _, atx := range allAtxs { - if !yield(atx.Bytes()) { - return - } - } - }, - Error: rangesync.NoSeqError, - }) // If it so happens that a full batch fails, we need to advance the clock to // trigger the retry. @@ -185,7 +168,7 @@ func TestAtxHandler_Retry(t *testing.T) { eg.Wait() }() - require.NoError(t, h.Commit(context.Background(), peer, baseSet, newSet)) + require.NoError(t, h.Commit(context.Background(), peer, baseSet, atxSeqResult(allAtxs))) require.ElementsMatch(t, allAtxs[1:], fetched) } @@ -204,19 +187,19 @@ func TestAtxHandler_Cancel(t *testing.T) { clock := clockwork.NewFakeClock() h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - newSet := mocks.NewMockOrderedSet(ctrl) baseSet.EXPECT().Has(rangesync.KeyBytes(atxID[:])).Return(false, nil) + f.EXPECT().RegisterPeerHash(peer, atxID.Hash32()) f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { return context.Canceled }) - newSet.EXPECT().Received().Return(rangesync.SeqResult{ + sr := rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { yield(atxID.Bytes()) }, Error: rangesync.NoSeqError, - }) - require.ErrorIs(t, h.Commit(context.Background(), peer, baseSet, newSet), context.Canceled) + } + require.ErrorIs(t, h.Commit(context.Background(), peer, baseSet, sr), context.Canceled) } func TestAtxHandler_BatchRetry(t *testing.T) { @@ -237,35 +220,17 @@ func TestAtxHandler_BatchRetry(t *testing.T) { f := NewMockFetcher(ctrl) h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) f.EXPECT().RegisterPeerHash(peer, id.Hash32()) - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) - add, err := h.Receive(id.Bytes(), peer) - require.False(t, add) - require.NoError(t, err) } - newSet.EXPECT().Received().Return(rangesync.SeqResult{ - Seq: func(yield func(k rangesync.KeyBytes) bool) { - // Received sequence may be cyclic and the handler should stop - // when it sees the first key again. - for { - for _, atx := range allAtxs { - if !yield(atx.Bytes()) { - return - } - } - } - }, - Error: rangesync.NoSeqError, - }) f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { return errors.New("fetch failed") }) var eg errgroup.Group eg.Go(func() error { - return h.Commit(context.Background(), peer, baseSet, newSet) + return h.Commit(context.Background(), peer, baseSet, atxSeqResult(allAtxs)) }) // wait for delay after 1st batch failure clock.BlockUntil(1) @@ -310,15 +275,11 @@ func TestAtxHandler_BatchRetry_Fail(t *testing.T) { f := NewMockFetcher(ctrl) h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - newSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { + baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) f.EXPECT().RegisterPeerHash(peer, id.Hash32()) - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])).Return(false, nil) - add, err := h.Receive(id.Bytes(), peer) - require.False(t, add) - require.NoError(t, err) } - newSet.EXPECT().Received().Return(rangesync.SeqResult{ + sr := rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { // Received sequence may be cyclic and the handler should stop // when it sees the first key again. @@ -331,14 +292,14 @@ func TestAtxHandler_BatchRetry_Fail(t *testing.T) { } }, Error: rangesync.NoSeqError, - }) + } f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { return errors.New("fetch failed") }).Times(3) var eg errgroup.Group eg.Go(func() error { - return h.Commit(context.Background(), peer, baseSet, newSet) + return h.Commit(context.Background(), peer, baseSet, sr) }) for range 2 { clock.BlockUntil(1) diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index 795ca24f1f..ed20293827 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -80,9 +80,10 @@ func (d *DBSet) EnsureLoaded() error { return d.snapshot.Load(d.db, d.handleIDfromDB) } -// Received returns a sequence of all items that have been received. +// Received returns a sequence of all items that have been received and the number of +// these items. // Implements rangesync.OrderedSet. -func (d *DBSet) Received() rangesync.SeqResult { +func (d *DBSet) Received() (rangesync.SeqResult, int) { return rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { for k := range d.received { @@ -92,7 +93,7 @@ func (d *DBSet) Received() rangesync.SeqResult { } }, Error: rangesync.NoSeqError, - } + }, len(d.received) } // Add adds an item to the DBSet. diff --git a/sync2/dbset/dbset_test.go b/sync2/dbset/dbset_test.go index 7970cb4830..1e0600a629 100644 --- a/sync2/dbset/dbset_test.go +++ b/sync2/dbset/dbset_test.go @@ -42,7 +42,9 @@ func TestDBSet_Empty(t *testing.T) { require.NoError(t, err) require.True(t, empty) requireEmpty(t, s.Items()) - requireEmpty(t, s.Received()) + sr, n := s.Received() + requireEmpty(t, sr) + require.Zero(t, n) info, err := s.GetRangeInfo(nil, nil) require.NoError(t, err) @@ -190,11 +192,12 @@ func TestDBSet_Receive(t *testing.T) { newID := rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000") require.NoError(t, s.Receive(newID)) - recvd := s.Received() + recvd, n := s.Received() items, err := recvd.FirstN(1) require.NoError(t, err) require.NoError(t, err) require.Equal(t, []rangesync.KeyBytes{newID}, items) + require.Equal(t, 1, n) info, err := s.GetRangeInfo(ids[2], ids[0]) require.NoError(t, err) @@ -235,9 +238,9 @@ func TestDBSet_Copy(t *testing.T) { require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) require.Equal(t, ids[2], firstKey(t, info.Items)) - items, err := s.Received().FirstN(100) - require.NoError(t, err) - require.Empty(t, items) + sr, n := s.Received() + requireEmpty(t, sr) + require.Zero(t, n) info, err = s.GetRangeInfo(ids[2], ids[0]) require.NoError(t, err) @@ -245,7 +248,9 @@ func TestDBSet_Copy(t *testing.T) { require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) require.Equal(t, ids[2], firstKey(t, info.Items)) - items, err = copy.(*dbset.DBSet).Received().FirstN(100) + sr, n = copy.(*dbset.DBSet).Received() + require.Equal(t, 1, n) + items, err := sr.FirstN(100) require.NoError(t, err) require.Equal(t, []rangesync.KeyBytes{newID}, items) @@ -356,29 +361,35 @@ func TestDBSet_Added(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - requireEmpty(t, s.Received()) + sr, n := s.Received() + requireEmpty(t, sr) + require.Zero(t, n) - add := []rangesync.KeyBytes{ + recv := []rangesync.KeyBytes{ rangesync.MustParseHexKeyBytes("3333333333333333333333333333333333333333333333333333333333333333"), rangesync.MustParseHexKeyBytes("4444444444444444444444444444444444444444444444444444444444444444"), } - for _, item := range add { + for _, item := range recv { require.NoError(t, s.Receive(item)) } require.NoError(t, s.EnsureLoaded()) - added, err := s.Received().FirstN(3) + sr, n = s.Received() + require.Equal(t, 2, n) + recvd, err := sr.FirstN(3) require.NoError(t, err) require.ElementsMatch(t, []rangesync.KeyBytes{ rangesync.MustParseHexKeyBytes("3333333333333333333333333333333333333333333333333333333333333333"), rangesync.MustParseHexKeyBytes("4444444444444444444444444444444444444444444444444444444444444444"), - }, added) + }, recvd) require.NoError(t, s.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { - added1, err := copy.(*dbset.DBSet).Received().FirstN(3) + sr, n := copy.(*dbset.DBSet).Received() + require.Equal(t, 2, n) + recvd1, err := sr.FirstN(3) require.NoError(t, err) - require.ElementsMatch(t, added, added1) + require.ElementsMatch(t, recvd, recvd1) return nil })) } diff --git a/sync2/dbset/p2p_test.go b/sync2/dbset/p2p_test.go index c636c461ba..457b538352 100644 --- a/sync2/dbset/p2p_test.go +++ b/sync2/dbset/p2p_test.go @@ -81,8 +81,9 @@ func (tr *syncTracer) OnRecent(receivedItems, sentItems int) { } func addReceived(t testing.TB, db sql.Executor, to, from *dbset.DBSet) { - sr := from.Received() + sr, n := from.Received() for k := range sr.Seq { + n-- has, err := to.Has(k) require.NoError(t, err) if !has { @@ -91,6 +92,7 @@ func addReceived(t testing.TB, db sql.Executor, to, from *dbset.DBSet) { } require.NoError(t, sr.Error()) require.NoError(t, to.Advance()) + require.Zero(t, n) } type startStopTimer interface { diff --git a/sync2/multipeer/interface.go b/sync2/multipeer/interface.go index b3a5236171..2d65699256 100644 --- a/sync2/multipeer/interface.go +++ b/sync2/multipeer/interface.go @@ -11,38 +11,28 @@ import ( //go:generate mockgen -typed -package=multipeer_test -destination=./mocks_test.go -source=./interface.go // SyncBase is a synchronization base which holds the original OrderedSet. -// It is used to derive per-peer PeerSyncers with their own copies of the OrderedSet, -// copy operation being O(1) in terms of memory and time complexity. +// It is used to sync against peers using derived OrderedSets. // It can also probe peers to decide on the synchronization strategy. type SyncBase interface { // Count returns the number of items in the set. Count() (int, error) - // WithPeerSyncer creates a Syncer for the specified peer and passes it to the specified function. - // When the function returns, the syncer is discarded, releasing the resources associated with it. - WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(PeerSyncer) error) error + // Sync synchronizes the set with the peer. + // It returns a sequence of new keys that were received from the peer and the + // number of received items. + Sync(ctx context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error + // Serve serves a synchronization request on the specified stream. + // It returns a sequence of new keys that were received from the peer and the + // number of received items. + Serve(ctx context.Context, p p2p.Peer, stream io.ReadWriter) error // Probe probes the specified peer, obtaining its set fingerprint, // the number of items and the similarity value. Probe(ctx context.Context, p p2p.Peer) (rangesync.ProbeResult, error) - // Wait waits for all the derived syncers' handlers to finish. - Wait() error -} - -// PeerSyncer is a synchronization interface for a single peer. -type PeerSyncer interface { - // Peer returns the peer this syncer is for. - Peer() p2p.Peer - // Sync synchronizes the set with the peer. - Sync(ctx context.Context, x, y rangesync.KeyBytes) error - // Serve serves a synchronization request on the specified stream. - Serve(ctx context.Context, stream io.ReadWriter) error } // SyncKeyHandler is a handler for keys that are received from peers. type SyncKeyHandler interface { - // Receive handles a key that was received from a peer. - Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) // Commit is invoked at the end of synchronization to apply the changes. - Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error + Commit(ctx context.Context, peer p2p.Peer, base rangesync.OrderedSet, received rangesync.SeqResult) error } // PairwiseSyncer is used to probe a peer or sync against a single peer. diff --git a/sync2/multipeer/mocks_test.go b/sync2/multipeer/mocks_test.go index 48c0401b5b..dd78ac583d 100644 --- a/sync2/multipeer/mocks_test.go +++ b/sync2/multipeer/mocks_test.go @@ -15,7 +15,6 @@ import ( reflect "reflect" p2p "github.com/spacemeshos/go-spacemesh/p2p" - multipeer "github.com/spacemeshos/go-spacemesh/sync2/multipeer" rangesync "github.com/spacemeshos/go-spacemesh/sync2/rangesync" gomock "go.uber.org/mock/gomock" ) @@ -122,216 +121,78 @@ func (c *MockSyncBaseProbeCall) DoAndReturn(f func(context.Context, p2p.Peer) (r return c } -// Wait mocks base method. -func (m *MockSyncBase) Wait() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Wait") - ret0, _ := ret[0].(error) - return ret0 -} - -// Wait indicates an expected call of Wait. -func (mr *MockSyncBaseMockRecorder) Wait() *MockSyncBaseWaitCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Wait", reflect.TypeOf((*MockSyncBase)(nil).Wait)) - return &MockSyncBaseWaitCall{Call: call} -} - -// MockSyncBaseWaitCall wrap *gomock.Call -type MockSyncBaseWaitCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSyncBaseWaitCall) Return(arg0 error) *MockSyncBaseWaitCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSyncBaseWaitCall) Do(f func() error) *MockSyncBaseWaitCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncBaseWaitCall) DoAndReturn(f func() error) *MockSyncBaseWaitCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// WithPeerSyncer mocks base method. -func (m *MockSyncBase) WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(multipeer.PeerSyncer) error) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WithPeerSyncer", ctx, p, toCall) - ret0, _ := ret[0].(error) - return ret0 -} - -// WithPeerSyncer indicates an expected call of WithPeerSyncer. -func (mr *MockSyncBaseMockRecorder) WithPeerSyncer(ctx, p, toCall any) *MockSyncBaseWithPeerSyncerCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithPeerSyncer", reflect.TypeOf((*MockSyncBase)(nil).WithPeerSyncer), ctx, p, toCall) - return &MockSyncBaseWithPeerSyncerCall{Call: call} -} - -// MockSyncBaseWithPeerSyncerCall wrap *gomock.Call -type MockSyncBaseWithPeerSyncerCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSyncBaseWithPeerSyncerCall) Return(arg0 error) *MockSyncBaseWithPeerSyncerCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSyncBaseWithPeerSyncerCall) Do(f func(context.Context, p2p.Peer, func(multipeer.PeerSyncer) error) error) *MockSyncBaseWithPeerSyncerCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncBaseWithPeerSyncerCall) DoAndReturn(f func(context.Context, p2p.Peer, func(multipeer.PeerSyncer) error) error) *MockSyncBaseWithPeerSyncerCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// MockPeerSyncer is a mock of PeerSyncer interface. -type MockPeerSyncer struct { - ctrl *gomock.Controller - recorder *MockPeerSyncerMockRecorder - isgomock struct{} -} - -// MockPeerSyncerMockRecorder is the mock recorder for MockPeerSyncer. -type MockPeerSyncerMockRecorder struct { - mock *MockPeerSyncer -} - -// NewMockPeerSyncer creates a new mock instance. -func NewMockPeerSyncer(ctrl *gomock.Controller) *MockPeerSyncer { - mock := &MockPeerSyncer{ctrl: ctrl} - mock.recorder = &MockPeerSyncerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPeerSyncer) EXPECT() *MockPeerSyncerMockRecorder { - return m.recorder -} - -// Peer mocks base method. -func (m *MockPeerSyncer) Peer() p2p.Peer { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Peer") - ret0, _ := ret[0].(p2p.Peer) - return ret0 -} - -// Peer indicates an expected call of Peer. -func (mr *MockPeerSyncerMockRecorder) Peer() *MockPeerSyncerPeerCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peer", reflect.TypeOf((*MockPeerSyncer)(nil).Peer)) - return &MockPeerSyncerPeerCall{Call: call} -} - -// MockPeerSyncerPeerCall wrap *gomock.Call -type MockPeerSyncerPeerCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockPeerSyncerPeerCall) Return(arg0 p2p.Peer) *MockPeerSyncerPeerCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockPeerSyncerPeerCall) Do(f func() p2p.Peer) *MockPeerSyncerPeerCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockPeerSyncerPeerCall) DoAndReturn(f func() p2p.Peer) *MockPeerSyncerPeerCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Serve mocks base method. -func (m *MockPeerSyncer) Serve(ctx context.Context, stream io.ReadWriter) error { +func (m *MockSyncBase) Serve(ctx context.Context, p p2p.Peer, stream io.ReadWriter) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Serve", ctx, stream) + ret := m.ctrl.Call(m, "Serve", ctx, p, stream) ret0, _ := ret[0].(error) return ret0 } // Serve indicates an expected call of Serve. -func (mr *MockPeerSyncerMockRecorder) Serve(ctx, stream any) *MockPeerSyncerServeCall { +func (mr *MockSyncBaseMockRecorder) Serve(ctx, p, stream any) *MockSyncBaseServeCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Serve", reflect.TypeOf((*MockPeerSyncer)(nil).Serve), ctx, stream) - return &MockPeerSyncerServeCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Serve", reflect.TypeOf((*MockSyncBase)(nil).Serve), ctx, p, stream) + return &MockSyncBaseServeCall{Call: call} } -// MockPeerSyncerServeCall wrap *gomock.Call -type MockPeerSyncerServeCall struct { +// MockSyncBaseServeCall wrap *gomock.Call +type MockSyncBaseServeCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockPeerSyncerServeCall) Return(arg0 error) *MockPeerSyncerServeCall { +func (c *MockSyncBaseServeCall) Return(arg0 error) *MockSyncBaseServeCall { c.Call = c.Call.Return(arg0) return c } // Do rewrite *gomock.Call.Do -func (c *MockPeerSyncerServeCall) Do(f func(context.Context, io.ReadWriter) error) *MockPeerSyncerServeCall { +func (c *MockSyncBaseServeCall) Do(f func(context.Context, p2p.Peer, io.ReadWriter) error) *MockSyncBaseServeCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockPeerSyncerServeCall) DoAndReturn(f func(context.Context, io.ReadWriter) error) *MockPeerSyncerServeCall { +func (c *MockSyncBaseServeCall) DoAndReturn(f func(context.Context, p2p.Peer, io.ReadWriter) error) *MockSyncBaseServeCall { c.Call = c.Call.DoAndReturn(f) return c } // Sync mocks base method. -func (m *MockPeerSyncer) Sync(ctx context.Context, x, y rangesync.KeyBytes) error { +func (m *MockSyncBase) Sync(ctx context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Sync", ctx, x, y) + ret := m.ctrl.Call(m, "Sync", ctx, p, x, y) ret0, _ := ret[0].(error) return ret0 } // Sync indicates an expected call of Sync. -func (mr *MockPeerSyncerMockRecorder) Sync(ctx, x, y any) *MockPeerSyncerSyncCall { +func (mr *MockSyncBaseMockRecorder) Sync(ctx, p, x, y any) *MockSyncBaseSyncCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sync", reflect.TypeOf((*MockPeerSyncer)(nil).Sync), ctx, x, y) - return &MockPeerSyncerSyncCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sync", reflect.TypeOf((*MockSyncBase)(nil).Sync), ctx, p, x, y) + return &MockSyncBaseSyncCall{Call: call} } -// MockPeerSyncerSyncCall wrap *gomock.Call -type MockPeerSyncerSyncCall struct { +// MockSyncBaseSyncCall wrap *gomock.Call +type MockSyncBaseSyncCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockPeerSyncerSyncCall) Return(arg0 error) *MockPeerSyncerSyncCall { +func (c *MockSyncBaseSyncCall) Return(arg0 error) *MockSyncBaseSyncCall { c.Call = c.Call.Return(arg0) return c } // Do rewrite *gomock.Call.Do -func (c *MockPeerSyncerSyncCall) Do(f func(context.Context, rangesync.KeyBytes, rangesync.KeyBytes) error) *MockPeerSyncerSyncCall { +func (c *MockSyncBaseSyncCall) Do(f func(context.Context, p2p.Peer, rangesync.KeyBytes, rangesync.KeyBytes) error) *MockSyncBaseSyncCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockPeerSyncerSyncCall) DoAndReturn(f func(context.Context, rangesync.KeyBytes, rangesync.KeyBytes) error) *MockPeerSyncerSyncCall { +func (c *MockSyncBaseSyncCall) DoAndReturn(f func(context.Context, p2p.Peer, rangesync.KeyBytes, rangesync.KeyBytes) error) *MockSyncBaseSyncCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -361,17 +222,17 @@ func (m *MockSyncKeyHandler) EXPECT() *MockSyncKeyHandlerMockRecorder { } // Commit mocks base method. -func (m *MockSyncKeyHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { +func (m *MockSyncKeyHandler) Commit(ctx context.Context, peer p2p.Peer, base rangesync.OrderedSet, received rangesync.SeqResult) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Commit", ctx, peer, base, new) + ret := m.ctrl.Call(m, "Commit", ctx, peer, base, received) ret0, _ := ret[0].(error) return ret0 } // Commit indicates an expected call of Commit. -func (mr *MockSyncKeyHandlerMockRecorder) Commit(ctx, peer, base, new any) *MockSyncKeyHandlerCommitCall { +func (mr *MockSyncKeyHandlerMockRecorder) Commit(ctx, peer, base, received any) *MockSyncKeyHandlerCommitCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockSyncKeyHandler)(nil).Commit), ctx, peer, base, new) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockSyncKeyHandler)(nil).Commit), ctx, peer, base, received) return &MockSyncKeyHandlerCommitCall{Call: call} } @@ -387,52 +248,13 @@ func (c *MockSyncKeyHandlerCommitCall) Return(arg0 error) *MockSyncKeyHandlerCom } // Do rewrite *gomock.Call.Do -func (c *MockSyncKeyHandlerCommitCall) Do(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncKeyHandlerCommitCall) DoAndReturn(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.OrderedSet) error) *MockSyncKeyHandlerCommitCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Receive mocks base method. -func (m *MockSyncKeyHandler) Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Receive", k, peer) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Receive indicates an expected call of Receive. -func (mr *MockSyncKeyHandlerMockRecorder) Receive(k, peer any) *MockSyncKeyHandlerReceiveCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockSyncKeyHandler)(nil).Receive), k, peer) - return &MockSyncKeyHandlerReceiveCall{Call: call} -} - -// MockSyncKeyHandlerReceiveCall wrap *gomock.Call -type MockSyncKeyHandlerReceiveCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSyncKeyHandlerReceiveCall) Return(arg0 bool, arg1 error) *MockSyncKeyHandlerReceiveCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSyncKeyHandlerReceiveCall) Do(f func(rangesync.KeyBytes, p2p.Peer) (bool, error)) *MockSyncKeyHandlerReceiveCall { +func (c *MockSyncKeyHandlerCommitCall) Do(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.SeqResult) error) *MockSyncKeyHandlerCommitCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSyncKeyHandlerReceiveCall) DoAndReturn(f func(rangesync.KeyBytes, p2p.Peer) (bool, error)) *MockSyncKeyHandlerReceiveCall { +func (c *MockSyncKeyHandlerCommitCall) DoAndReturn(f func(context.Context, p2p.Peer, rangesync.OrderedSet, rangesync.SeqResult) error) *MockSyncKeyHandlerCommitCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/sync2/multipeer/multipeer.go b/sync2/multipeer/multipeer.go index 00832edc38..f0b961f9d8 100644 --- a/sync2/multipeer/multipeer.go +++ b/sync2/multipeer/multipeer.go @@ -3,7 +3,6 @@ package multipeer import ( "context" "errors" - "fmt" "math" "math/rand/v2" "sync/atomic" @@ -277,24 +276,17 @@ func (mpr *MultiPeerReconciler) fullSync(ctx context.Context, syncPeers []p2p.Pe var someSucceeded atomic.Bool for _, p := range syncPeers { eg.Go(func() error { - if err := mpr.syncBase.WithPeerSyncer(ctx, p, func(ps PeerSyncer) error { - err := ps.Sync(ctx, nil, nil) - switch { - case err == nil: - someSucceeded.Store(true) - mpr.sl.NoteSync() - case errors.Is(err, context.Canceled): - return err - default: - // failing to sync against a particular peer is not considered - // a fatal sync failure, so we just log the error - mpr.logger.Error("error syncing peer", - zap.Stringer("peer", p), - zap.Error(err)) - } - return nil - }); err != nil { - return fmt.Errorf("sync %s: %w", p, err) + err := mpr.syncBase.Sync(ctx, p, nil, nil) + switch { + case err == nil: + someSucceeded.Store(true) + mpr.sl.NoteSync() + case errors.Is(err, context.Canceled): + return err + default: + // failing to sync against a particular peer is not considered + // a fatal sync failure, so we just log the error + mpr.logger.Error("error syncing peer", zap.Stringer("peer", p), zap.Error(err)) } return nil }) @@ -356,11 +348,6 @@ func (mpr *MultiPeerReconciler) syncOnce(ctx context.Context, lastWasSplit bool) } } - // handler errors are not fatal - if handlerErr := mpr.syncBase.Wait(); handlerErr != nil { - mpr.logger.Error("error handling synced keys", zap.Error(handlerErr)) - } - return full, err } @@ -431,9 +418,6 @@ LOOP: case <-kickCh: } } - // The loop is only exited upon context cancellation. - // Thus, syncBase.Wait() is guaranteed not to block indefinitely here. - mpr.syncBase.Wait() return err } diff --git a/sync2/multipeer/multipeer_test.go b/sync2/multipeer/multipeer_test.go index 4bd05227fe..70061e94f0 100644 --- a/sync2/multipeer/multipeer_test.go +++ b/sync2/multipeer/multipeer_test.go @@ -157,23 +157,17 @@ func (mt *multiPeerSyncTester) expectFullSync(pl *peerList, times, numFails int) // delegate to the real fullsync return mt.reconciler.FullSync(ctx, peers) }) - mt.syncBase.EXPECT().WithPeerSyncer(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func( - _ context.Context, - p p2p.Peer, - toCall func(multipeer.PeerSyncer) error, - ) error { + mt.syncBase.EXPECT(). + Sync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error { mt.mtx.Lock() defer mt.mtx.Unlock() require.Contains(mt, pl.get(), p) - s := NewMockPeerSyncer(mt.ctrl) - s.EXPECT().Peer().Return(p).AnyTimes() - expSync := s.EXPECT().Sync(gomock.Any(), gomock.Nil(), gomock.Nil()) if numFails != 0 { - expSync.Return(errors.New("sync failed")) numFails-- + return errors.New("sync failed") } - return toCall(s) + return nil }).Times(times) } @@ -210,7 +204,6 @@ func TestMultiPeerSync(t *testing.T) { require.ElementsMatch(t, plSplit.get(), peers) return nil }) - mt.syncBase.EXPECT().Wait() mt.clock.BlockUntilContext(ctx, 1) plFull := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{ FP: "foo", @@ -218,7 +211,6 @@ func TestMultiPeerSync(t *testing.T) { Sim: 1, // after sync }) mt.expectFullSync(plFull, numSyncPeers, 0) - mt.syncBase.EXPECT().Wait() if i > 0 { mt.clock.Advance(time.Minute) } else if i < numSyncs-1 { @@ -226,7 +218,6 @@ func TestMultiPeerSync(t *testing.T) { } mt.satisfy() } - mt.syncBase.EXPECT().Wait() }) t.Run("full sync", func(t *testing.T) { @@ -240,7 +231,6 @@ func TestMultiPeerSync(t *testing.T) { Sim: 0.99, // high enough for full sync }) mt.expectFullSync(pl, numSyncPeers, 0) - mt.syncBase.EXPECT().Wait() } expect() // first full sync happens immediately @@ -254,7 +244,6 @@ func TestMultiPeerSync(t *testing.T) { mt.satisfy() } require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() }) t.Run("sync after kick", func(t *testing.T) { @@ -268,7 +257,6 @@ func TestMultiPeerSync(t *testing.T) { Sim: 0.99, // high enough for full sync }) mt.expectFullSync(pl, numSyncPeers, 0) - mt.syncBase.EXPECT().Wait() } expect() // first full sync happens immediately @@ -282,7 +270,6 @@ func TestMultiPeerSync(t *testing.T) { mt.satisfy() } require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() }) t.Run("full sync, peers with low count ignored", func(t *testing.T) { @@ -306,7 +293,6 @@ func TestMultiPeerSync(t *testing.T) { Sim: 0.9, }) mt.expectFullSync(&pl, 5, 0) - mt.syncBase.EXPECT().Wait() } expect() // first full sync happens immediately @@ -320,7 +306,6 @@ func TestMultiPeerSync(t *testing.T) { mt.satisfy() } require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() }) t.Run("full sync due to low peer count", func(t *testing.T) { @@ -333,7 +318,6 @@ func TestMultiPeerSync(t *testing.T) { Sim: 0.5, // too low for full sync, but will have it anyway }) mt.expectFullSync(pl, 1, 0) - mt.syncBase.EXPECT().Wait() } expect() ctx := mt.start() @@ -346,7 +330,6 @@ func TestMultiPeerSync(t *testing.T) { mt.satisfy() } require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() }) t.Run("probe failure", func(t *testing.T) { @@ -357,7 +340,6 @@ func TestMultiPeerSync(t *testing.T) { pl := mt.expectProbe(5, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) // just 5 peers for which the probe worked will be checked mt.expectFullSync(pl, 5, 0) - mt.syncBase.EXPECT().Wait().Times(2) ctx := mt.start() mt.clock.BlockUntilContext(ctx, 1) }) @@ -369,7 +351,6 @@ func TestMultiPeerSync(t *testing.T) { expect := func() { pl := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) mt.expectFullSync(pl, numSyncPeers, numFails) - mt.syncBase.EXPECT().Wait() } expect() ctx := mt.start() @@ -382,7 +363,6 @@ func TestMultiPeerSync(t *testing.T) { mt.satisfy() } require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() }) t.Run("all peers failed during full sync", func(t *testing.T) { @@ -391,7 +371,6 @@ func TestMultiPeerSync(t *testing.T) { pl := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) mt.expectFullSync(pl, numSyncPeers, numSyncPeers) - mt.syncBase.EXPECT().Wait().AnyTimes() ctx := mt.start() mt.clock.BlockUntilContext(ctx, 1) @@ -407,28 +386,6 @@ func TestMultiPeerSync(t *testing.T) { require.True(t, mt.reconciler.Synced()) }) - t.Run("failed synced key handling during full sync", func(t *testing.T) { - mt := newMultiPeerSyncTester(t, 10) - mt.syncBase.EXPECT().Count().Return(100, nil).AnyTimes() - expect := func() { - pl := mt.expectProbe(numSyncPeers, rangesync.ProbeResult{FP: "foo", Count: 100, Sim: 0.99}) - mt.expectFullSync(pl, numSyncPeers, 0) - mt.syncBase.EXPECT().Wait().Return(errors.New("some handlers failed")) - } - expect() - ctx := mt.start() - mt.clock.BlockUntilContext(ctx, 1) - mt.satisfy() - for i := 0; i < numSyncs; i++ { - expect() - mt.clock.Advance(time.Minute) - mt.clock.BlockUntilContext(ctx, 1) - mt.satisfy() - } - require.True(t, mt.reconciler.Synced()) - mt.syncBase.EXPECT().Wait() - }) - t.Run("cancellation during sync", func(t *testing.T) { mt := newMultiPeerSyncTester(t, 10) mt.syncBase.EXPECT().Count().Return(100, nil).AnyTimes() @@ -438,7 +395,6 @@ func TestMultiPeerSync(t *testing.T) { mt.cancel() return ctx.Err() }) - mt.syncBase.EXPECT().Wait().Times(2) ctx := mt.start() mt.clock.BlockUntilContext(ctx, 1) require.ErrorIs(t, mt.eg.Wait(), context.Canceled) diff --git a/sync2/multipeer/setsyncbase.go b/sync2/multipeer/setsyncbase.go index 4c4b95a261..d0b44225d0 100644 --- a/sync2/multipeer/setsyncbase.go +++ b/sync2/multipeer/setsyncbase.go @@ -2,14 +2,10 @@ package multipeer import ( "context" - "errors" "fmt" "io" "sync" - "go.uber.org/zap" - "golang.org/x/sync/singleflight" - "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/sync2/rangesync" ) @@ -20,25 +16,20 @@ import ( // has not been yet received and validated. type SetSyncBase struct { mtx sync.Mutex - logger *zap.Logger ps PairwiseSyncer os rangesync.OrderedSet handler SyncKeyHandler - waiting []<-chan singleflight.Result - g singleflight.Group } var _ SyncBase = &SetSyncBase{} // NewSetSyncBase creates a new SetSyncBase. func NewSetSyncBase( - logger *zap.Logger, ps PairwiseSyncer, os rangesync.OrderedSet, handler SyncKeyHandler, ) *SetSyncBase { return &SetSyncBase{ - logger: logger, ps: ps, os: os, handler: handler, @@ -66,133 +57,56 @@ func (ssb *SetSyncBase) Count() (int, error) { return info.Count, nil } -// WithPeerSyncer implements SyncBase. -func (ssb *SetSyncBase) WithPeerSyncer(ctx context.Context, p p2p.Peer, toCall func(PeerSyncer) error) error { - return ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { - return toCall(&peerSetSyncer{ - SetSyncBase: ssb, - OrderedSet: os, - p: p, - handler: ssb.handler, - }) - }) -} - -// Probe implements SyncBase. -func (ssb *SetSyncBase) Probe(ctx context.Context, p p2p.Peer) (pr rangesync.ProbeResult, err error) { - // Use a snapshot of the store to avoid holding the mutex for a long time +func (ssb *SetSyncBase) syncPeer( + ctx context.Context, + p p2p.Peer, + toCall func(rangesync.OrderedSet) error, +) error { + sr := rangesync.EmptySeqResult() + var n int if err := ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { - pr, err = ssb.ps.Probe(ctx, p, os, nil, nil) - if err != nil { - return fmt.Errorf("probing peer %s: %w", p, err) + if err := toCall(os); err != nil { + return err } + sr, n = os.Received() return nil }); err != nil { - return rangesync.ProbeResult{}, fmt.Errorf("using set copy for probe: %w", err) - } - - return pr, nil -} - -func (ssb *SetSyncBase) receiveKey(k rangesync.KeyBytes, p p2p.Peer) error { - ssb.mtx.Lock() - defer ssb.mtx.Unlock() - key := k.String() - has, err := ssb.os.Has(k) - if err != nil { - return fmt.Errorf("checking if the key is present: %w", err) - } - if !has { - ssb.waiting = append(ssb.waiting, - ssb.g.DoChan(key, func() (any, error) { - addToOrig, err := ssb.handler.Receive(k, p) - if err == nil && addToOrig { - ssb.mtx.Lock() - defer ssb.mtx.Unlock() - err = ssb.os.Receive(k) - } - return key, err - })) + return fmt.Errorf("sync: %w", err) } - return nil -} - -// Wait waits for all the handlers used by derived syncers to finish. -func (ssb *SetSyncBase) Wait() error { - // At this point, the derived syncers should be done syncing, and we only want to - // wait for the remaining handlers to complete. In case if some syncers happen to - // be still running at this point, let's not fail too badly. - // TODO: wait for any derived running syncers here, too - ssb.mtx.Lock() - waiting := ssb.waiting - ssb.waiting = nil - ssb.mtx.Unlock() - gotError := false - for _, w := range waiting { - r := <-w - key := r.Val.(string) - ssb.g.Forget(key) - if r.Err != nil { - gotError = true - ssb.logger.Error("error from key handler", zap.String("key", key), zap.Error(r.Err)) + if n > 0 { + if err := ssb.handler.Commit(ctx, p, ssb.os, sr); err != nil { + return fmt.Errorf("commit: %w", err) } } - if gotError { - return errors.New("some key handlers failed") - } - return nil -} - -func (ssb *SetSyncBase) advance() error { ssb.mtx.Lock() defer ssb.mtx.Unlock() return ssb.os.Advance() } -type peerSetSyncer struct { - *SetSyncBase - rangesync.OrderedSet - p p2p.Peer - handler SyncKeyHandler -} - -var ( - _ PeerSyncer = &peerSetSyncer{} - _ rangesync.OrderedSet = &peerSetSyncer{} -) - -// Peer implements Syncer. -func (pss *peerSetSyncer) Peer() p2p.Peer { - return pss.p -} - -// Sync implements Syncer. -func (pss *peerSetSyncer) Sync(ctx context.Context, x, y rangesync.KeyBytes) error { - if err := pss.ps.Sync(ctx, pss.p, pss, x, y); err != nil { - return err - } - return pss.commit(ctx) +func (ssb *SetSyncBase) Sync(ctx context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error { + return ssb.syncPeer(ctx, p, func(os rangesync.OrderedSet) error { + return ssb.ps.Sync(ctx, p, os, x, y) + }) } -// Serve implements Syncer. -func (pss *peerSetSyncer) Serve(ctx context.Context, stream io.ReadWriter) error { - if err := pss.ps.Serve(ctx, stream, pss); err != nil { - return err - } - return pss.commit(ctx) +func (ssb *SetSyncBase) Serve(ctx context.Context, p p2p.Peer, stream io.ReadWriter) error { + return ssb.syncPeer(ctx, p, func(os rangesync.OrderedSet) error { + return ssb.ps.Serve(ctx, stream, os) + }) } -// Receive implements OrderedSet. -func (pss *peerSetSyncer) Receive(k rangesync.KeyBytes) error { - if err := pss.receiveKey(k, pss.p); err != nil { - return err +// Probe implements SyncBase. +func (ssb *SetSyncBase) Probe(ctx context.Context, p p2p.Peer) (pr rangesync.ProbeResult, err error) { + // Use a snapshot of the store to avoid holding the mutex for a long time + if err := ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { + pr, err = ssb.ps.Probe(ctx, p, os, nil, nil) + if err != nil { + return fmt.Errorf("probing peer %s: %w", p, err) + } + return nil + }); err != nil { + return rangesync.ProbeResult{}, fmt.Errorf("using set copy for probe: %w", err) } - return pss.OrderedSet.Receive(k) -} -func (pss *peerSetSyncer) commit(ctx context.Context) error { - if err := pss.handler.Commit(ctx, pss.p, pss.SetSyncBase.os, pss.OrderedSet); err != nil { - return err - } - return pss.SetSyncBase.advance() + return pr, nil } diff --git a/sync2/multipeer/setsyncbase_test.go b/sync2/multipeer/setsyncbase_test.go index 234f454cc9..4a2ae3d365 100644 --- a/sync2/multipeer/setsyncbase_test.go +++ b/sync2/multipeer/setsyncbase_test.go @@ -2,14 +2,10 @@ package multipeer_test import ( "context" - "errors" - "sync" "testing" "github.com/stretchr/testify/require" gomock "go.uber.org/mock/gomock" - "go.uber.org/zap/zaptest" - "golang.org/x/sync/errgroup" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/sync2/multipeer" @@ -24,97 +20,33 @@ type setSyncBaseTester struct { handler *MockSyncKeyHandler os *mocks.MockOrderedSet ssb *multipeer.SetSyncBase - waitMtx sync.Mutex - waitChs map[string]chan error - doneCh chan rangesync.KeyBytes } func newSetSyncBaseTester(t *testing.T, os rangesync.OrderedSet) *setSyncBaseTester { ctrl := gomock.NewController(t) st := &setSyncBaseTester{ - T: t, - ctrl: ctrl, - ps: NewMockPairwiseSyncer(ctrl), - waitChs: make(map[string]chan error), - doneCh: make(chan rangesync.KeyBytes), + T: t, + ctrl: ctrl, + ps: NewMockPairwiseSyncer(ctrl), } if os == nil { st.os = mocks.NewMockOrderedSet(ctrl) - st.os.EXPECT().Items().DoAndReturn(func() rangesync.SeqResult { - return rangesync.EmptySeqResult() - }).AnyTimes() os = st.os } st.handler = NewMockSyncKeyHandler(ctrl) - st.handler.EXPECT().Receive(gomock.Any(), gomock.Any()). - DoAndReturn(func(k rangesync.KeyBytes, p p2p.Peer) (bool, error) { - err := <-st.getWaitCh(k) - st.doneCh <- k - return true, err - }).AnyTimes() - st.ssb = multipeer.NewSetSyncBase(zaptest.NewLogger(t), st.ps, os, st.handler) + st.ssb = multipeer.NewSetSyncBase(st.ps, os, st.handler) return st } -func (st *setSyncBaseTester) getWaitCh(k rangesync.KeyBytes) chan error { - st.waitMtx.Lock() - defer st.waitMtx.Unlock() - ch, found := st.waitChs[string(k)] - if !found { - ch = make(chan error) - st.waitChs[string(k)] = ch - } - return ch -} - -func (st *setSyncBaseTester) expectCopy(addedKeys ...rangesync.KeyBytes) *mocks.MockOrderedSet { +func (st *setSyncBaseTester) expectCopy() *mocks.MockOrderedSet { copy := mocks.NewMockOrderedSet(st.ctrl) st.os.EXPECT().WithCopy(gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, toCall func(rangesync.OrderedSet) error) error { - copy.EXPECT().Items().DoAndReturn(func() rangesync.SeqResult { - return rangesync.EmptySeqResult() - }).AnyTimes() - for _, k := range addedKeys { - copy.EXPECT().Receive(k) - } return toCall(copy) }) return copy } -func (st *setSyncBaseTester) expectSync( - p p2p.Peer, - ss multipeer.PeerSyncer, - addedKeys ...rangesync.KeyBytes, -) { - st.ps.EXPECT().Sync(gomock.Any(), p, ss, nil, nil). - DoAndReturn(func( - _ context.Context, - p p2p.Peer, - os rangesync.OrderedSet, - x, y rangesync.KeyBytes, - ) error { - for _, k := range addedKeys { - require.NoError(st, os.Receive(k)) - } - return nil - }) -} - -func (st *setSyncBaseTester) wait(count int) ([]rangesync.KeyBytes, error) { - var eg errgroup.Group - eg.Go(st.ssb.Wait) - var handledKeys []rangesync.KeyBytes - for k := range st.doneCh { - handledKeys = append(handledKeys, k.Clone()) - count-- - if count == 0 { - break - } - } - return handledKeys, eg.Wait() -} - func TestSetSyncBase(t *testing.T) { t.Run("probe", func(t *testing.T) { t.Parallel() @@ -131,160 +63,56 @@ func TestSetSyncBase(t *testing.T) { require.Equal(t, expPr, pr) }) - t.Run("single key one-time sync", func(t *testing.T) { - t.Parallel() - st := newSetSyncBaseTester(t, nil) - - addedKey := rangesync.RandomKeyBytes(32) - st.expectCopy(addedKey) - require.NoError(t, st.ssb.WithPeerSyncer( - context.Background(), p2p.Peer("p1"), - func(ps multipeer.PeerSyncer) error { - require.Equal(t, p2p.Peer("p1"), ps.Peer()) - - x := rangesync.RandomKeyBytes(32) - y := rangesync.RandomKeyBytes(32) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), ps, x, y) - require.NoError(t, ps.Sync(context.Background(), x, y)) - - st.os.EXPECT().Has(addedKey) - st.os.EXPECT().Receive(addedKey) - st.expectSync(p2p.Peer("p1"), ps, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ps.Sync(context.Background(), nil, nil)) - close(st.getWaitCh(addedKey)) - return nil - })) - - handledKeys, err := st.wait(1) - require.NoError(t, err) - require.ElementsMatch(t, []rangesync.KeyBytes{addedKey}, handledKeys) - }) - - t.Run("single key synced multiple times", func(t *testing.T) { + t.Run("sync", func(t *testing.T) { t.Parallel() st := newSetSyncBaseTester(t, nil) - addedKey := rangesync.RandomKeyBytes(32) - st.expectCopy(addedKey, addedKey, addedKey) - require.NoError(t, st.ssb.WithPeerSyncer( - context.Background(), p2p.Peer("p1"), - func(ps multipeer.PeerSyncer) error { - require.Equal(t, p2p.Peer("p1"), ps.Peer()) - // added just once - st.os.EXPECT().Receive(addedKey) - for i := 0; i < 3; i++ { - st.os.EXPECT().Has(addedKey) - st.expectSync(p2p.Peer("p1"), ps, addedKey) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ps.Sync(context.Background(), nil, nil)) - } - close(st.getWaitCh(addedKey)) + os := st.expectCopy() + x := rangesync.RandomKeyBytes(32) + y := rangesync.RandomKeyBytes(32) + st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), os, x, y) + addedKeys := []rangesync.KeyBytes{rangesync.RandomKeyBytes(32)} + sr := rangesync.MakeSeqResult(addedKeys) + os.EXPECT().Received().Return(sr, 1) + st.handler.EXPECT().Commit(gomock.Any(), p2p.Peer("p1"), st.os, gomock.Any()). + DoAndReturn(func( + _ context.Context, + _ p2p.Peer, + _ rangesync.OrderedSet, + sr rangesync.SeqResult, + ) error { + items, err := sr.Collect() + require.NoError(t, err) + require.ElementsMatch(t, addedKeys, items) return nil - })) - - handledKeys, err := st.wait(1) - require.NoError(t, err) - require.ElementsMatch(t, []rangesync.KeyBytes{addedKey}, handledKeys) + }) + st.os.EXPECT().Advance() + st.ssb.Sync(context.Background(), p2p.Peer("p1"), x, y) }) - t.Run("multiple keys", func(t *testing.T) { + t.Run("count empty", func(t *testing.T) { t.Parallel() st := newSetSyncBaseTester(t, nil) - k1 := rangesync.RandomKeyBytes(32) - k2 := rangesync.RandomKeyBytes(32) - st.expectCopy(k1, k2) - require.NoError(t, st.ssb.WithPeerSyncer( - context.Background(), p2p.Peer("p1"), - func(ps multipeer.PeerSyncer) error { - require.Equal(t, p2p.Peer("p1"), ps.Peer()) - - st.os.EXPECT().Has(k1) - st.os.EXPECT().Has(k2) - st.os.EXPECT().Receive(k1) - st.os.EXPECT().Receive(k2) - st.expectSync(p2p.Peer("p1"), ps, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ps.Sync(context.Background(), nil, nil)) - close(st.getWaitCh(k1)) - close(st.getWaitCh(k2)) - return nil - })) - handledKeys, err := st.wait(2) + st.os.EXPECT().Empty().Return(true, nil) + count, err := st.ssb.Count() require.NoError(t, err) - require.ElementsMatch(t, []rangesync.KeyBytes{k1, k2}, handledKeys) + require.Zero(t, count) }) - t.Run("handler failure", func(t *testing.T) { + t.Run("count non-empty", func(t *testing.T) { t.Parallel() st := newSetSyncBaseTester(t, nil) - k1 := rangesync.RandomKeyBytes(32) - k2 := rangesync.RandomKeyBytes(32) - st.expectCopy(k1, k2) - require.NoError(t, st.ssb.WithPeerSyncer( - context.Background(), p2p.Peer("p1"), - func(ps multipeer.PeerSyncer) error { - require.Equal(t, p2p.Peer("p1"), ps.Peer()) - - st.os.EXPECT().Has(k1) - st.os.EXPECT().Has(k2) - // k1 is not propagated to syncBase due to the handler failure - st.os.EXPECT().Receive(k2) - st.expectSync(p2p.Peer("p1"), ps, k1, k2) - st.handler.EXPECT().Commit(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) - st.os.EXPECT().Advance() - require.NoError(t, ps.Sync(context.Background(), nil, nil)) - st.getWaitCh(k1) <- errors.New("fail") - close(st.getWaitCh(k2)) - return nil - })) - - handledKeys, err := st.wait(2) - require.ErrorContains(t, err, "some key handlers failed") - require.ElementsMatch(t, []rangesync.KeyBytes{k1, k2}, handledKeys) - }) - - t.Run("real item set", func(t *testing.T) { - t.Parallel() - hs := make([]rangesync.KeyBytes, 4) - for n := range hs { - hs[n] = rangesync.RandomKeyBytes(32) + st.os.EXPECT().Empty().Return(false, nil) + items := []rangesync.KeyBytes{ + rangesync.RandomKeyBytes(32), + rangesync.RandomKeyBytes(32), } - var os rangesync.DumbSet - os.AddUnchecked(hs[0]) - os.AddUnchecked(hs[1]) - st := newSetSyncBaseTester(t, &os) - require.NoError(t, st.ssb.WithPeerSyncer( - context.Background(), p2p.Peer("p1"), - func(ps multipeer.PeerSyncer) error { - ps.(rangesync.OrderedSet).Receive(hs[2]) - ps.(rangesync.OrderedSet).Add(hs[2]) - ps.(rangesync.OrderedSet).Receive(hs[3]) - ps.(rangesync.OrderedSet).Add(hs[3]) - // syncer's cloned set has new key immediately - has, err := ps.(rangesync.OrderedSet).Has(hs[2]) - require.NoError(t, err) - require.True(t, has) - has, err = ps.(rangesync.OrderedSet).Has(hs[3]) - require.NoError(t, err) - require.True(t, has) - return nil - })) - st.getWaitCh(hs[2]) <- errors.New("fail") - close(st.getWaitCh(hs[3])) - handledKeys, err := st.wait(2) - require.ErrorContains(t, err, "some key handlers failed") - require.ElementsMatch(t, hs[2:], handledKeys) - // only successfully handled keys propagate the syncBase - received, err := os.Received().Collect() + st.os.EXPECT().Items().Return(rangesync.MakeSeqResult(items)) + st.os.EXPECT().GetRangeInfo(items[0], items[0]).Return(rangesync.RangeInfo{Count: 2}, nil) + count, err := st.ssb.Count() require.NoError(t, err) - require.ElementsMatch(t, hs[3:], received) + require.Equal(t, 2, count) }) } diff --git a/sync2/multipeer/split_sync.go b/sync2/multipeer/split_sync.go index 9ea1bd437f..d530894ab9 100644 --- a/sync2/multipeer/split_sync.go +++ b/sync2/multipeer/split_sync.go @@ -3,7 +3,6 @@ package multipeer import ( "context" "errors" - "fmt" "slices" "time" @@ -17,8 +16,8 @@ import ( ) type syncResult struct { - ps PeerSyncer - err error + peer p2p.Peer + err error } // splitSync is a synchronization implementation that synchronizes the set against @@ -87,19 +86,14 @@ func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange s.numRunning++ doneCh := make(chan struct{}) s.eg.Go(func() error { - if err := s.syncBase.WithPeerSyncer(ctx, p, func(ps PeerSyncer) error { - err := ps.Sync(ctx, sr.X, sr.Y) - close(doneCh) - select { - case <-ctx.Done(): - return ctx.Err() - case s.resCh <- syncResult{ps: ps, err: err}: - return nil - } - }); err != nil { - return fmt.Errorf("sync peer %s: %w", p, err) + err := s.syncBase.Sync(ctx, p, sr.X, sr.Y) + close(doneCh) + select { + case <-ctx.Done(): + return ctx.Err() + case s.resCh <- syncResult{peer: p, err: err}: + return nil } - return nil }) gpTimer := s.clock.After(s.gracePeriod) s.eg.Go(func() error { @@ -122,18 +116,18 @@ func (s *splitSync) startPeerSync(ctx context.Context, p p2p.Peer, sr *syncRange } func (s *splitSync) handleSyncResult(r syncResult) error { - sr, found := s.syncMap[r.ps.Peer()] + sr, found := s.syncMap[r.peer] if !found { panic("BUG: error in split sync syncMap handling") } s.numRunning-- - delete(s.syncMap, r.ps.Peer()) + delete(s.syncMap, r.peer) sr.NumSyncers-- if r.err != nil { s.numPeers-- - s.failedPeers[r.ps.Peer()] = struct{}{} + s.failedPeers[r.peer] = struct{}{} s.logger.Debug("remove failed peer", - zap.Stringer("peer", r.ps.Peer()), + zap.Stringer("peer", r.peer), zap.Int("numPeers", s.numPeers), zap.Int("numRemaining", s.numRemaining), zap.Int("numRunning", s.numRunning), @@ -146,20 +140,21 @@ func (s *splitSync) handleSyncResult(r syncResult) error { // sync with no active syncs remaining s.sq.Update(sr, time.Time{}) } - } else { - sr.Done = true - s.syncPeers = append(s.syncPeers, r.ps.Peer()) - s.numRemaining-- - s.logger.Debug("peer synced successfully", - log.ZShortStringer("x", sr.X), - log.ZShortStringer("y", sr.Y), - zap.Stringer("peer", r.ps.Peer()), - zap.Int("numPeers", s.numPeers), - zap.Int("numRemaining", s.numRemaining), - zap.Int("numRunning", s.numRunning), - zap.Int("availPeers", len(s.syncPeers))) + return nil } + sr.Done = true + s.syncPeers = append(s.syncPeers, r.peer) + s.numRemaining-- + s.logger.Debug("peer synced successfully", + log.ZShortStringer("x", sr.X), + log.ZShortStringer("y", sr.Y), + zap.Stringer("peer", r.peer), + zap.Int("numPeers", s.numPeers), + zap.Int("numRemaining", s.numRemaining), + zap.Int("numRunning", s.numRunning), + zap.Int("availPeers", len(s.syncPeers))) + return nil } diff --git a/sync2/multipeer/split_sync_test.go b/sync2/multipeer/split_sync_test.go index 4dbfdb8980..6e1c3bc173 100644 --- a/sync2/multipeer/split_sync_test.go +++ b/sync2/multipeer/split_sync_test.go @@ -23,7 +23,6 @@ import ( type splitSyncTester struct { testing.TB - ctrl *gomock.Controller syncPeers []p2p.Peer clock clockwork.FakeClock mtx sync.Mutex @@ -58,7 +57,6 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { ctrl := gomock.NewController(t) tst := &splitSyncTester{ TB: t, - ctrl: ctrl, syncPeers: make([]p2p.Peer, 4), clock: clockwork.NewFakeClock(), fail: make(map[hexRange]bool), @@ -92,38 +90,27 @@ func newTestSplitSync(t testing.TB) *splitSyncTester { func (tst *splitSyncTester) expectPeerSync(p p2p.Peer) { tst.syncBase.EXPECT(). - WithPeerSyncer(gomock.Any(), p, gomock.Any()). - DoAndReturn(func( - _ context.Context, - peer p2p.Peer, - toCall func(multipeer.PeerSyncer) error, - ) error { - s := NewMockPeerSyncer(tst.ctrl) - s.EXPECT().Peer().Return(p).AnyTimes() - s.EXPECT(). - Sync(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, x, y rangesync.KeyBytes) error { - tst.mtx.Lock() - defer tst.mtx.Unlock() - require.NotNil(tst, x) - require.NotNil(tst, y) - k := hexRange{x.String(), y.String()} - tst.peerRanges[k] = append(tst.peerRanges[k], peer) - count, found := tst.expPeerRanges[k] - require.True(tst, found, "peer range not found: x %s y %s", x, y) - if tst.fail[k] { - tst.Logf("ERR: peer %s x %s y %s", - string(p), x.String(), y.String()) - tst.fail[k] = false - return errors.New("injected fault") - } else { - tst.Logf("OK: peer %s x %s y %s", - string(p), x.String(), y.String()) - tst.expPeerRanges[k] = count + 1 - } - return nil - }) - return toCall(s) + Sync(gomock.Any(), p, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error { + tst.mtx.Lock() + defer tst.mtx.Unlock() + require.NotNil(tst, x) + require.NotNil(tst, y) + k := hexRange{x.String(), y.String()} + tst.peerRanges[k] = append(tst.peerRanges[k], p) + count, found := tst.expPeerRanges[k] + require.True(tst, found, "peer range not found: x %s y %s", x, y) + if tst.fail[k] { + tst.Logf("ERR: peer %s x %s y %s", + string(p), x.String(), y.String()) + tst.fail[k] = false + return errors.New("injected fault") + } else { + tst.Logf("OK: peer %s x %s y %s", + string(p), x.String(), y.String()) + tst.expPeerRanges[k] = count + 1 + } + return nil }).AnyTimes() } @@ -169,21 +156,10 @@ func TestSplitSync_SlowPeers(t *testing.T) { for _, p := range tst.syncPeers[2:] { tst.syncBase.EXPECT(). - WithPeerSyncer(gomock.Any(), p, gomock.Any()). - DoAndReturn(func( - _ context.Context, - peer p2p.Peer, - toCall func(multipeer.PeerSyncer) error, - ) error { - s := NewMockPeerSyncer(tst.ctrl) - s.EXPECT().Peer().Return(p).AnyTimes() - s.EXPECT(). - Sync(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, x, y rangesync.KeyBytes) error { - <-ctx.Done() - return nil - }) - return toCall(s) + Sync(gomock.Any(), p, gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, p p2p.Peer, x, y rangesync.KeyBytes) error { + <-ctx.Done() + return nil }) } diff --git a/sync2/p2p.go b/sync2/p2p.go index e43155155b..0f173d85eb 100644 --- a/sync2/p2p.go +++ b/sync2/p2p.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "sync" "sync/atomic" "time" @@ -13,7 +12,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/spacemeshos/go-spacemesh/fetch/peers" - "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/sync2/multipeer" "github.com/spacemeshos/go-spacemesh/sync2/rangesync" ) @@ -81,22 +79,14 @@ func NewP2PHashSync( enableActiveSync: enableActiveSync, } ps := rangesync.NewPairwiseSetSyncer(logger, d, name, cfg.RangeSetReconcilerConfig) - s.syncBase = multipeer.NewSetSyncBase(logger, ps, s.os, handler) + s.syncBase = multipeer.NewSetSyncBase(ps, s.os, handler) s.reconciler = multipeer.NewMultiPeerReconciler( logger, cfg.MultiPeerReconcilerConfig, s.syncBase, peers, keyLen, cfg.MaxDepth) - d.Register(name, s.serve) + d.Register(name, s.syncBase.Serve) return s } -func (s *P2PHashSync) serve(ctx context.Context, peer p2p.Peer, stream io.ReadWriter) error { - // We derive a dedicated Syncer for the peer being served to pass all the received - // items through the handler before adding them to the main OrderedSet. - return s.syncBase.WithPeerSyncer(ctx, peer, func(syncer multipeer.PeerSyncer) error { - return syncer.Serve(ctx, stream) - }) -} - // Set returns the OrderedSet that is being synchronized. func (s *P2PHashSync) Set() rangesync.OrderedSet { return s.os diff --git a/sync2/p2p_test.go b/sync2/p2p_test.go index dd2d1ba7d2..61197aeb96 100644 --- a/sync2/p2p_test.go +++ b/sync2/p2p_test.go @@ -21,40 +21,22 @@ import ( "github.com/spacemeshos/go-spacemesh/sync2/rangesync" ) -type addedKey struct { - // The fields are actually used to make sure each key is synced just once between - // each pair of peers. - //nolint:unused - fromPeer, toPeer p2p.Peer - //nolint:unused - key string -} - type fakeHandler struct { - mtx *sync.Mutex - localPeerID p2p.Peer - synced map[addedKey]struct{} - committed map[string]struct{} -} - -func (fh *fakeHandler) Receive(k rangesync.KeyBytes, peer p2p.Peer) (bool, error) { - fh.mtx.Lock() - defer fh.mtx.Unlock() - ak := addedKey{ - toPeer: fh.localPeerID, - key: string(k), - } - fh.synced[ak] = struct{}{} - return true, nil + mtx *sync.Mutex + committed map[string]struct{} } -func (fh *fakeHandler) Commit(ctx context.Context, peer p2p.Peer, base, new rangesync.OrderedSet) error { +func (fh *fakeHandler) Commit( + ctx context.Context, + peer p2p.Peer, + base rangesync.OrderedSet, + received rangesync.SeqResult, +) error { fh.mtx.Lock() defer fh.mtx.Unlock() - for k := range fh.synced { - fh.committed[k.key] = struct{}{} + for k := range received.Seq { + fh.committed[string(k)] = struct{}{} } - clear(fh.synced) return nil } @@ -100,10 +82,8 @@ func TestP2P(t *testing.T) { cfg.MaxDepth = maxDepth host := mesh.Hosts()[n] handlers[n] = &fakeHandler{ - mtx: &mtx, - localPeerID: host.ID(), - synced: make(map[addedKey]struct{}), - committed: make(map[string]struct{}), + mtx: &mtx, + committed: make(map[string]struct{}), } var os rangesync.DumbSet d := rangesync.NewDispatcher(logger) diff --git a/sync2/rangesync/dumbset.go b/sync2/rangesync/dumbset.go index 5c4abfc782..3d67d1aac8 100644 --- a/sync2/rangesync/dumbset.go +++ b/sync2/rangesync/dumbset.go @@ -141,7 +141,7 @@ func (ds *DumbSet) AddUnchecked(id KeyBytes) { // AddReceived adds all the received items to the set. func (ds *DumbSet) AddReceived() { - sr := ds.Received() + sr, _ := ds.Received() for k := range sr.Seq { ds.AddUnchecked(KeyBytes(k)) } @@ -187,7 +187,7 @@ func (ds *DumbSet) Receive(id KeyBytes) error { } // Received implements the OrderedSet. -func (ds *DumbSet) Received() SeqResult { +func (ds *DumbSet) Received() (SeqResult, int) { return SeqResult{ Seq: func(yield func(KeyBytes) bool) { for k := range ds.received { @@ -197,7 +197,7 @@ func (ds *DumbSet) Received() SeqResult { } }, Error: NoSeqError, - } + }, len(ds.received) } // seq returns an endless sequence as a SeqResult starting from the given index. diff --git a/sync2/rangesync/interface.go b/sync2/rangesync/interface.go index 517efc18ad..9a941ee352 100644 --- a/sync2/rangesync/interface.go +++ b/sync2/rangesync/interface.go @@ -31,6 +31,10 @@ type SplitInfo struct { // OrderedSet represents the set that can be synced against a remote peer. // OrderedSet methods are non-threadsafe except for WithCopy, Loaded and EnsureLoaded. +// SeqResult values obtained by method calls on an OrderedSet passed to WithCopy +// callback are valid only within the callback and should not be used outside of it, +// with exception of SeqResult returned by Received, which is expected to be valid +// outside of the callback as well. type OrderedSet interface { // Add adds a new key to the set. // It should not perform any additional actions related to handling @@ -39,8 +43,12 @@ type OrderedSet interface { // Receive handles a new key received from the peer. // It should not add the key to the set. Receive(k KeyBytes) error - // Received returns the sequence containing all the items received from the peer. - Received() SeqResult + // Received returns the sequence containing all the items received from the peer, + // and the total number of received items. + // Unlike other methods, SeqResult returned by Received called on a copy of the + // OrderedSet passed to WithCopy callback is expected to be valid outside of the + // callback as well. + Received() (SeqResult, int) // GetRangeInfo returns RangeInfo for the item range in the ordered set, // bounded by [x, y). // x == y indicates the whole set. diff --git a/sync2/rangesync/mocks/mocks.go b/sync2/rangesync/mocks/mocks.go index 304aa1d9a4..7201c59d66 100644 --- a/sync2/rangesync/mocks/mocks.go +++ b/sync2/rangesync/mocks/mocks.go @@ -388,11 +388,12 @@ func (c *MockOrderedSetReceiveCall) DoAndReturn(f func(rangesync.KeyBytes) error } // Received mocks base method. -func (m *MockOrderedSet) Received() rangesync.SeqResult { +func (m *MockOrderedSet) Received() (rangesync.SeqResult, int) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Received") ret0, _ := ret[0].(rangesync.SeqResult) - return ret0 + ret1, _ := ret[1].(int) + return ret0, ret1 } // Received indicates an expected call of Received. @@ -408,19 +409,19 @@ type MockOrderedSetReceivedCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetReceivedCall) Return(arg0 rangesync.SeqResult) *MockOrderedSetReceivedCall { - c.Call = c.Call.Return(arg0) +func (c *MockOrderedSetReceivedCall) Return(arg0 rangesync.SeqResult, arg1 int) *MockOrderedSetReceivedCall { + c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetReceivedCall) Do(f func() rangesync.SeqResult) *MockOrderedSetReceivedCall { +func (c *MockOrderedSetReceivedCall) Do(f func() (rangesync.SeqResult, int)) *MockOrderedSetReceivedCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetReceivedCall) DoAndReturn(f func() rangesync.SeqResult) *MockOrderedSetReceivedCall { +func (c *MockOrderedSetReceivedCall) DoAndReturn(f func() (rangesync.SeqResult, int)) *MockOrderedSetReceivedCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/sync2/rangesync/seq.go b/sync2/rangesync/seq.go index f1695114cc..86be62772b 100644 --- a/sync2/rangesync/seq.go +++ b/sync2/rangesync/seq.go @@ -129,7 +129,7 @@ func (s SeqResult) Collect() ([]KeyBytes, error) { func EmptySeqResult() SeqResult { return SeqResult{ Seq: EmptySeq(), - Error: func() error { return nil }, + Error: NoSeqError, } } @@ -140,3 +140,20 @@ func ErrorSeqResult(err error) SeqResult { Error: SeqError(err), } } + +// MakeSeqResult makes a SeqResult out of a slice. +// The sequence is made cyclic, starting over after the last element. +func MakeSeqResult(items []KeyBytes) SeqResult { + return SeqResult{ + Seq: func(yield func(k KeyBytes) bool) { + for { + for _, item := range items { + if !yield(item) { + return + } + } + } + }, + Error: NoSeqError, + } +} From 2987bbce0a57c669236196bfba6b3a43844f2d8d Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 6 Dec 2024 23:04:47 +0400 Subject: [PATCH 15/22] sync2: address comments --- sync2/atxs.go | 33 +++++++++++++++-------------- sync2/dbset/dbset.go | 7 +++--- sync2/dbset/dbset_test.go | 28 ++++++------------------ sync2/dbset/p2p_test.go | 4 +--- sync2/multipeer/setsyncbase.go | 9 +++++--- sync2/multipeer/setsyncbase_test.go | 2 +- sync2/rangesync/dumbset.go | 6 +++--- sync2/rangesync/interface.go | 5 ++--- sync2/rangesync/mocks/mocks.go | 13 ++++++------ sync2/rangesync/seq.go | 9 ++++++++ sync2/rangesync/seq_test.go | 21 ++++++++++++++++++ sync2/sqlstore/dbseq.go | 1 + 12 files changed, 77 insertions(+), 61 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 4bb2a57abb..69e63d5613 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -99,13 +99,7 @@ func (h *ATXHandler) Commit( batchAttemptsRemaining := h.maxBatchRetries for len(state) > 0 { items = items[:0] - for id, n := range state { - if n >= h.maxAttempts { - h.logger.Debug("failed to download ATX: max attempts reached", - zap.String("atx", id.ShortString())) - delete(state, id) - continue - } + for id, _ := range state { items = append(items, id) if len(items) == h.batchSize { break @@ -126,11 +120,17 @@ func (h *ATXHandler) Commit( someSucceeded = true delete(state, id) case errors.Is(err, pubsub.ErrValidationReject): - // if the atx invalid there's no point downloading it again - state[id] = h.maxAttempts + h.logger.Debug("failed to download ATX", + zap.String("atx", id.ShortString()), zap.Error(err)) + delete(state, id) + case state[id] >= h.maxAttempts-1: + h.logger.Debug("failed to download ATX: max attempts reached", + zap.String("atx", id.ShortString())) + delete(state, id) default: state[id]++ } + })) if err != nil { if errors.Is(err, context.Canceled) { @@ -152,15 +152,16 @@ func (h *ATXHandler) Commit( case <-ctx.Done(): return ctx.Err() case <-h.clock.After(h.failedBatchDelay): + continue } - } else { - batchAttemptsRemaining = h.maxBatchRetries - elapsed := h.clock.Since(startTime) - h.logger.Debug("fetched atxs", - zap.Int("total", total), - zap.Int("downloaded", numDownloaded), - zap.Float64("rate per sec", float64(numDownloaded)/elapsed.Seconds())) } + + batchAttemptsRemaining = h.maxBatchRetries + elapsed := h.clock.Since(startTime) + h.logger.Debug("fetched atxs", + zap.Int("total", total), + zap.Int("downloaded", numDownloaded), + zap.Float64("rate per sec", float64(numDownloaded)/elapsed.Seconds())) } return nil } diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index ed20293827..795ca24f1f 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -80,10 +80,9 @@ func (d *DBSet) EnsureLoaded() error { return d.snapshot.Load(d.db, d.handleIDfromDB) } -// Received returns a sequence of all items that have been received and the number of -// these items. +// Received returns a sequence of all items that have been received. // Implements rangesync.OrderedSet. -func (d *DBSet) Received() (rangesync.SeqResult, int) { +func (d *DBSet) Received() rangesync.SeqResult { return rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { for k := range d.received { @@ -93,7 +92,7 @@ func (d *DBSet) Received() (rangesync.SeqResult, int) { } }, Error: rangesync.NoSeqError, - }, len(d.received) + } } // Add adds an item to the DBSet. diff --git a/sync2/dbset/dbset_test.go b/sync2/dbset/dbset_test.go index 1e0600a629..76c117560b 100644 --- a/sync2/dbset/dbset_test.go +++ b/sync2/dbset/dbset_test.go @@ -42,9 +42,7 @@ func TestDBSet_Empty(t *testing.T) { require.NoError(t, err) require.True(t, empty) requireEmpty(t, s.Items()) - sr, n := s.Received() - requireEmpty(t, sr) - require.Zero(t, n) + requireEmpty(t, s.Received()) info, err := s.GetRangeInfo(nil, nil) require.NoError(t, err) @@ -192,12 +190,10 @@ func TestDBSet_Receive(t *testing.T) { newID := rangesync.MustParseHexKeyBytes("abcdef1234567890000000000000000000000000000000000000000000000000") require.NoError(t, s.Receive(newID)) - recvd, n := s.Received() - items, err := recvd.FirstN(1) + items, err := s.Received().FirstN(1) require.NoError(t, err) require.NoError(t, err) require.Equal(t, []rangesync.KeyBytes{newID}, items) - require.Equal(t, 1, n) info, err := s.GetRangeInfo(ids[2], ids[0]) require.NoError(t, err) @@ -238,9 +234,7 @@ func TestDBSet_Copy(t *testing.T) { require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) require.Equal(t, ids[2], firstKey(t, info.Items)) - sr, n := s.Received() - requireEmpty(t, sr) - require.Zero(t, n) + requireEmpty(t, s.Received()) info, err = s.GetRangeInfo(ids[2], ids[0]) require.NoError(t, err) @@ -248,9 +242,7 @@ func TestDBSet_Copy(t *testing.T) { require.Equal(t, "dddddddddddddddddddddddd", info.Fingerprint.String()) require.Equal(t, ids[2], firstKey(t, info.Items)) - sr, n = copy.(*dbset.DBSet).Received() - require.Equal(t, 1, n) - items, err := sr.FirstN(100) + items, err := copy.(*dbset.DBSet).Received().FirstN(100) require.NoError(t, err) require.Equal(t, []rangesync.KeyBytes{newID}, items) @@ -361,9 +353,7 @@ func TestDBSet_Added(t *testing.T) { IDColumn: "id", } s := dbset.NewDBSet(db, st, testKeyLen, testDepth) - sr, n := s.Received() - requireEmpty(t, sr) - require.Zero(t, n) + requireEmpty(t, s.Received()) recv := []rangesync.KeyBytes{ rangesync.MustParseHexKeyBytes("3333333333333333333333333333333333333333333333333333333333333333"), @@ -375,9 +365,7 @@ func TestDBSet_Added(t *testing.T) { require.NoError(t, s.EnsureLoaded()) - sr, n = s.Received() - require.Equal(t, 2, n) - recvd, err := sr.FirstN(3) + recvd, err := s.Received().FirstN(3) require.NoError(t, err) require.ElementsMatch(t, []rangesync.KeyBytes{ rangesync.MustParseHexKeyBytes("3333333333333333333333333333333333333333333333333333333333333333"), @@ -385,9 +373,7 @@ func TestDBSet_Added(t *testing.T) { }, recvd) require.NoError(t, s.WithCopy(context.Background(), func(copy rangesync.OrderedSet) error { - sr, n := copy.(*dbset.DBSet).Received() - require.Equal(t, 2, n) - recvd1, err := sr.FirstN(3) + recvd1, err := copy.(*dbset.DBSet).Received().FirstN(3) require.NoError(t, err) require.ElementsMatch(t, recvd, recvd1) return nil diff --git a/sync2/dbset/p2p_test.go b/sync2/dbset/p2p_test.go index 457b538352..c636c461ba 100644 --- a/sync2/dbset/p2p_test.go +++ b/sync2/dbset/p2p_test.go @@ -81,9 +81,8 @@ func (tr *syncTracer) OnRecent(receivedItems, sentItems int) { } func addReceived(t testing.TB, db sql.Executor, to, from *dbset.DBSet) { - sr, n := from.Received() + sr := from.Received() for k := range sr.Seq { - n-- has, err := to.Has(k) require.NoError(t, err) if !has { @@ -92,7 +91,6 @@ func addReceived(t testing.TB, db sql.Executor, to, from *dbset.DBSet) { } require.NoError(t, sr.Error()) require.NoError(t, to.Advance()) - require.Zero(t, n) } type startStopTimer interface { diff --git a/sync2/multipeer/setsyncbase.go b/sync2/multipeer/setsyncbase.go index d0b44225d0..d679be4595 100644 --- a/sync2/multipeer/setsyncbase.go +++ b/sync2/multipeer/setsyncbase.go @@ -63,17 +63,20 @@ func (ssb *SetSyncBase) syncPeer( toCall func(rangesync.OrderedSet) error, ) error { sr := rangesync.EmptySeqResult() - var n int if err := ssb.os.WithCopy(ctx, func(os rangesync.OrderedSet) error { if err := toCall(os); err != nil { return err } - sr, n = os.Received() + sr = os.Received() return nil }); err != nil { return fmt.Errorf("sync: %w", err) } - if n > 0 { + empty, err := sr.IsEmpty() + if err != nil { + return fmt.Errorf("check if the sequence result is empty: %w", err) + } + if !empty { if err := ssb.handler.Commit(ctx, p, ssb.os, sr); err != nil { return fmt.Errorf("commit: %w", err) } diff --git a/sync2/multipeer/setsyncbase_test.go b/sync2/multipeer/setsyncbase_test.go index 4a2ae3d365..d09c70ec09 100644 --- a/sync2/multipeer/setsyncbase_test.go +++ b/sync2/multipeer/setsyncbase_test.go @@ -73,7 +73,7 @@ func TestSetSyncBase(t *testing.T) { st.ps.EXPECT().Sync(gomock.Any(), p2p.Peer("p1"), os, x, y) addedKeys := []rangesync.KeyBytes{rangesync.RandomKeyBytes(32)} sr := rangesync.MakeSeqResult(addedKeys) - os.EXPECT().Received().Return(sr, 1) + os.EXPECT().Received().Return(sr) st.handler.EXPECT().Commit(gomock.Any(), p2p.Peer("p1"), st.os, gomock.Any()). DoAndReturn(func( _ context.Context, diff --git a/sync2/rangesync/dumbset.go b/sync2/rangesync/dumbset.go index 3d67d1aac8..5c4abfc782 100644 --- a/sync2/rangesync/dumbset.go +++ b/sync2/rangesync/dumbset.go @@ -141,7 +141,7 @@ func (ds *DumbSet) AddUnchecked(id KeyBytes) { // AddReceived adds all the received items to the set. func (ds *DumbSet) AddReceived() { - sr, _ := ds.Received() + sr := ds.Received() for k := range sr.Seq { ds.AddUnchecked(KeyBytes(k)) } @@ -187,7 +187,7 @@ func (ds *DumbSet) Receive(id KeyBytes) error { } // Received implements the OrderedSet. -func (ds *DumbSet) Received() (SeqResult, int) { +func (ds *DumbSet) Received() SeqResult { return SeqResult{ Seq: func(yield func(KeyBytes) bool) { for k := range ds.received { @@ -197,7 +197,7 @@ func (ds *DumbSet) Received() (SeqResult, int) { } }, Error: NoSeqError, - }, len(ds.received) + } } // seq returns an endless sequence as a SeqResult starting from the given index. diff --git a/sync2/rangesync/interface.go b/sync2/rangesync/interface.go index 9a941ee352..1f6f64c779 100644 --- a/sync2/rangesync/interface.go +++ b/sync2/rangesync/interface.go @@ -43,12 +43,11 @@ type OrderedSet interface { // Receive handles a new key received from the peer. // It should not add the key to the set. Receive(k KeyBytes) error - // Received returns the sequence containing all the items received from the peer, - // and the total number of received items. + // Received returns the sequence containing all the items received from the peer. // Unlike other methods, SeqResult returned by Received called on a copy of the // OrderedSet passed to WithCopy callback is expected to be valid outside of the // callback as well. - Received() (SeqResult, int) + Received() SeqResult // GetRangeInfo returns RangeInfo for the item range in the ordered set, // bounded by [x, y). // x == y indicates the whole set. diff --git a/sync2/rangesync/mocks/mocks.go b/sync2/rangesync/mocks/mocks.go index 7201c59d66..304aa1d9a4 100644 --- a/sync2/rangesync/mocks/mocks.go +++ b/sync2/rangesync/mocks/mocks.go @@ -388,12 +388,11 @@ func (c *MockOrderedSetReceiveCall) DoAndReturn(f func(rangesync.KeyBytes) error } // Received mocks base method. -func (m *MockOrderedSet) Received() (rangesync.SeqResult, int) { +func (m *MockOrderedSet) Received() rangesync.SeqResult { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Received") ret0, _ := ret[0].(rangesync.SeqResult) - ret1, _ := ret[1].(int) - return ret0, ret1 + return ret0 } // Received indicates an expected call of Received. @@ -409,19 +408,19 @@ type MockOrderedSetReceivedCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockOrderedSetReceivedCall) Return(arg0 rangesync.SeqResult, arg1 int) *MockOrderedSetReceivedCall { - c.Call = c.Call.Return(arg0, arg1) +func (c *MockOrderedSetReceivedCall) Return(arg0 rangesync.SeqResult) *MockOrderedSetReceivedCall { + c.Call = c.Call.Return(arg0) return c } // Do rewrite *gomock.Call.Do -func (c *MockOrderedSetReceivedCall) Do(f func() (rangesync.SeqResult, int)) *MockOrderedSetReceivedCall { +func (c *MockOrderedSetReceivedCall) Do(f func() rangesync.SeqResult) *MockOrderedSetReceivedCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockOrderedSetReceivedCall) DoAndReturn(f func() (rangesync.SeqResult, int)) *MockOrderedSetReceivedCall { +func (c *MockOrderedSetReceivedCall) DoAndReturn(f func() rangesync.SeqResult) *MockOrderedSetReceivedCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/sync2/rangesync/seq.go b/sync2/rangesync/seq.go index 86be62772b..082d3ad081 100644 --- a/sync2/rangesync/seq.go +++ b/sync2/rangesync/seq.go @@ -125,6 +125,15 @@ func (s SeqResult) Collect() ([]KeyBytes, error) { return s.Seq.Collect(), s.Error() } +// IsEmpty returns true if the sequence in SeqResult is empty. +// It also checks for errors. +func (s SeqResult) IsEmpty() (bool, error) { + for range s.Seq { + return false, s.Error() + } + return true, s.Error() +} + // EmptySeqResult returns an empty sequence result. func EmptySeqResult() SeqResult { return SeqResult{ diff --git a/sync2/rangesync/seq_test.go b/sync2/rangesync/seq_test.go index 78feb232d4..e7e4c41191 100644 --- a/sync2/rangesync/seq_test.go +++ b/sync2/rangesync/seq_test.go @@ -1,6 +1,7 @@ package rangesync_test import ( + "errors" "slices" "testing" @@ -23,3 +24,23 @@ func TestGetN(t *testing.T) { require.Equal(t, []rangesync.KeyBytes{{1}, {2}, {3}, {4}}, seq.FirstN(4)) require.Equal(t, []rangesync.KeyBytes{{1}, {2}, {3}, {4}}, seq.FirstN(5)) } + +func TestIsEmpty(t *testing.T) { + empty, err := rangesync.EmptySeqResult().IsEmpty() + require.NoError(t, err) + require.True(t, empty) + sr := rangesync.MakeSeqResult([]rangesync.KeyBytes{{1}}) + empty, err = sr.IsEmpty() + require.NoError(t, err) + require.False(t, empty) + sampleErr := errors.New("error") + sr = rangesync.ErrorSeqResult(sampleErr) + _, err = sr.IsEmpty() + require.ErrorIs(t, err, sampleErr) + sr = rangesync.SeqResult{ + Seq: rangesync.Seq(slices.Values([]rangesync.KeyBytes{{1}})), + Error: rangesync.SeqError(sampleErr), + } + _, err = sr.IsEmpty() + require.ErrorIs(t, err, sampleErr) +} diff --git a/sync2/sqlstore/dbseq.go b/sync2/sqlstore/dbseq.go index bdcadddca0..a00abeabf5 100644 --- a/sync2/sqlstore/dbseq.go +++ b/sync2/sqlstore/dbseq.go @@ -68,6 +68,7 @@ func idsFromTable( chunk: make([]rangesync.KeyBytes, 1), singleChunk: false, } + // QQQQQ: do not load eagerly if err = s.load(); err != nil { return } From 7dab83a9b406e1b0887386ebd6cb4a6502ffab6f Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 6 Dec 2024 23:29:35 +0400 Subject: [PATCH 16/22] sync2: make most Seqs/SeqResults non-cyclic --- sync2/atxs.go | 6 ----- sync2/atxs_test.go | 12 +++------ sync2/dbset/dbset.go | 5 ++-- sync2/fptree/fptree.go | 10 +++++--- sync2/fptree/fptree_test.go | 16 ++++++------ sync2/rangesync/dumbset.go | 13 ++-------- sync2/rangesync/seq.go | 49 ++++++++++++++++++++---------------- sync2/rangesync/seq_test.go | 35 ++++++++++++++++++++++++++ sync2/sqlstore/dbseq.go | 1 - sync2/sqlstore/dbseq_test.go | 2 ++ sync2/sqlstore/interface.go | 2 ++ sync2/sqlstore/sqlidstore.go | 3 +++ 12 files changed, 92 insertions(+), 62 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 69e63d5613..525e2674b4 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -70,15 +70,9 @@ func (h *ATXHandler) Commit( ) error { h.logger.Debug("begin atx commit") defer h.logger.Debug("end atx commit") - var firstK rangesync.KeyBytes numDownloaded := 0 state := make(map[types.ATXID]int) for k := range received.Seq { - if firstK == nil { - firstK = k - } else if firstK.Compare(k) == 0 { - break - } found, err := base.Has(k) if err != nil { return fmt.Errorf("check if ATX exists: %w", err) diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go index 46db742898..5708f571ea 100644 --- a/sync2/atxs_test.go +++ b/sync2/atxs_test.go @@ -26,8 +26,6 @@ import ( func atxSeqResult(atxs []types.ATXID) rangesync.SeqResult { return rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { - // Received sequence may be cyclic and the handler should stop - // when it sees the first key again. for _, atx := range atxs { if !yield(atx.Bytes()) { return @@ -281,13 +279,9 @@ func TestAtxHandler_BatchRetry_Fail(t *testing.T) { } sr := rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { - // Received sequence may be cyclic and the handler should stop - // when it sees the first key again. - for { - for _, atx := range allAtxs { - if !yield(atx.Bytes()) { - return - } + for _, atx := range allAtxs { + if !yield(atx.Bytes()) { + return } } }, diff --git a/sync2/dbset/dbset.go b/sync2/dbset/dbset.go index 795ca24f1f..9f90554bda 100644 --- a/sync2/dbset/dbset.go +++ b/sync2/dbset/dbset.go @@ -198,7 +198,7 @@ func (d *DBSet) Items() rangesync.SeqResult { if err := d.EnsureLoaded(); err != nil { return rangesync.ErrorSeqResult(err) } - return d.ft.All() + return d.ft.All().Limit(d.ft.Count()) } // Empty returns true if the DBSet is empty. @@ -281,7 +281,8 @@ func (d *DBSet) Has(k rangesync.KeyBytes) (bool, error) { // Recent returns a sequence of items that have been added to the DBSet since the given time. // Implements rangesync.OrderedSet. func (d *DBSet) Recent(since time.Time) (rangesync.SeqResult, int) { - return d.dbStore.Since(make(rangesync.KeyBytes, d.keyLen), since.UnixNano()) + sr, n := d.dbStore.Since(make(rangesync.KeyBytes, d.keyLen), since.UnixNano()) + return sr.Limit(n), n } func (d *DBSet) release() { diff --git a/sync2/fptree/fptree.go b/sync2/fptree/fptree.go index 56f620c886..4f10d9b797 100644 --- a/sync2/fptree/fptree.go +++ b/sync2/fptree/fptree.go @@ -363,6 +363,7 @@ func (ft *FPTree) traverseFrom( } // All returns all the items currently in the tree (including those in the IDStore). +// The sequence in SeqResult is either empty or infinite. // Implements sqlstore.All. func (ft *FPTree) All() rangesync.SeqResult { ft.np.lockRead() @@ -386,6 +387,7 @@ func (ft *FPTree) All() rangesync.SeqResult { } // From returns all the items in the tree that are greater than or equal to the given key. +// The sequence in SeqResult is either empty or infinite. // Implements sqlstore.IDStore. func (ft *FPTree) From(from rangesync.KeyBytes, sizeHint int) rangesync.SeqResult { ft.np.lockRead() @@ -1108,9 +1110,9 @@ func (ft *FPTree) fingerprintInterval(x, y rangesync.KeyBytes, limit int) (fpr F if ac.items.Seq != nil { ft.log("fingerprintInterval: items %v", ac.items) - fpr.Items = ac.items + fpr.Items = ac.items.Limit(int(ac.count)) } else { - fpr.Items = ft.from(x, 1) + fpr.Items = ft.from(x, 1).Limit(int(ac.count)) ft.log("fingerprintInterval: start from x: %v", fpr.Items) } @@ -1187,7 +1189,7 @@ func (ft *FPTree) easySplit(x, y rangesync.KeyBytes, limit int) (sr SplitResult, FP: ac.fp0, Count: ac.count0, IType: ac.itype, - Items: items, + Items: items.Limit(int(ac.count0)), // Next is only used during splitting itself, and thus not included } items = ft.startFromPrefix(&ac, *ac.lastPrefix0) @@ -1195,7 +1197,7 @@ func (ft *FPTree) easySplit(x, y rangesync.KeyBytes, limit int) (sr SplitResult, FP: ac.fp, Count: ac.count, IType: ac.itype, - Items: items, + Items: items.Limit(int(ac.count0)), // Next is only used during splitting itself, and thus not included } return SplitResult{ diff --git a/sync2/fptree/fptree_test.go b/sync2/fptree/fptree_test.go index 87e3a9f3e0..a78d9cb50f 100644 --- a/sync2/fptree/fptree_test.go +++ b/sync2/fptree/fptree_test.go @@ -123,8 +123,8 @@ func testFPTree(t *testing.T, makeFPTrees mkFPTreesFunc) { fp: "000000000000000000000000", count: 0, itype: 0, - startIdx: 0, - endIdx: 0, + startIdx: -1, + endIdx: -1, }, { xIdx: 0, @@ -193,8 +193,8 @@ func testFPTree(t *testing.T, makeFPTrees mkFPTreesFunc) { fp: "000000000000000000000000", count: 0, itype: -1, - startIdx: 0, - endIdx: 0, + startIdx: -1, + endIdx: -1, }, { xIdx: 1, @@ -233,8 +233,8 @@ func testFPTree(t *testing.T, makeFPTrees mkFPTreesFunc) { fp: "000000000000000000000000", count: 0, itype: 1, - startIdx: 2, - endIdx: 2, + startIdx: -1, + endIdx: -1, }, { xIdx: 3, @@ -273,8 +273,8 @@ func testFPTree(t *testing.T, makeFPTrees mkFPTreesFunc) { fp: "000000000000000000000000", count: 0, itype: -1, - startIdx: 0, - endIdx: 0, + startIdx: -1, + endIdx: -1, }, }, }, diff --git a/sync2/rangesync/dumbset.go b/sync2/rangesync/dumbset.go index 5c4abfc782..f18acfd672 100644 --- a/sync2/rangesync/dumbset.go +++ b/sync2/rangesync/dumbset.go @@ -259,7 +259,7 @@ func (ds *DumbSet) getRangeInfo( if start == nil || end == nil { panic("empty start/end from naiveRange") } - r.Items = ds.seqFor(start) + r.Items = ds.seqFor(start).Limit(r.Count) } else { r.Items = EmptySeqResult() } @@ -301,10 +301,7 @@ func (ds *DumbSet) Empty() (bool, error) { // Items implements OrderedSet. func (ds *DumbSet) Items() SeqResult { - if len(ds.keys) == 0 { - return EmptySeqResult() - } - return ds.seq(0) + return MakeSeqResult(ds.keys) } // WithCopy implements OrderedSet. @@ -334,14 +331,8 @@ func (ds *DumbSet) Advance() error { // Has implements OrderedSet. func (ds *DumbSet) Has(k KeyBytes) (bool, error) { - var first KeyBytes sr := ds.Items() for cur := range sr.Seq { - if first == nil { - first = cur - } else if first.Compare(cur) == 0 { - return false, sr.Error() - } if k.Compare(cur) == 0 { return true, sr.Error() } diff --git a/sync2/rangesync/seq.go b/sync2/rangesync/seq.go index 082d3ad081..cd1865c63b 100644 --- a/sync2/rangesync/seq.go +++ b/sync2/rangesync/seq.go @@ -2,13 +2,14 @@ package rangesync import ( "iter" + "slices" "go.uber.org/zap/zapcore" ) // Seq represents an ordered sequence of elements. -// Unless the sequence is empty or an error occurs while iterating, it yields elements -// endlessly, wrapping around to the first element after the last one. +// Most sequences are finite. Infinite sequences are explicitly mentioned in the +// documentation of functions/methods that return them. type Seq iter.Seq[KeyBytes] var _ zapcore.ArrayMarshaler = Seq(nil) @@ -38,19 +39,7 @@ func (s Seq) FirstN(n int) []KeyBytes { // It may not be very efficient due to reallocations, and thus it should only be used for // small sequences or for testing. func (s Seq) Collect() []KeyBytes { - var ( - first KeyBytes - r []KeyBytes - ) - for v := range s { - if first == nil { - first = v - } else if v.Compare(first) == 0 { - break - } - r = append(r, v) - } - return r + return slices.Collect(iter.Seq[KeyBytes](s)) } // MarshalLogArray implements zapcore.ArrayMarshaler. @@ -70,6 +59,19 @@ func (s Seq) MarshalLogArray(enc zapcore.ArrayEncoder) error { return nil } +// Limit limits sequence to n elements. +func (s Seq) Limit(n int) Seq { + return Seq(func(yield func(KeyBytes) bool) { + n := n // ensure reusability + for k := range s { + if n == 0 || !yield(k) { + return + } + n-- + } + }) +} + // EmptySeq returns an empty sequence. func EmptySeq() Seq { return Seq(func(yield func(KeyBytes) bool) {}) @@ -134,6 +136,14 @@ func (s SeqResult) IsEmpty() (bool, error) { return true, s.Error() } +// Limit limits SeqResult to n elements. +func (s SeqResult) Limit(n int) SeqResult { + return SeqResult{ + Seq: s.Seq.Limit(n), + Error: s.Error, + } +} + // EmptySeqResult returns an empty sequence result. func EmptySeqResult() SeqResult { return SeqResult{ @@ -151,15 +161,12 @@ func ErrorSeqResult(err error) SeqResult { } // MakeSeqResult makes a SeqResult out of a slice. -// The sequence is made cyclic, starting over after the last element. func MakeSeqResult(items []KeyBytes) SeqResult { return SeqResult{ Seq: func(yield func(k KeyBytes) bool) { - for { - for _, item := range items { - if !yield(item) { - return - } + for _, item := range items { + if !yield(item) { + return } } }, diff --git a/sync2/rangesync/seq_test.go b/sync2/rangesync/seq_test.go index e7e4c41191..ac67bdd83c 100644 --- a/sync2/rangesync/seq_test.go +++ b/sync2/rangesync/seq_test.go @@ -44,3 +44,38 @@ func TestIsEmpty(t *testing.T) { _, err = sr.IsEmpty() require.ErrorIs(t, err, sampleErr) } + +func TestSeqLimit(t *testing.T) { + seq := rangesync.Seq(func(yield func(rangesync.KeyBytes) bool) { + i := 0 + for yield(rangesync.KeyBytes{byte(i & 0xff)}) { + i++ + } + }) + require.Empty(t, seq.Limit(0).Collect()) + limited := seq.Limit(3) + for range 3 { + require.Equal(t, []rangesync.KeyBytes{{0}, {1}, {2}}, limited.Collect()) + } + + sr := rangesync.SeqResult{ + Seq: seq, + Error: rangesync.NoSeqError, + } + limitedSR := sr.Limit(3) + for range 3 { + items, err := limitedSR.Collect() + require.NoError(t, err) + require.Equal(t, []rangesync.KeyBytes{{0}, {1}, {2}}, items) + } + + sampleErr := errors.New("error") + sr = rangesync.SeqResult{ + Seq: seq, + Error: rangesync.SeqError(sampleErr), + } + for range 3 { + _, err := sr.Limit(3).Collect() + require.ErrorIs(t, err, sampleErr) + } +} diff --git a/sync2/sqlstore/dbseq.go b/sync2/sqlstore/dbseq.go index a00abeabf5..bdcadddca0 100644 --- a/sync2/sqlstore/dbseq.go +++ b/sync2/sqlstore/dbseq.go @@ -68,7 +68,6 @@ func idsFromTable( chunk: make([]rangesync.KeyBytes, 1), singleChunk: false, } - // QQQQQ: do not load eagerly if err = s.load(); err != nil { return } diff --git a/sync2/sqlstore/dbseq_test.go b/sync2/sqlstore/dbseq_test.go index 94a9b830b1..34a4805bf0 100644 --- a/sync2/sqlstore/dbseq_test.go +++ b/sync2/sqlstore/dbseq_test.go @@ -235,6 +235,8 @@ func TestDBRangeIterator(t *testing.T) { var collected []rangesync.KeyBytes var firstK rangesync.KeyBytes for k := range sr.Seq { + // sequences returned by IDSFromTable is either empty + // or cyclic if firstK == nil { firstK = k } else if k.Compare(firstK) == 0 { diff --git a/sync2/sqlstore/interface.go b/sync2/sqlstore/interface.go index ecb38e93e1..c6544fdc65 100644 --- a/sync2/sqlstore/interface.go +++ b/sync2/sqlstore/interface.go @@ -14,8 +14,10 @@ type IDStore interface { // RegisterKey registers the key with the store. RegisterKey(k rangesync.KeyBytes) error // All returns all keys in the store. + // The sequence in SeqResult returned by All is either empty or infinite. All() rangesync.SeqResult // From returns all keys in the store starting from the given key. // sizeHint is a hint for the expected number of keys to be returned. + // The sequence in SeqResult returned by From is either empty or infinite. From(from rangesync.KeyBytes, sizeHint int) rangesync.SeqResult } diff --git a/sync2/sqlstore/sqlidstore.go b/sync2/sqlstore/sqlidstore.go index e26fa37ad7..df20dfbf1e 100644 --- a/sync2/sqlstore/sqlidstore.go +++ b/sync2/sqlstore/sqlidstore.go @@ -41,12 +41,14 @@ func (s *SQLIDStore) RegisterKey(k rangesync.KeyBytes) error { } // All returns all IDs in the store. +// The sequence in SeqResult returned by All is either empty or infinite. // Implements IDStore. func (s *SQLIDStore) All() rangesync.SeqResult { return s.From(make(rangesync.KeyBytes, s.keyLen), 1) } // From returns IDs in the store starting from the given key. +// The sequence in SeqResult returned by From is either empty or infinite. // Implements IDStore. func (s *SQLIDStore) From(from rangesync.KeyBytes, sizeHint int) rangesync.SeqResult { if len(from) != s.keyLen { @@ -56,6 +58,7 @@ func (s *SQLIDStore) From(from rangesync.KeyBytes, sizeHint int) rangesync.SeqRe } // Since returns IDs in the store starting from the given key and timestamp. +// The sequence in SeqResult returned by Since is either empty or infinite. func (s *SQLIDStore) Since(from rangesync.KeyBytes, since int64) (rangesync.SeqResult, int) { if len(from) != s.keyLen { panic("BUG: invalid key length") From 858787a48e4e9be5eaf6fca45546ee418acee8cd Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 6 Dec 2024 23:54:23 +0400 Subject: [PATCH 17/22] sync2: address comments --- sync2/atxs.go | 39 ++++---- sync2/atxs_test.go | 7 +- syncer/interface.go | 3 +- syncer/mocks/mocks.go | 226 ++++++++++++++++++++++++++++++++++++++++++ syncer/syncer.go | 7 +- syncer/syncer_test.go | 1 - 6 files changed, 251 insertions(+), 32 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 525e2674b4..966ccb04b0 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -92,16 +92,16 @@ func (h *ATXHandler) Commit( startTime := h.clock.Now() batchAttemptsRemaining := h.maxBatchRetries for len(state) > 0 { + if len(state) == 0 { + break + } items = items[:0] - for id, _ := range state { + for id := range state { items = append(items, id) if len(items) == h.batchSize { break } } - if len(items) == 0 { - break - } someSucceeded := false var mtx sync.Mutex @@ -124,7 +124,6 @@ func (h *ATXHandler) Commit( default: state[id]++ } - })) if err != nil { if errors.Is(err, context.Canceled) { @@ -236,15 +235,16 @@ func (s *MultiEpochATXSyncer) EnsureSync( } for epoch := types.EpochID(1); epoch <= newEpoch; epoch++ { syncer := s.atxSyncers[epoch-1] - if epoch <= lastWaitEpoch { - s.logger.Info("waiting for epoch to sync", zap.Uint32("epoch", epoch.Uint32())) - if err := syncer.StartAndSync(ctx); err != nil { - return lastSynced, fmt.Errorf("error syncing old ATXs: %w", err) - } - lastSynced = epoch - } else { + if epoch > lastWaitEpoch { syncer.Start() + continue + } + + s.logger.Info("waiting for epoch to sync", zap.Uint32("epoch", epoch.Uint32())) + if err := syncer.StartAndSync(ctx); err != nil { + return lastSynced, fmt.Errorf("error syncing old ATXs: %w", err) } + lastSynced = epoch } return lastSynced, nil } @@ -275,18 +275,15 @@ func NewATXSyncer( d *rangesync.Dispatcher, name string, cfg Config, - db sql.StateDatabase, + db sql.Database, f Fetcher, epoch types.EpochID, enableActiveSync bool, ) *P2PHashSync { curSet := dbset.NewDBSet(db, atxsTable(epoch), 32, cfg.MaxDepth) - return NewP2PHashSync( - logger, d, name, curSet, 32, f.Peers(), - NewATXHandler( - logger, f, cfg.BatchSize, cfg.MaxAttempts, - cfg.MaxBatchRetries, cfg.FailedBatchDelay, nil), - cfg, enableActiveSync) + handler := NewATXHandler(logger, f, cfg.BatchSize, cfg.MaxAttempts, + cfg.MaxBatchRetries, cfg.FailedBatchDelay, nil) + return NewP2PHashSync(logger, d, name, curSet, 32, f.Peers(), handler, cfg, enableActiveSync) } func NewDispatcher(logger *zap.Logger, f Fetcher, opts []server.Opt) *rangesync.Dispatcher { @@ -298,7 +295,7 @@ func NewDispatcher(logger *zap.Logger, f Fetcher, opts []server.Opt) *rangesync. type ATXSyncSource struct { logger *zap.Logger d *rangesync.Dispatcher - db sql.StateDatabase + db sql.Database f Fetcher enableActiveSync bool } @@ -308,7 +305,7 @@ var _ HashSyncSource = &ATXSyncSource{} func NewATXSyncSource( logger *zap.Logger, d *rangesync.Dispatcher, - db sql.StateDatabase, + db sql.Database, f Fetcher, enableActiveSync bool, ) *ATXSyncSource { diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go index 5708f571ea..bf11dffb9c 100644 --- a/sync2/atxs_test.go +++ b/sync2/atxs_test.go @@ -145,6 +145,7 @@ func TestAtxHandler_Retry(t *testing.T) { // If it so happens that a full batch fails, we need to advance the clock to // trigger the retry. ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var eg errgroup.Group eg.Go(func() error { for { @@ -161,13 +162,11 @@ func TestAtxHandler_Retry(t *testing.T) { clock.Advance(batchRetryDelay) } }) - defer func() { - cancel() - eg.Wait() - }() require.NoError(t, h.Commit(context.Background(), peer, baseSet, atxSeqResult(allAtxs))) require.ElementsMatch(t, allAtxs[1:], fetched) + cancel() + require.NoError(t, eg.Wait()) } func TestAtxHandler_Cancel(t *testing.T) { diff --git a/syncer/interface.go b/syncer/interface.go index e1d9c5205c..2e99b8a4ac 100644 --- a/syncer/interface.go +++ b/syncer/interface.go @@ -7,6 +7,7 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/fetch" "github.com/spacemeshos/go-spacemesh/p2p" + "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/system" ) @@ -47,7 +48,7 @@ type fetcher interface { GetLayerOpinions(context.Context, p2p.Peer, types.LayerID) ([]byte, error) GetCert(context.Context, types.LayerID, types.BlockID, []p2p.Peer) (*types.Certificate, error) - system.AtxFetcher + sync2.Fetcher system.MalfeasanceProofFetcher GetBallots(context.Context, []types.BallotID) error GetBlocks(context.Context, []types.BlockID) error diff --git a/syncer/mocks/mocks.go b/syncer/mocks/mocks.go index 7980900806..052ead48ab 100644 --- a/syncer/mocks/mocks.go +++ b/syncer/mocks/mocks.go @@ -14,8 +14,10 @@ import ( reflect "reflect" time "time" + host "github.com/libp2p/go-libp2p/core/host" types "github.com/spacemeshos/go-spacemesh/common/types" fetch "github.com/spacemeshos/go-spacemesh/fetch" + peers "github.com/spacemeshos/go-spacemesh/fetch/peers" p2p "github.com/spacemeshos/go-spacemesh/p2p" system "github.com/spacemeshos/go-spacemesh/system" gomock "go.uber.org/mock/gomock" @@ -458,6 +460,44 @@ func (c *MockfetchLogicGetMaliciousIDsCall) DoAndReturn(f func(context.Context, return c } +// Host mocks base method. +func (m *MockfetchLogic) Host() host.Host { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Host") + ret0, _ := ret[0].(host.Host) + return ret0 +} + +// Host indicates an expected call of Host. +func (mr *MockfetchLogicMockRecorder) Host() *MockfetchLogicHostCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*MockfetchLogic)(nil).Host)) + return &MockfetchLogicHostCall{Call: call} +} + +// MockfetchLogicHostCall wrap *gomock.Call +type MockfetchLogicHostCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetchLogicHostCall) Return(arg0 host.Host) *MockfetchLogicHostCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetchLogicHostCall) Do(f func() host.Host) *MockfetchLogicHostCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetchLogicHostCall) DoAndReturn(f func() host.Host) *MockfetchLogicHostCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // PeerEpochInfo mocks base method. func (m *MockfetchLogic) PeerEpochInfo(arg0 context.Context, arg1 p2p.Peer, arg2 types.EpochID) (*fetch.EpochData, error) { m.ctrl.T.Helper() @@ -536,6 +576,44 @@ func (c *MockfetchLogicPeerMeshHashesCall) DoAndReturn(f func(context.Context, p return c } +// Peers mocks base method. +func (m *MockfetchLogic) Peers() *peers.Peers { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Peers") + ret0, _ := ret[0].(*peers.Peers) + return ret0 +} + +// Peers indicates an expected call of Peers. +func (mr *MockfetchLogicMockRecorder) Peers() *MockfetchLogicPeersCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*MockfetchLogic)(nil).Peers)) + return &MockfetchLogicPeersCall{Call: call} +} + +// MockfetchLogicPeersCall wrap *gomock.Call +type MockfetchLogicPeersCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetchLogicPeersCall) Return(arg0 *peers.Peers) *MockfetchLogicPeersCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetchLogicPeersCall) Do(f func() *peers.Peers) *MockfetchLogicPeersCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetchLogicPeersCall) DoAndReturn(f func() *peers.Peers) *MockfetchLogicPeersCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // PollLayerData mocks base method. func (m *MockfetchLogic) PollLayerData(arg0 context.Context, arg1 types.LayerID, arg2 ...p2p.Peer) error { m.ctrl.T.Helper() @@ -619,6 +697,42 @@ func (c *MockfetchLogicPollLayerOpinionsCall) DoAndReturn(f func(context.Context return c } +// RegisterPeerHash mocks base method. +func (m *MockfetchLogic) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RegisterPeerHash", peer, hash) +} + +// RegisterPeerHash indicates an expected call of RegisterPeerHash. +func (mr *MockfetchLogicMockRecorder) RegisterPeerHash(peer, hash any) *MockfetchLogicRegisterPeerHashCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*MockfetchLogic)(nil).RegisterPeerHash), peer, hash) + return &MockfetchLogicRegisterPeerHashCall{Call: call} +} + +// MockfetchLogicRegisterPeerHashCall wrap *gomock.Call +type MockfetchLogicRegisterPeerHashCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetchLogicRegisterPeerHashCall) Return() *MockfetchLogicRegisterPeerHashCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetchLogicRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockfetchLogicRegisterPeerHashCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetchLogicRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockfetchLogicRegisterPeerHashCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // RegisterPeerHashes mocks base method. func (m *MockfetchLogic) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) { m.ctrl.T.Helper() @@ -1192,6 +1306,44 @@ func (c *MockfetcherGetMaliciousIDsCall) DoAndReturn(f func(context.Context, p2p return c } +// Host mocks base method. +func (m *Mockfetcher) Host() host.Host { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Host") + ret0, _ := ret[0].(host.Host) + return ret0 +} + +// Host indicates an expected call of Host. +func (mr *MockfetcherMockRecorder) Host() *MockfetcherHostCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*Mockfetcher)(nil).Host)) + return &MockfetcherHostCall{Call: call} +} + +// MockfetcherHostCall wrap *gomock.Call +type MockfetcherHostCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetcherHostCall) Return(arg0 host.Host) *MockfetcherHostCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetcherHostCall) Do(f func() host.Host) *MockfetcherHostCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetcherHostCall) DoAndReturn(f func() host.Host) *MockfetcherHostCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // PeerEpochInfo mocks base method. func (m *Mockfetcher) PeerEpochInfo(arg0 context.Context, arg1 p2p.Peer, arg2 types.EpochID) (*fetch.EpochData, error) { m.ctrl.T.Helper() @@ -1270,6 +1422,80 @@ func (c *MockfetcherPeerMeshHashesCall) DoAndReturn(f func(context.Context, p2p. return c } +// Peers mocks base method. +func (m *Mockfetcher) Peers() *peers.Peers { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Peers") + ret0, _ := ret[0].(*peers.Peers) + return ret0 +} + +// Peers indicates an expected call of Peers. +func (mr *MockfetcherMockRecorder) Peers() *MockfetcherPeersCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*Mockfetcher)(nil).Peers)) + return &MockfetcherPeersCall{Call: call} +} + +// MockfetcherPeersCall wrap *gomock.Call +type MockfetcherPeersCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetcherPeersCall) Return(arg0 *peers.Peers) *MockfetcherPeersCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetcherPeersCall) Do(f func() *peers.Peers) *MockfetcherPeersCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetcherPeersCall) DoAndReturn(f func() *peers.Peers) *MockfetcherPeersCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// RegisterPeerHash mocks base method. +func (m *Mockfetcher) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RegisterPeerHash", peer, hash) +} + +// RegisterPeerHash indicates an expected call of RegisterPeerHash. +func (mr *MockfetcherMockRecorder) RegisterPeerHash(peer, hash any) *MockfetcherRegisterPeerHashCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*Mockfetcher)(nil).RegisterPeerHash), peer, hash) + return &MockfetcherRegisterPeerHashCall{Call: call} +} + +// MockfetcherRegisterPeerHashCall wrap *gomock.Call +type MockfetcherRegisterPeerHashCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockfetcherRegisterPeerHashCall) Return() *MockfetcherRegisterPeerHashCall { + c.Call = c.Call.Return() + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockfetcherRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockfetcherRegisterPeerHashCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockfetcherRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockfetcherRegisterPeerHashCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // RegisterPeerHashes mocks base method. func (m *Mockfetcher) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) { m.ctrl.T.Helper() diff --git a/syncer/syncer.go b/syncer/syncer.go index 65fd3042f8..7d9b3c0e2f 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -19,7 +19,6 @@ import ( "github.com/spacemeshos/go-spacemesh/mesh" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/server" - "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/sync2/rangesync" "github.com/spacemeshos/go-spacemesh/syncer/atxsync" @@ -252,9 +251,8 @@ func NewSyncer( s.cfg.ReconcSync.ServerConfig.ToOpts(), server.WithHardTimeout(s.cfg.ReconcSync.HardTimeout)) s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher), serverOpts) - hss := sync2.NewATXSyncSource( - s.logger, s.dispatcher, cdb.Database.(sql.StateDatabase), - fetcher.(sync2.Fetcher), s.cfg.ReconcSync.EnableActiveSync) + hss := sync2.NewATXSyncSource(s.logger, s.dispatcher, cdb.Database, fetcher, + s.cfg.ReconcSync.EnableActiveSync) s.asv2 = sync2.NewMultiEpochATXSyncer( s.logger, hss, s.cfg.ReconcSync.OldAtxSyncCfg, s.cfg.ReconcSync.NewAtxSyncCfg, s.cfg.ReconcSync.ParallelLoadLimit) @@ -561,7 +559,6 @@ func (s *Syncer) ensureATXsInSync(ctx context.Context) error { s.backgroundSync.epoch.Store(0) } if s.backgroundSync.epoch.Load() == 0 && publish.Uint32() != 0 { - // TODO: syncv2 s.logger.Debug("download atx for epoch in background", zap.Stringer("publish", publish), log.ZContext(ctx)) s.backgroundSync.epoch.Store(publish.Uint32()) ctx, cancel := context.WithCancel(ctx) diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 057ac29311..cc8064366f 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -573,7 +573,6 @@ func startWithSyncedState_SyncV2(tb testing.TB, ts *testSyncer) types.LayerID { ts.mTicker.advanceToLayer(gLayer) ts.expectMalEnsureInSync(gLayer) ts.mASV2.EXPECT().EnsureSync(gomock.Any(), types.EpochID(0), types.EpochID(1)).MinTimes(1) - // ts.mAtxSyncer.EXPECT().Download(gomock.Any(), gLayer.GetEpoch(), gomock.Any()) require.True(tb, ts.syncer.synchronize(context.Background())) ts.syncer.waitBackgroundSync() require.True(tb, ts.syncer.ListenToATXGossip()) From f869ded4d2eea4281646ee9db33112218660b4cf Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Sat, 7 Dec 2024 00:15:30 +0400 Subject: [PATCH 18/22] sync2: refactor ATXHandler.Commit() --- sync2/atxs.go | 121 +++++++++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 966ccb04b0..76ea4d8a2e 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -62,20 +62,23 @@ func NewATXHandler( } } -func (h *ATXHandler) Commit( - ctx context.Context, +type commitState struct { + state map[types.ATXID]int + total int + numDownloaded int + items []types.ATXID +} + +func (h *ATXHandler) setupState( peer p2p.Peer, base rangesync.OrderedSet, received rangesync.SeqResult, -) error { - h.logger.Debug("begin atx commit") - defer h.logger.Debug("end atx commit") - numDownloaded := 0 +) (*commitState, error) { state := make(map[types.ATXID]int) for k := range received.Seq { found, err := base.Has(k) if err != nil { - return fmt.Errorf("check if ATX exists: %w", err) + return nil, fmt.Errorf("check if ATX exists: %w", err) } if found { continue @@ -85,53 +88,71 @@ func (h *ATXHandler) Commit( state[id] = 0 } if err := received.Error(); err != nil { - return fmt.Errorf("get item: %w", err) + return nil, fmt.Errorf("get item: %w", err) } - total := len(state) - items := make([]types.ATXID, 0, h.batchSize) - startTime := h.clock.Now() - batchAttemptsRemaining := h.maxBatchRetries - for len(state) > 0 { - if len(state) == 0 { + return &commitState{ + state: state, + total: len(state), + items: make([]types.ATXID, 0, h.batchSize), + }, nil +} + +func (h *ATXHandler) getAtxs(ctx context.Context, cs *commitState) (bool, error) { + cs.items = cs.items[:0] // reuse the slice to reduce allocations + for id := range cs.state { + cs.items = append(cs.items, id) + if len(cs.items) == h.batchSize { break } - items = items[:0] - for id := range state { - items = append(items, id) - if len(items) == h.batchSize { - break - } + } + someSucceeded := false + var mtx sync.Mutex + err := h.f.GetAtxs(ctx, cs.items, system.WithATXCallback(func(id types.ATXID, err error) { + mtx.Lock() + defer mtx.Unlock() + switch { + case err == nil: + cs.numDownloaded++ + someSucceeded = true + delete(cs.state, id) + case errors.Is(err, pubsub.ErrValidationReject): + h.logger.Debug("failed to download ATX", + zap.String("atx", id.ShortString()), zap.Error(err)) + delete(cs.state, id) + case cs.state[id] >= h.maxAttempts-1: + h.logger.Debug("failed to download ATX: max attempts reached", + zap.String("atx", id.ShortString())) + delete(cs.state, id) + default: + cs.state[id]++ } + })) + return someSucceeded, err +} - someSucceeded := false - var mtx sync.Mutex - err := h.f.GetAtxs(ctx, items, system.WithATXCallback(func(id types.ATXID, err error) { - mtx.Lock() - defer mtx.Unlock() - switch { - case err == nil: - numDownloaded++ - someSucceeded = true - delete(state, id) - case errors.Is(err, pubsub.ErrValidationReject): - h.logger.Debug("failed to download ATX", - zap.String("atx", id.ShortString()), zap.Error(err)) - delete(state, id) - case state[id] >= h.maxAttempts-1: - h.logger.Debug("failed to download ATX: max attempts reached", - zap.String("atx", id.ShortString())) - delete(state, id) - default: - state[id]++ - } - })) - if err != nil { - if errors.Is(err, context.Canceled) { - return err - } - if !errors.Is(err, &fetch.BatchError{}) { - h.logger.Debug("failed to download ATXs", zap.Error(err)) - } +func (h *ATXHandler) Commit( + ctx context.Context, + peer p2p.Peer, + base rangesync.OrderedSet, + received rangesync.SeqResult, +) error { + h.logger.Debug("begin atx commit") + defer h.logger.Debug("end atx commit") + numDownloaded := 0 + cs, err := h.setupState(peer, base, received) + if err != nil { + return err + } + startTime := h.clock.Now() + batchAttemptsRemaining := h.maxBatchRetries + for len(cs.state) > 0 { + someSucceeded, err := h.getAtxs(ctx, cs) + switch { + case err == nil: + case errors.Is(err, context.Canceled): + return err + case !errors.Is(err, &fetch.BatchError{}): + h.logger.Debug("failed to download ATXs", zap.Error(err)) } if !someSucceeded { if batchAttemptsRemaining == 0 { @@ -152,7 +173,7 @@ func (h *ATXHandler) Commit( batchAttemptsRemaining = h.maxBatchRetries elapsed := h.clock.Since(startTime) h.logger.Debug("fetched atxs", - zap.Int("total", total), + zap.Int("total", cs.total), zap.Int("downloaded", numDownloaded), zap.Float64("rate per sec", float64(numDownloaded)/elapsed.Seconds())) } From ad4924b998ca085bad90ebb9df2fcc16c5ab9c6c Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Sat, 7 Dec 2024 00:17:36 +0400 Subject: [PATCH 19/22] sync2: fixup --- sync2/atxs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 76ea4d8a2e..d1c771f5e5 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -138,7 +138,6 @@ func (h *ATXHandler) Commit( ) error { h.logger.Debug("begin atx commit") defer h.logger.Debug("end atx commit") - numDownloaded := 0 cs, err := h.setupState(peer, base, received) if err != nil { return err @@ -174,8 +173,8 @@ func (h *ATXHandler) Commit( elapsed := h.clock.Since(startTime) h.logger.Debug("fetched atxs", zap.Int("total", cs.total), - zap.Int("downloaded", numDownloaded), - zap.Float64("rate per sec", float64(numDownloaded)/elapsed.Seconds())) + zap.Int("downloaded", cs.numDownloaded), + zap.Float64("rate per sec", float64(cs.numDownloaded)/elapsed.Seconds())) } return nil } From 8a1e6930aa0b7993c671af0dda637c1a3e3d099c Mon Sep 17 00:00:00 2001 From: Matthias <5011972+fasmat@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:56:27 +0000 Subject: [PATCH 20/22] Simplify interfaces --- fetch/fetch.go | 20 +-- node/node.go | 16 ++- sync2/atxs.go | 20 +-- sync2/atxs_test.go | 20 +-- sync2/interface.go | 9 +- sync2/mocks_test.go | 102 ++----------- syncer/interface.go | 6 +- syncer/malsync/syncer.go | 3 +- syncer/mocks/mocks.go | 304 --------------------------------------- syncer/syncer.go | 29 ++-- syncer/syncer_test.go | 4 + system/fetcher.go | 5 - system/mocks/fetcher.go | 62 -------- 13 files changed, 78 insertions(+), 522 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index 15865c1633..9ea164a4e6 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -11,7 +11,6 @@ import ( "sync" "time" - corehost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/protocol" "go.uber.org/zap" @@ -271,6 +270,7 @@ func NewFetch( cdb *datastore.CachedDB, proposals *store.Store, host *p2p.Host, + peersCache *peers.Peers, opts ...Option, ) (*Fetch, error) { bs := datastore.NewBlobStore(cdb, proposals) @@ -294,7 +294,7 @@ func NewFetch( opt(f) } f.getAtxsLimiter = semaphore.NewWeighted(f.cfg.GetAtxsConcurrency) - f.peers = peers.New() + f.peers = peersCache // NOTE(dshulyak) this is to avoid tests refactoring. // there is one test that covers this part. if host != nil { @@ -1009,14 +1009,6 @@ func (f *Fetch) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) { f.hashToPeers.RegisterPeerHashes(peer, hashes) } -// RegisterPeerHashes registers provided peer for a hash. -func (f *Fetch) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { - if peer == f.host.ID() { - return - } - f.hashToPeers.Add(hash, peer) -} - func (f *Fetch) SelectBestShuffled(n int) []p2p.Peer { // shuffle to split the load between peers with good latency. // and it avoids sticky behavior, when temporarily faulty peer had good latency in the past. @@ -1026,11 +1018,3 @@ func (f *Fetch) SelectBestShuffled(n int) []p2p.Peer { }) return peers } - -func (f *Fetch) Host() corehost.Host { - return f.host.(corehost.Host) -} - -func (f *Fetch) Peers() *peers.Peers { - return f.peers -} diff --git a/node/node.go b/node/node.go index e392424eb1..cd2edccf61 100644 --- a/node/node.go +++ b/node/node.go @@ -51,6 +51,7 @@ import ( "github.com/spacemeshos/go-spacemesh/datastore" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/fetch/peers" vm "github.com/spacemeshos/go-spacemesh/genvm" "github.com/spacemeshos/go-spacemesh/hare3" "github.com/spacemeshos/go-spacemesh/hare3/compat" @@ -741,17 +742,22 @@ func (app *App) initServices(ctx context.Context) error { store.WithCapacity(app.Config.Tortoise.Zdist+1), ) - flog := app.addLogger(Fetcher, lg) - fetcher, err := fetch.NewFetch(app.cachedDB, proposalsStore, app.host, + peersCache := peers.New() + flog := app.addLogger(Fetcher, lg).Zap() + fetcher, err := fetch.NewFetch( + app.cachedDB, + proposalsStore, + app.host, + peersCache, fetch.WithContext(ctx), fetch.WithConfig(app.Config.FETCH), - fetch.WithLogger(flog.Zap()), + fetch.WithLogger(flog), ) if err != nil { return fmt.Errorf("create fetcher: %w", err) } app.eg.Go(func() error { - return blockssync.Sync(ctx, flog.Zap(), msh.MissingBlocks(), fetcher) + return blockssync.Sync(ctx, flog, msh.MissingBlocks(), fetcher) }) hOracle, err := eligibility.New( @@ -807,6 +813,8 @@ func (app *App) initServices(ctx context.Context) error { msh, trtl, fetcher, + peersCache, + app.host, patrol, certifier, atxsync.New(fetcher, app.db, app.localDB, diff --git a/sync2/atxs.go b/sync2/atxs.go index d1c771f5e5..2df403e861 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -8,11 +8,13 @@ import ( "time" "github.com/jonboulle/clockwork" + "github.com/libp2p/go-libp2p/core/host" "go.uber.org/zap" "golang.org/x/sync/errgroup" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/pubsub" "github.com/spacemeshos/go-spacemesh/p2p/server" @@ -84,7 +86,7 @@ func (h *ATXHandler) setupState( continue } id := types.BytesToATXID(k) - h.f.RegisterPeerHash(peer, id.Hash32()) + h.f.RegisterPeerHashes(peer, []types.Hash32{id.Hash32()}) state[id] = 0 } if err := received.Error(); err != nil { @@ -297,18 +299,18 @@ func NewATXSyncer( cfg Config, db sql.Database, f Fetcher, + peers *peers.Peers, epoch types.EpochID, enableActiveSync bool, ) *P2PHashSync { curSet := dbset.NewDBSet(db, atxsTable(epoch), 32, cfg.MaxDepth) - handler := NewATXHandler(logger, f, cfg.BatchSize, cfg.MaxAttempts, - cfg.MaxBatchRetries, cfg.FailedBatchDelay, nil) - return NewP2PHashSync(logger, d, name, curSet, 32, f.Peers(), handler, cfg, enableActiveSync) + handler := NewATXHandler(logger, f, cfg.BatchSize, cfg.MaxAttempts, cfg.MaxBatchRetries, cfg.FailedBatchDelay, nil) + return NewP2PHashSync(logger, d, name, curSet, 32, peers, handler, cfg, enableActiveSync) } -func NewDispatcher(logger *zap.Logger, f Fetcher, opts []server.Opt) *rangesync.Dispatcher { +func NewDispatcher(logger *zap.Logger, host host.Host, opts []server.Opt) *rangesync.Dispatcher { d := rangesync.NewDispatcher(logger) - d.SetupServer(f.Host(), proto, opts...) + d.SetupServer(host, proto, opts...) return d } @@ -317,6 +319,7 @@ type ATXSyncSource struct { d *rangesync.Dispatcher db sql.Database f Fetcher + peers *peers.Peers enableActiveSync bool } @@ -327,12 +330,13 @@ func NewATXSyncSource( d *rangesync.Dispatcher, db sql.Database, f Fetcher, + peers *peers.Peers, enableActiveSync bool, ) *ATXSyncSource { - return &ATXSyncSource{logger: logger, d: d, db: db, f: f, enableActiveSync: enableActiveSync} + return &ATXSyncSource{logger: logger, d: d, db: db, f: f, peers: peers, enableActiveSync: enableActiveSync} } // CreateHashSync implements HashSyncSource. func (as *ATXSyncSource) CreateHashSync(name string, cfg Config, epoch types.EpochID) HashSync { - return NewATXSyncer(as.logger.Named(name), as.d, name, cfg, as.db, as.f, epoch, as.enableActiveSync) + return NewATXSyncer(as.logger.Named(name), as.d, name, cfg, as.db, as.f, as.peers, epoch, as.enableActiveSync) } diff --git a/sync2/atxs_test.go b/sync2/atxs_test.go index bf11dffb9c..bced77da4a 100644 --- a/sync2/atxs_test.go +++ b/sync2/atxs_test.go @@ -55,8 +55,8 @@ func TestAtxHandler_Success(t *testing.T) { h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) - f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id.Bytes())) + f.EXPECT().RegisterPeerHashes(peer, []types.Hash32{id.Hash32()}) } toFetch := make(map[types.ATXID]bool) for _, id := range allAtxs { @@ -102,8 +102,8 @@ func TestAtxHandler_Retry(t *testing.T) { h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) - f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id.Bytes())) + f.EXPECT().RegisterPeerHashes(peer, []types.Hash32{id.Hash32()}) } failCount := 0 var fetched []types.ATXID @@ -184,8 +184,8 @@ func TestAtxHandler_Cancel(t *testing.T) { clock := clockwork.NewFakeClock() h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) - baseSet.EXPECT().Has(rangesync.KeyBytes(atxID[:])).Return(false, nil) - f.EXPECT().RegisterPeerHash(peer, atxID.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(atxID.Bytes())).Return(false, nil) + f.EXPECT().RegisterPeerHashes(peer, []types.Hash32{atxID.Hash32()}) f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { return context.Canceled @@ -218,8 +218,8 @@ func TestAtxHandler_BatchRetry(t *testing.T) { h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) - f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id.Bytes())) + f.EXPECT().RegisterPeerHashes(peer, []types.Hash32{id.Hash32()}) } f.EXPECT().GetAtxs(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, atxs []types.ATXID, opts ...system.GetAtxOpt) error { @@ -273,8 +273,8 @@ func TestAtxHandler_BatchRetry_Fail(t *testing.T) { h := sync2.NewATXHandler(logger, f, batchSize, maxAttempts, maxBatchRetries, batchRetryDelay, clock) baseSet := mocks.NewMockOrderedSet(ctrl) for _, id := range allAtxs { - baseSet.EXPECT().Has(rangesync.KeyBytes(id[:])) - f.EXPECT().RegisterPeerHash(peer, id.Hash32()) + baseSet.EXPECT().Has(rangesync.KeyBytes(id.Bytes())) + f.EXPECT().RegisterPeerHashes(peer, []types.Hash32{id.Hash32()}) } sr := rangesync.SeqResult{ Seq: func(yield func(k rangesync.KeyBytes) bool) { diff --git a/sync2/interface.go b/sync2/interface.go index 624c373d2d..9e04a25c80 100644 --- a/sync2/interface.go +++ b/sync2/interface.go @@ -3,10 +3,7 @@ package sync2 import ( "context" - "github.com/libp2p/go-libp2p/core/host" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/system" ) @@ -14,10 +11,8 @@ import ( //go:generate mockgen -typed -package=sync2_test -destination=./mocks_test.go -source=./interface.go type Fetcher interface { - system.AtxFetcher - Host() host.Host - Peers() *peers.Peers - RegisterPeerHash(peer p2p.Peer, hash types.Hash32) + GetAtxs(context.Context, []types.ATXID, ...system.GetAtxOpt) error + RegisterPeerHashes(peer p2p.Peer, hash []types.Hash32) } type HashSync interface { diff --git a/sync2/mocks_test.go b/sync2/mocks_test.go index 640dde1143..01231fcc25 100644 --- a/sync2/mocks_test.go +++ b/sync2/mocks_test.go @@ -13,9 +13,7 @@ import ( context "context" reflect "reflect" - host "github.com/libp2p/go-libp2p/core/host" types "github.com/spacemeshos/go-spacemesh/common/types" - peers "github.com/spacemeshos/go-spacemesh/fetch/peers" p2p "github.com/spacemeshos/go-spacemesh/p2p" sync2 "github.com/spacemeshos/go-spacemesh/sync2" system "github.com/spacemeshos/go-spacemesh/system" @@ -89,114 +87,38 @@ func (c *MockFetcherGetAtxsCall) DoAndReturn(f func(context.Context, []types.ATX return c } -// Host mocks base method. -func (m *MockFetcher) Host() host.Host { +// RegisterPeerHashes mocks base method. +func (m *MockFetcher) RegisterPeerHashes(peer p2p.Peer, hash []types.Hash32) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Host") - ret0, _ := ret[0].(host.Host) - return ret0 -} - -// Host indicates an expected call of Host. -func (mr *MockFetcherMockRecorder) Host() *MockFetcherHostCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*MockFetcher)(nil).Host)) - return &MockFetcherHostCall{Call: call} -} - -// MockFetcherHostCall wrap *gomock.Call -type MockFetcherHostCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockFetcherHostCall) Return(arg0 host.Host) *MockFetcherHostCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockFetcherHostCall) Do(f func() host.Host) *MockFetcherHostCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockFetcherHostCall) DoAndReturn(f func() host.Host) *MockFetcherHostCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Peers mocks base method. -func (m *MockFetcher) Peers() *peers.Peers { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Peers") - ret0, _ := ret[0].(*peers.Peers) - return ret0 -} - -// Peers indicates an expected call of Peers. -func (mr *MockFetcherMockRecorder) Peers() *MockFetcherPeersCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*MockFetcher)(nil).Peers)) - return &MockFetcherPeersCall{Call: call} -} - -// MockFetcherPeersCall wrap *gomock.Call -type MockFetcherPeersCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockFetcherPeersCall) Return(arg0 *peers.Peers) *MockFetcherPeersCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockFetcherPeersCall) Do(f func() *peers.Peers) *MockFetcherPeersCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockFetcherPeersCall) DoAndReturn(f func() *peers.Peers) *MockFetcherPeersCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// RegisterPeerHash mocks base method. -func (m *MockFetcher) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RegisterPeerHash", peer, hash) + m.ctrl.Call(m, "RegisterPeerHashes", peer, hash) } -// RegisterPeerHash indicates an expected call of RegisterPeerHash. -func (mr *MockFetcherMockRecorder) RegisterPeerHash(peer, hash any) *MockFetcherRegisterPeerHashCall { +// RegisterPeerHashes indicates an expected call of RegisterPeerHashes. +func (mr *MockFetcherMockRecorder) RegisterPeerHashes(peer, hash any) *MockFetcherRegisterPeerHashesCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*MockFetcher)(nil).RegisterPeerHash), peer, hash) - return &MockFetcherRegisterPeerHashCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHashes", reflect.TypeOf((*MockFetcher)(nil).RegisterPeerHashes), peer, hash) + return &MockFetcherRegisterPeerHashesCall{Call: call} } -// MockFetcherRegisterPeerHashCall wrap *gomock.Call -type MockFetcherRegisterPeerHashCall struct { +// MockFetcherRegisterPeerHashesCall wrap *gomock.Call +type MockFetcherRegisterPeerHashesCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *MockFetcherRegisterPeerHashCall) Return() *MockFetcherRegisterPeerHashCall { +func (c *MockFetcherRegisterPeerHashesCall) Return() *MockFetcherRegisterPeerHashesCall { c.Call = c.Call.Return() return c } // Do rewrite *gomock.Call.Do -func (c *MockFetcherRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockFetcherRegisterPeerHashCall { +func (c *MockFetcherRegisterPeerHashesCall) Do(f func(p2p.Peer, []types.Hash32)) *MockFetcherRegisterPeerHashesCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockFetcherRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockFetcherRegisterPeerHashCall { +func (c *MockFetcherRegisterPeerHashesCall) DoAndReturn(f func(p2p.Peer, []types.Hash32)) *MockFetcherRegisterPeerHashesCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/syncer/interface.go b/syncer/interface.go index 2e99b8a4ac..95d8f17125 100644 --- a/syncer/interface.go +++ b/syncer/interface.go @@ -7,7 +7,6 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/fetch" "github.com/spacemeshos/go-spacemesh/p2p" - "github.com/spacemeshos/go-spacemesh/sync2" "github.com/spacemeshos/go-spacemesh/system" ) @@ -43,13 +42,12 @@ type malSyncer interface { // fetcher is the interface to the low-level fetching. type fetcher interface { - GetMaliciousIDs(context.Context, p2p.Peer) ([]types.NodeID, error) GetLayerData(context.Context, p2p.Peer, types.LayerID) ([]byte, error) GetLayerOpinions(context.Context, p2p.Peer, types.LayerID) ([]byte, error) GetCert(context.Context, types.LayerID, types.BlockID, []p2p.Peer) (*types.Certificate, error) - sync2.Fetcher - system.MalfeasanceProofFetcher + GetAtxs(context.Context, []types.ATXID, ...system.GetAtxOpt) error + GetMalfeasanceProofs(context.Context, []types.NodeID) error GetBallots(context.Context, []types.BallotID) error GetBlocks(context.Context, []types.BlockID) error RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) diff --git a/syncer/malsync/syncer.go b/syncer/malsync/syncer.go index 4633f07de3..a9b2605d36 100644 --- a/syncer/malsync/syncer.go +++ b/syncer/malsync/syncer.go @@ -19,7 +19,6 @@ import ( "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/identities" "github.com/spacemeshos/go-spacemesh/sql/malsync" - "github.com/spacemeshos/go-spacemesh/system" ) //go:generate mockgen -typed -package=mocks -destination=./mocks/mocks.go -source=./syncer.go @@ -27,7 +26,7 @@ import ( type fetcher interface { SelectBestShuffled(int) []p2p.Peer GetMaliciousIDs(context.Context, p2p.Peer) ([]types.NodeID, error) - system.MalfeasanceProofFetcher + GetMalfeasanceProofs(context.Context, []types.NodeID) error } type Opt func(*Syncer) diff --git a/syncer/mocks/mocks.go b/syncer/mocks/mocks.go index 052ead48ab..8ad4707d01 100644 --- a/syncer/mocks/mocks.go +++ b/syncer/mocks/mocks.go @@ -14,10 +14,8 @@ import ( reflect "reflect" time "time" - host "github.com/libp2p/go-libp2p/core/host" types "github.com/spacemeshos/go-spacemesh/common/types" fetch "github.com/spacemeshos/go-spacemesh/fetch" - peers "github.com/spacemeshos/go-spacemesh/fetch/peers" p2p "github.com/spacemeshos/go-spacemesh/p2p" system "github.com/spacemeshos/go-spacemesh/system" gomock "go.uber.org/mock/gomock" @@ -421,83 +419,6 @@ func (c *MockfetchLogicGetMalfeasanceProofsCall) DoAndReturn(f func(context.Cont return c } -// GetMaliciousIDs mocks base method. -func (m *MockfetchLogic) GetMaliciousIDs(arg0 context.Context, arg1 p2p.Peer) ([]types.NodeID, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetMaliciousIDs", arg0, arg1) - ret0, _ := ret[0].([]types.NodeID) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetMaliciousIDs indicates an expected call of GetMaliciousIDs. -func (mr *MockfetchLogicMockRecorder) GetMaliciousIDs(arg0, arg1 any) *MockfetchLogicGetMaliciousIDsCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaliciousIDs", reflect.TypeOf((*MockfetchLogic)(nil).GetMaliciousIDs), arg0, arg1) - return &MockfetchLogicGetMaliciousIDsCall{Call: call} -} - -// MockfetchLogicGetMaliciousIDsCall wrap *gomock.Call -type MockfetchLogicGetMaliciousIDsCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetchLogicGetMaliciousIDsCall) Return(arg0 []types.NodeID, arg1 error) *MockfetchLogicGetMaliciousIDsCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetchLogicGetMaliciousIDsCall) Do(f func(context.Context, p2p.Peer) ([]types.NodeID, error)) *MockfetchLogicGetMaliciousIDsCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetchLogicGetMaliciousIDsCall) DoAndReturn(f func(context.Context, p2p.Peer) ([]types.NodeID, error)) *MockfetchLogicGetMaliciousIDsCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Host mocks base method. -func (m *MockfetchLogic) Host() host.Host { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Host") - ret0, _ := ret[0].(host.Host) - return ret0 -} - -// Host indicates an expected call of Host. -func (mr *MockfetchLogicMockRecorder) Host() *MockfetchLogicHostCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*MockfetchLogic)(nil).Host)) - return &MockfetchLogicHostCall{Call: call} -} - -// MockfetchLogicHostCall wrap *gomock.Call -type MockfetchLogicHostCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetchLogicHostCall) Return(arg0 host.Host) *MockfetchLogicHostCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetchLogicHostCall) Do(f func() host.Host) *MockfetchLogicHostCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetchLogicHostCall) DoAndReturn(f func() host.Host) *MockfetchLogicHostCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // PeerEpochInfo mocks base method. func (m *MockfetchLogic) PeerEpochInfo(arg0 context.Context, arg1 p2p.Peer, arg2 types.EpochID) (*fetch.EpochData, error) { m.ctrl.T.Helper() @@ -576,44 +497,6 @@ func (c *MockfetchLogicPeerMeshHashesCall) DoAndReturn(f func(context.Context, p return c } -// Peers mocks base method. -func (m *MockfetchLogic) Peers() *peers.Peers { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Peers") - ret0, _ := ret[0].(*peers.Peers) - return ret0 -} - -// Peers indicates an expected call of Peers. -func (mr *MockfetchLogicMockRecorder) Peers() *MockfetchLogicPeersCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*MockfetchLogic)(nil).Peers)) - return &MockfetchLogicPeersCall{Call: call} -} - -// MockfetchLogicPeersCall wrap *gomock.Call -type MockfetchLogicPeersCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetchLogicPeersCall) Return(arg0 *peers.Peers) *MockfetchLogicPeersCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetchLogicPeersCall) Do(f func() *peers.Peers) *MockfetchLogicPeersCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetchLogicPeersCall) DoAndReturn(f func() *peers.Peers) *MockfetchLogicPeersCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // PollLayerData mocks base method. func (m *MockfetchLogic) PollLayerData(arg0 context.Context, arg1 types.LayerID, arg2 ...p2p.Peer) error { m.ctrl.T.Helper() @@ -697,42 +580,6 @@ func (c *MockfetchLogicPollLayerOpinionsCall) DoAndReturn(f func(context.Context return c } -// RegisterPeerHash mocks base method. -func (m *MockfetchLogic) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RegisterPeerHash", peer, hash) -} - -// RegisterPeerHash indicates an expected call of RegisterPeerHash. -func (mr *MockfetchLogicMockRecorder) RegisterPeerHash(peer, hash any) *MockfetchLogicRegisterPeerHashCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*MockfetchLogic)(nil).RegisterPeerHash), peer, hash) - return &MockfetchLogicRegisterPeerHashCall{Call: call} -} - -// MockfetchLogicRegisterPeerHashCall wrap *gomock.Call -type MockfetchLogicRegisterPeerHashCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetchLogicRegisterPeerHashCall) Return() *MockfetchLogicRegisterPeerHashCall { - c.Call = c.Call.Return() - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetchLogicRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockfetchLogicRegisterPeerHashCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetchLogicRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockfetchLogicRegisterPeerHashCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // RegisterPeerHashes mocks base method. func (m *MockfetchLogic) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) { m.ctrl.T.Helper() @@ -1267,83 +1114,6 @@ func (c *MockfetcherGetMalfeasanceProofsCall) DoAndReturn(f func(context.Context return c } -// GetMaliciousIDs mocks base method. -func (m *Mockfetcher) GetMaliciousIDs(arg0 context.Context, arg1 p2p.Peer) ([]types.NodeID, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetMaliciousIDs", arg0, arg1) - ret0, _ := ret[0].([]types.NodeID) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetMaliciousIDs indicates an expected call of GetMaliciousIDs. -func (mr *MockfetcherMockRecorder) GetMaliciousIDs(arg0, arg1 any) *MockfetcherGetMaliciousIDsCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaliciousIDs", reflect.TypeOf((*Mockfetcher)(nil).GetMaliciousIDs), arg0, arg1) - return &MockfetcherGetMaliciousIDsCall{Call: call} -} - -// MockfetcherGetMaliciousIDsCall wrap *gomock.Call -type MockfetcherGetMaliciousIDsCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetcherGetMaliciousIDsCall) Return(arg0 []types.NodeID, arg1 error) *MockfetcherGetMaliciousIDsCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetcherGetMaliciousIDsCall) Do(f func(context.Context, p2p.Peer) ([]types.NodeID, error)) *MockfetcherGetMaliciousIDsCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetcherGetMaliciousIDsCall) DoAndReturn(f func(context.Context, p2p.Peer) ([]types.NodeID, error)) *MockfetcherGetMaliciousIDsCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Host mocks base method. -func (m *Mockfetcher) Host() host.Host { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Host") - ret0, _ := ret[0].(host.Host) - return ret0 -} - -// Host indicates an expected call of Host. -func (mr *MockfetcherMockRecorder) Host() *MockfetcherHostCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Host", reflect.TypeOf((*Mockfetcher)(nil).Host)) - return &MockfetcherHostCall{Call: call} -} - -// MockfetcherHostCall wrap *gomock.Call -type MockfetcherHostCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetcherHostCall) Return(arg0 host.Host) *MockfetcherHostCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetcherHostCall) Do(f func() host.Host) *MockfetcherHostCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetcherHostCall) DoAndReturn(f func() host.Host) *MockfetcherHostCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // PeerEpochInfo mocks base method. func (m *Mockfetcher) PeerEpochInfo(arg0 context.Context, arg1 p2p.Peer, arg2 types.EpochID) (*fetch.EpochData, error) { m.ctrl.T.Helper() @@ -1422,80 +1192,6 @@ func (c *MockfetcherPeerMeshHashesCall) DoAndReturn(f func(context.Context, p2p. return c } -// Peers mocks base method. -func (m *Mockfetcher) Peers() *peers.Peers { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Peers") - ret0, _ := ret[0].(*peers.Peers) - return ret0 -} - -// Peers indicates an expected call of Peers. -func (mr *MockfetcherMockRecorder) Peers() *MockfetcherPeersCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Peers", reflect.TypeOf((*Mockfetcher)(nil).Peers)) - return &MockfetcherPeersCall{Call: call} -} - -// MockfetcherPeersCall wrap *gomock.Call -type MockfetcherPeersCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetcherPeersCall) Return(arg0 *peers.Peers) *MockfetcherPeersCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetcherPeersCall) Do(f func() *peers.Peers) *MockfetcherPeersCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetcherPeersCall) DoAndReturn(f func() *peers.Peers) *MockfetcherPeersCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// RegisterPeerHash mocks base method. -func (m *Mockfetcher) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RegisterPeerHash", peer, hash) -} - -// RegisterPeerHash indicates an expected call of RegisterPeerHash. -func (mr *MockfetcherMockRecorder) RegisterPeerHash(peer, hash any) *MockfetcherRegisterPeerHashCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterPeerHash", reflect.TypeOf((*Mockfetcher)(nil).RegisterPeerHash), peer, hash) - return &MockfetcherRegisterPeerHashCall{Call: call} -} - -// MockfetcherRegisterPeerHashCall wrap *gomock.Call -type MockfetcherRegisterPeerHashCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockfetcherRegisterPeerHashCall) Return() *MockfetcherRegisterPeerHashCall { - c.Call = c.Call.Return() - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockfetcherRegisterPeerHashCall) Do(f func(p2p.Peer, types.Hash32)) *MockfetcherRegisterPeerHashCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockfetcherRegisterPeerHashCall) DoAndReturn(f func(p2p.Peer, types.Hash32)) *MockfetcherRegisterPeerHashCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // RegisterPeerHashes mocks base method. func (m *Mockfetcher) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) { m.ctrl.T.Helper() diff --git a/syncer/syncer.go b/syncer/syncer.go index 7d9b3c0e2f..bc20a488bf 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/libp2p/go-libp2p/core/host" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -15,6 +16,7 @@ import ( "github.com/spacemeshos/go-spacemesh/datastore" "github.com/spacemeshos/go-spacemesh/events" "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/mesh" "github.com/spacemeshos/go-spacemesh/p2p" @@ -212,6 +214,8 @@ func NewSyncer( mesh *mesh.Mesh, tortoise system.Tortoise, fetcher fetcher, + peersCache *peers.Peers, + host host.Host, patrol layerPatrol, ch certHandler, atxSyncer atxSyncer, @@ -247,15 +251,24 @@ func NewSyncer( s.lastLayerSynced.Store(s.mesh.LatestLayer().Uint32()) s.lastEpochSynced.Store(types.GetEffectiveGenesis().GetEpoch().Uint32() - 1) if s.cfg.ReconcSync.Enable && s.asv2 == nil { - serverOpts := append( - s.cfg.ReconcSync.ServerConfig.ToOpts(), - server.WithHardTimeout(s.cfg.ReconcSync.HardTimeout)) - s.dispatcher = sync2.NewDispatcher(s.logger, fetcher.(sync2.Fetcher), serverOpts) - hss := sync2.NewATXSyncSource(s.logger, s.dispatcher, cdb.Database, fetcher, - s.cfg.ReconcSync.EnableActiveSync) + serverOpts := s.cfg.ReconcSync.ServerConfig.ToOpts() + serverOpts = append(serverOpts, server.WithHardTimeout(s.cfg.ReconcSync.HardTimeout)) + s.dispatcher = sync2.NewDispatcher(s.logger, host, serverOpts) + hss := sync2.NewATXSyncSource( + s.logger, + s.dispatcher, + cdb.Database, + fetcher, + peersCache, + s.cfg.ReconcSync.EnableActiveSync, + ) s.asv2 = sync2.NewMultiEpochATXSyncer( - s.logger, hss, s.cfg.ReconcSync.OldAtxSyncCfg, s.cfg.ReconcSync.NewAtxSyncCfg, - s.cfg.ReconcSync.ParallelLoadLimit) + s.logger, + hss, + s.cfg.ReconcSync.OldAtxSyncCfg, + s.cfg.ReconcSync.NewAtxSyncCfg, + s.cfg.ReconcSync.ParallelLoadLimit, + ) } return s } diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index cc8064366f..a16546f596 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -143,6 +143,8 @@ func newTestSyncerWithConfig(tb testing.TB, cfg Config) *testSyncer { ts.msh, ts.mTortoise, nil, + nil, + nil, ts.mLyrPatrol, ts.mCertHdr, ts.mAtxSyncer, @@ -873,6 +875,8 @@ func TestSynchronize_RecoverFromCheckpoint(t *testing.T) { ts.msh, ts.mTortoise, nil, + nil, + nil, ts.mLyrPatrol, ts.mCertHdr, ts.mAtxSyncer, diff --git a/system/fetcher.go b/system/fetcher.go index 343f874516..2d8c24b560 100644 --- a/system/fetcher.go +++ b/system/fetcher.go @@ -83,11 +83,6 @@ type ActiveSetFetcher interface { GetActiveSet(context.Context, types.Hash32) error } -// MalfeasanceProofFetcher defines an interface for fetching malfeasance proofs. -type MalfeasanceProofFetcher interface { - GetMalfeasanceProofs(context.Context, []types.NodeID) error -} - // PeerTracker defines an interface to track peer hashes. type PeerTracker interface { RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) diff --git a/system/mocks/fetcher.go b/system/mocks/fetcher.go index 4bb3426974..4c4f2324dc 100644 --- a/system/mocks/fetcher.go +++ b/system/mocks/fetcher.go @@ -865,68 +865,6 @@ func (c *MockActiveSetFetcherGetActiveSetCall) DoAndReturn(f func(context.Contex return c } -// MockMalfeasanceProofFetcher is a mock of MalfeasanceProofFetcher interface. -type MockMalfeasanceProofFetcher struct { - ctrl *gomock.Controller - recorder *MockMalfeasanceProofFetcherMockRecorder - isgomock struct{} -} - -// MockMalfeasanceProofFetcherMockRecorder is the mock recorder for MockMalfeasanceProofFetcher. -type MockMalfeasanceProofFetcherMockRecorder struct { - mock *MockMalfeasanceProofFetcher -} - -// NewMockMalfeasanceProofFetcher creates a new mock instance. -func NewMockMalfeasanceProofFetcher(ctrl *gomock.Controller) *MockMalfeasanceProofFetcher { - mock := &MockMalfeasanceProofFetcher{ctrl: ctrl} - mock.recorder = &MockMalfeasanceProofFetcherMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMalfeasanceProofFetcher) EXPECT() *MockMalfeasanceProofFetcherMockRecorder { - return m.recorder -} - -// GetMalfeasanceProofs mocks base method. -func (m *MockMalfeasanceProofFetcher) GetMalfeasanceProofs(arg0 context.Context, arg1 []types.NodeID) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetMalfeasanceProofs", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// GetMalfeasanceProofs indicates an expected call of GetMalfeasanceProofs. -func (mr *MockMalfeasanceProofFetcherMockRecorder) GetMalfeasanceProofs(arg0, arg1 any) *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMalfeasanceProofs", reflect.TypeOf((*MockMalfeasanceProofFetcher)(nil).GetMalfeasanceProofs), arg0, arg1) - return &MockMalfeasanceProofFetcherGetMalfeasanceProofsCall{Call: call} -} - -// MockMalfeasanceProofFetcherGetMalfeasanceProofsCall wrap *gomock.Call -type MockMalfeasanceProofFetcherGetMalfeasanceProofsCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall) Return(arg0 error) *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall) Do(f func(context.Context, []types.NodeID) error) *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall) DoAndReturn(f func(context.Context, []types.NodeID) error) *MockMalfeasanceProofFetcherGetMalfeasanceProofsCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // MockPeerTracker is a mock of PeerTracker interface. type MockPeerTracker struct { ctrl *gomock.Controller From c22bd45d0c4e92543df01eec9852d158cdf098f1 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 16 Dec 2024 03:17:42 +0400 Subject: [PATCH 21/22] Fixup after Fetcher update --- fetch/fetch.go | 4 ++-- fetch/fetch_test.go | 4 ++++ fetch/mesh_data_test.go | 2 ++ fetch/p2p_test.go | 3 +++ node/node.go | 6 +++--- syncer/syncer.go | 4 ++-- systest/tests/distributed_post_verification_test.go | 2 ++ 7 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index 9ea164a4e6..453ce11a7c 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -270,7 +270,7 @@ func NewFetch( cdb *datastore.CachedDB, proposals *store.Store, host *p2p.Host, - peersCache *peers.Peers, + peerCache *peers.Peers, opts ...Option, ) (*Fetch, error) { bs := datastore.NewBlobStore(cdb, proposals) @@ -294,7 +294,7 @@ func NewFetch( opt(f) } f.getAtxsLimiter = semaphore.NewWeighted(f.cfg.GetAtxsConcurrency) - f.peers = peersCache + f.peers = peerCache // NOTE(dshulyak) this is to avoid tests refactoring. // there is one test that covers this part. if host != nil { diff --git a/fetch/fetch_test.go b/fetch/fetch_test.go index 84cf24125b..e3ba8ba665 100644 --- a/fetch/fetch_test.go +++ b/fetch/fetch_test.go @@ -18,6 +18,7 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/datastore" "github.com/spacemeshos/go-spacemesh/fetch/mocks" + "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/pubsub" "github.com/spacemeshos/go-spacemesh/p2p/server" @@ -87,6 +88,7 @@ func createFetch(tb testing.TB) *testFetch { cdb, store.New(), nil, + peers.New(), WithContext(context.Background()), WithConfig(cfg), WithLogger(lg), @@ -133,6 +135,7 @@ func TestFetch_Start(t *testing.T) { cdb, store.New(), nil, + peers.New(), WithContext(context.Background()), WithConfig(DefaultConfig()), WithLogger(lg), @@ -407,6 +410,7 @@ func TestFetch_PeerDroppedWhenMessageResultsInValidationReject(t *testing.T) { cdb, store.New(), h, + peers.New(), WithContext(ctx), WithConfig(cfg), WithLogger(lg), diff --git a/fetch/mesh_data_test.go b/fetch/mesh_data_test.go index f2a4a40292..f7c414c8df 100644 --- a/fetch/mesh_data_test.go +++ b/fetch/mesh_data_test.go @@ -21,6 +21,7 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/datastore" "github.com/spacemeshos/go-spacemesh/fetch/mocks" + "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/peerinfo" @@ -1023,6 +1024,7 @@ func Test_GetAtxsLimiting(t *testing.T) { host, err := p2p.Upgrade(mesh.Hosts()[0]) require.NoError(t, err) f, err := NewFetch(cdb, store.New(), host, + peers.New(), WithContext(context.Background()), withServers(map[string]requester{hashProtocol: client}), WithConfig(cfg), diff --git a/fetch/p2p_test.go b/fetch/p2p_test.go index 64fe26511d..183e5cafa1 100644 --- a/fetch/p2p_test.go +++ b/fetch/p2p_test.go @@ -15,6 +15,7 @@ import ( "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/datastore" + "github.com/spacemeshos/go-spacemesh/fetch/peers" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/server" "github.com/spacemeshos/go-spacemesh/proposals/store" @@ -132,6 +133,7 @@ func createP2PFetch( tpf.serverCDB, tpf.serverPDB, serverHost, + peers.New(), WithContext(ctx), WithConfig(p2pFetchCfg(serverStreaming)), WithLogger(lg), @@ -153,6 +155,7 @@ func createP2PFetch( tpf.clientCDB, tpf.clientPDB, clientHost, + peers.New(), WithContext(ctx), WithConfig(p2pFetchCfg(clientStreaming)), WithLogger(lg), diff --git a/node/node.go b/node/node.go index cd2edccf61..841aaade2f 100644 --- a/node/node.go +++ b/node/node.go @@ -742,13 +742,13 @@ func (app *App) initServices(ctx context.Context) error { store.WithCapacity(app.Config.Tortoise.Zdist+1), ) - peersCache := peers.New() + peerCache := peers.New() flog := app.addLogger(Fetcher, lg).Zap() fetcher, err := fetch.NewFetch( app.cachedDB, proposalsStore, app.host, - peersCache, + peerCache, fetch.WithContext(ctx), fetch.WithConfig(app.Config.FETCH), fetch.WithLogger(flog), @@ -813,7 +813,7 @@ func (app *App) initServices(ctx context.Context) error { msh, trtl, fetcher, - peersCache, + peerCache, app.host, patrol, certifier, diff --git a/syncer/syncer.go b/syncer/syncer.go index bc20a488bf..c57ce6c2d1 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -214,7 +214,7 @@ func NewSyncer( mesh *mesh.Mesh, tortoise system.Tortoise, fetcher fetcher, - peersCache *peers.Peers, + peerCache *peers.Peers, host host.Host, patrol layerPatrol, ch certHandler, @@ -259,7 +259,7 @@ func NewSyncer( s.dispatcher, cdb.Database, fetcher, - peersCache, + peerCache, s.cfg.ReconcSync.EnableActiveSync, ) s.asv2 = sync2.NewMultiEpochATXSyncer( diff --git a/systest/tests/distributed_post_verification_test.go b/systest/tests/distributed_post_verification_test.go index d5cbbcc2d0..94784843f9 100644 --- a/systest/tests/distributed_post_verification_test.go +++ b/systest/tests/distributed_post_verification_test.go @@ -26,6 +26,7 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/datastore" "github.com/spacemeshos/go-spacemesh/fetch" + "github.com/spacemeshos/go-spacemesh/fetch/peers" mwire "github.com/spacemeshos/go-spacemesh/malfeasance/wire" "github.com/spacemeshos/go-spacemesh/p2p" "github.com/spacemeshos/go-spacemesh/p2p/handshake" @@ -120,6 +121,7 @@ func TestPostMalfeasanceProof(t *testing.T) { ) fetcher, err := fetch.NewFetch(cdb, proposalsStore, host, + peers.New(), fetch.WithContext(ctx), fetch.WithConfig(cfg.FETCH), fetch.WithLogger(logger.Named("fetcher")), From 6e40bda867ebcd64f08f710d5d5938b2055a964c Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Mon, 16 Dec 2024 03:30:36 +0400 Subject: [PATCH 22/22] Address comments --- sync2/atxs.go | 3 ++- sync2/fptree/nodepool_test.go | 1 - syncer/syncer.go | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sync2/atxs.go b/sync2/atxs.go index 2df403e861..bec0aabf03 100644 --- a/sync2/atxs.go +++ b/sync2/atxs.go @@ -148,11 +148,12 @@ func (h *ATXHandler) Commit( batchAttemptsRemaining := h.maxBatchRetries for len(cs.state) > 0 { someSucceeded, err := h.getAtxs(ctx, cs) + batchErr := &fetch.BatchError{} switch { case err == nil: case errors.Is(err, context.Canceled): return err - case !errors.Is(err, &fetch.BatchError{}): + case !errors.As(err, &batchErr): h.logger.Debug("failed to download ATXs", zap.Error(err)) } if !someSucceeded { diff --git a/sync2/fptree/nodepool_test.go b/sync2/fptree/nodepool_test.go index 32dcb2bbd8..156c5bedb5 100644 --- a/sync2/fptree/nodepool_test.go +++ b/sync2/fptree/nodepool_test.go @@ -42,7 +42,6 @@ func TestNodePool(t *testing.T) { require.Equal(t, uint32(1), np.refCount(idx2)) require.Equal(t, nodeIndex(2), idx3) - require.Nil(t, nil, idx3) require.Equal(t, idx1, np.left(idx3)) require.Equal(t, idx2, np.right(idx3)) require.False(t, np.leaf(idx3)) diff --git a/syncer/syncer.go b/syncer/syncer.go index c57ce6c2d1..74bbbdda1f 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -648,7 +648,6 @@ func (s *Syncer) ensureATXsInSyncV2(ctx context.Context) error { func (s *Syncer) ensureMalfeasanceInSync(ctx context.Context) error { // TODO: use syncv2 for malfeasance proofs: - // https://github.com/spacemeshos/go-spacemesh/issues/3987 current := s.ticker.CurrentLayer() if !s.ListenToATXGossip() { s.logger.Info("syncing malicious proofs", log.ZContext(ctx))