Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Merged by Bors] - in memory index with atx data for consensus layer #5013

Closed
wants to merge 46 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
b3fbc3e
add data structure for cache
dshulyak Sep 14, 2023
d5e1e10
warmup and evicion handler
dshulyak Sep 15, 2023
e4009bd
populat cache and evict
dshulyak Sep 15, 2023
dc74a14
Merge branch 'develop' into consensus-cache
dshulyak Sep 19, 2023
f3b4fdb
support slow and fast path using cache in proposals handler
dshulyak Sep 19, 2023
bcbb769
bad init
dshulyak Sep 19, 2023
22dd804
Merge branch 'develop' into consensus-cache
dshulyak Sep 20, 2023
bfda8d9
proposals back to 100
dshulyak Sep 20, 2023
2ed1e21
finish basic coverage
dshulyak Sep 21, 2023
fd8b5aa
reuse lru cache to set malicious
dshulyak Sep 21, 2023
94d20be
Merge branch 'develop' into consensus-cache
dshulyak Sep 21, 2023
d976e04
resolve conflicts
dshulyak Sep 21, 2023
b9f59b2
Merge branch 'develop' into consensus-cache
dshulyak Sep 21, 2023
14338fe
add relevant benchmark and memory estimate
dshulyak Sep 21, 2023
5cc11d6
fix test in miner
dshulyak Sep 21, 2023
ee6a98b
add metrics to track atxs/identities in cache
dshulyak Sep 21, 2023
7f47c0f
change default capacity to 2
dshulyak Sep 21, 2023
2b0410f
Merge branch 'develop' into consensus-cache
dshulyak Sep 21, 2023
5a70266
memory assertion
dshulyak Sep 21, 2023
4c790c3
regen with 0.3.0
dshulyak Sep 21, 2023
2a9830c
Merge branch 'develop' into consensus-cache
dshulyak Sep 25, 2023
d6f896b
fix some mistakes
dshulyak Sep 27, 2023
31f226f
remove cachedb from handler
dshulyak Sep 27, 2023
c0f7a20
Merge branch 'develop' into consensus-cache
dshulyak Oct 19, 2023
e887d07
refactor to use btree
dshulyak Oct 19, 2023
b510321
switch to pointers
dshulyak Oct 19, 2023
fcf9e2e
add weight for set routine
dshulyak Oct 19, 2023
ba31735
add benchmark
dshulyak Oct 20, 2023
dbfee1c
use binary search instead of map
dshulyak Oct 20, 2023
5eefe50
integrate in proposals
dshulyak Oct 20, 2023
7782971
Merge branch 'develop' into consensus-cache
dshulyak Oct 20, 2023
edd6406
btree dep
dshulyak Oct 20, 2023
8990e36
add with capacity from layers
dshulyak Oct 22, 2023
e6d1ca4
configure capacity based on tortoise window
dshulyak Oct 22, 2023
de00596
drop slowpath code
dshulyak Oct 22, 2023
4696054
drop todo
dshulyak Oct 22, 2023
184b9e3
rename cache to atxsdata
dshulyak Oct 22, 2023
36a02f8
cleanup
dshulyak Oct 22, 2023
1d3d3ea
Merge branch 'develop' into consensus-cache
dshulyak Oct 22, 2023
6a65672
add hashmap for malicious
dshulyak Oct 22, 2023
7986ca5
refactor to store index by atxid
dshulyak Oct 22, 2023
c62029e
drop binary search bench
dshulyak Oct 22, 2023
15644cf
fix test
dshulyak Oct 23, 2023
9959c38
cleanup locking
dshulyak Oct 24, 2023
85e48c8
Merge branch 'develop' into consensus-cache
dshulyak Oct 24, 2023
fbf6667
review
dshulyak Oct 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 57 additions & 6 deletions activation/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"github.com/spacemeshos/post/shared"
"golang.org/x/exp/maps"

"github.com/spacemeshos/go-spacemesh/atxsdata"
"github.com/spacemeshos/go-spacemesh/codec"
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/datastore"
Expand All @@ -35,6 +36,7 @@
type Handler struct {
local p2p.Peer
cdb *datastore.CachedDB
atxsdata *atxsdata.Data
edVerifier *signing.EdVerifier
clock layerClock
publisher pubsub.Publisher
Expand All @@ -53,6 +55,7 @@
func NewHandler(
local p2p.Peer,
cdb *datastore.CachedDB,
atxsdata *atxsdata.Data,
edVerifier *signing.EdVerifier,
c layerClock,
pub pubsub.Publisher,
Expand All @@ -68,6 +71,7 @@
return &Handler{
local: local,
cdb: cdb,
atxsdata: atxsdata,
edVerifier: edVerifier,
clock: c,
publisher: pub,
Expand Down Expand Up @@ -178,7 +182,10 @@
return nil
}

func (h *Handler) SyntacticallyValidateDeps(ctx context.Context, atx *types.ActivationTx) (*types.VerifiedActivationTx, error) {
func (h *Handler) SyntacticallyValidateDeps(
ctx context.Context,
atx *types.ActivationTx,
) (*types.VerifiedActivationTx, error) {
var (
commitmentATX *types.ATXID
err error
Expand Down Expand Up @@ -209,9 +216,18 @@
}

expectedChallengeHash := atx.NIPostChallenge.Hash()
h.log.WithContext(ctx).With().Info("validating nipost", log.String("expected_challenge_hash", expectedChallengeHash.String()), atx.ID())

leaves, err := h.nipostValidator.NIPost(ctx, atx.SmesherID, *commitmentATX, atx.NIPost, expectedChallengeHash, atx.NumUnits)
h.log.WithContext(ctx).
With().
Info("validating nipost", log.String("expected_challenge_hash", expectedChallengeHash.String()), atx.ID())

leaves, err := h.nipostValidator.NIPost(
ctx,
atx.SmesherID,
*commitmentATX,
atx.NIPost,
expectedChallengeHash,
atx.NumUnits,
)
if err != nil {
return nil, fmt.Errorf("invalid nipost: %w", err)
}
Expand Down Expand Up @@ -289,7 +305,11 @@

if err == nil && atx.PrevATXID == types.EmptyATXID {
// no previous atx declared, but already seen at least one atx from node
return fmt.Errorf("no prev atx reported, but other atx with same node id (%v) found: %v", atx.SmesherID, lastAtx.ShortString())
return fmt.Errorf(
"no prev atx reported, but other atx with same node id (%v) found: %v",
atx.SmesherID,
lastAtx.ShortString(),
)

Check warning on line 312 in activation/handler.go

View check run for this annotation

Codecov / codecov/patch

activation/handler.go#L308-L312

Added lines #L308 - L312 were not covered by tests
}

if err == nil && atx.PrevATXID != lastAtx {
Expand All @@ -315,6 +335,31 @@
return err
}

func (h *Handler) cacheAtx(ctx context.Context, atx *types.ActivationTxHeader) {
if !h.atxsdata.IsEvicted(atx.TargetEpoch()) {
nonce, err := h.cdb.VRFNonce(atx.NodeID, atx.TargetEpoch())
if err != nil {
h.log.With().Error("failed vrf nonce read", log.Err(err), log.Context(ctx))
return
}
malicious, err := h.cdb.IsMalicious(atx.NodeID)
if err != nil {
h.log.With().Error("failed is malicious read", log.Err(err), log.Context(ctx))
return
}

Check warning on line 349 in activation/handler.go

View check run for this annotation

Codecov / codecov/patch

activation/handler.go#L347-L349

Added lines #L347 - L349 were not covered by tests
h.atxsdata.Add(
atx.TargetEpoch(),
atx.NodeID,
atx.ID,
atx.GetWeight(),
atx.BaseTickHeight,
atx.TickHeight(),
nonce,
malicious,
)
}
}

// storeAtx stores an ATX and notifies subscribers of the ATXID.
func (h *Handler) storeAtx(ctx context.Context, atx *types.VerifiedActivationTx) error {
malicious, err := h.cdb.IsMalicious(atx.SmesherID)
Expand Down Expand Up @@ -380,6 +425,7 @@
}
h.beacon.OnAtx(header)
h.tortoise.OnAtx(header.ToData())
h.cacheAtx(ctx, header)

h.log.WithContext(ctx).With().Debug("finished storing atx in epoch", atx.ID(), atx.PublishEpoch)

Expand Down Expand Up @@ -488,7 +534,12 @@
}

if expHash != (types.Hash32{}) && vAtx.ID().Hash32() != expHash {
return fmt.Errorf("%w: atx want %s, got %s", errWrongHash, expHash.ShortString(), vAtx.ID().Hash32().ShortString())
return fmt.Errorf(
"%w: atx want %s, got %s",
errWrongHash,
expHash.ShortString(),
vAtx.ID().Hash32().ShortString(),
)
}

if err := h.ProcessAtx(ctx, vAtx); err != nil {
Expand Down
Loading
Loading