diff --git a/sstable/colblk/data_block.go b/sstable/colblk/data_block.go index 5e8ef7c87a..803dd76c6f 100644 --- a/sstable/colblk/data_block.go +++ b/sstable/colblk/data_block.go @@ -1009,13 +1009,9 @@ func (i *DataBlockIter) IsDataInvalidated() bool { // ResetForReuse resets the iterator for reuse, retaining buffers and // configuration supplied to InitOnce, to avoid future allocations. -func (i *DataBlockIter) ResetForReuse() DataBlockIter { - return DataBlockIter{ - keySchema: i.keySchema, - getLazyValuer: i.getLazyValuer, - keyIter: PrefixBytesIter{Buf: i.keyIter.Buf}, - keySeeker: i.keySeeker, - } +func (i *DataBlockIter) ResetForReuse() { + i.d = nil + i.kv = base.InternalKV{} } // IsLowerBound implements the block.DataBlockIterator interface. diff --git a/sstable/colblk/index_block.go b/sstable/colblk/index_block.go index de46035d5a..adbca5bce6 100644 --- a/sstable/colblk/index_block.go +++ b/sstable/colblk/index_block.go @@ -207,19 +207,15 @@ var _ block.IndexBlockIterator = (*IndexIter)(nil) func (i *IndexIter) InitWithDecoder( compare base.Compare, split base.Split, d *IndexBlockDecoder, transforms block.IterTransforms, ) { - *i = IndexIter{ - compare: compare, - split: split, - d: d, - n: int(d.bd.header.Rows), - row: -1, - h: i.h, - allocDecoder: i.allocDecoder, - keyBuf: i.keyBuf, - syntheticPrefix: transforms.SyntheticPrefix, - syntheticSuffix: transforms.SyntheticSuffix, - noTransforms: !transforms.SyntheticPrefix.IsSet() && !transforms.SyntheticSuffix.IsSet(), - } + i.compare = compare + i.split = split + i.d = d + i.n = int(d.bd.header.Rows) + i.row = -1 + i.syntheticPrefix = transforms.SyntheticPrefix + i.syntheticSuffix = transforms.SyntheticSuffix + i.noTransforms = !transforms.SyntheticPrefix.IsSet() && !transforms.SyntheticSuffix.IsSet() + // Leave h, allocDecoder, keyBuf unchanged. } // Init initializes an iterator from the provided block data slice. @@ -252,11 +248,12 @@ func (i *IndexIter) RowIndex() int { // ResetForReuse resets the IndexIter for reuse, retaining buffers to avoid // future allocations. -func (i *IndexIter) ResetForReuse() IndexIter { +func (i *IndexIter) ResetForReuse() { if invariants.Enabled && i.h != (block.BufferHandle{}) { panic(errors.AssertionFailedf("IndexIter reset for reuse with non-empty handle")) } - return IndexIter{ /* nothing to retain */ } + // Nothing to retain. + *i = IndexIter{} } // Valid returns true if the iterator is currently positioned at a valid block diff --git a/sstable/reader_iter.go b/sstable/reader_iter.go index 131e1bec32..58db713ceb 100644 --- a/sstable/reader_iter.go +++ b/sstable/reader_iter.go @@ -28,7 +28,7 @@ type dataBlockIterator[D any] interface { // ResetForReuse resets the data block iterator for reuse, retaining buffers // to avoid future allocations. - ResetForReuse() D + ResetForReuse() *D // non-interface type constraint element } @@ -45,7 +45,7 @@ type indexBlockIterator[I any] interface { // ResetForReuse resets the index iterator for reuse, retaining buffers to // avoid future allocations. - ResetForReuse() I + ResetForReuse() *I // non-interface type constraint element } diff --git a/sstable/reader_iter_single_lvl.go b/sstable/reader_iter_single_lvl.go index 611969b6de..06acf62f29 100644 --- a/sstable/reader_iter_single_lvl.go +++ b/sstable/reader_iter_single_lvl.go @@ -56,10 +56,8 @@ type singleLevelIterator[I any, PI indexBlockIterator[I], D any, PD dataBlockIte // endKeyInclusive is set to force the iterator to treat the upper field as // inclusive while iterating instead of exclusive. endKeyInclusive bool - index I indexFilterRH objstorage.ReadHandle indexFilterRHPrealloc objstorageprovider.PreallocatedReadHandle - data D dataRH objstorage.ReadHandle dataRHPrealloc objstorageprovider.PreallocatedReadHandle // dataBH refers to the last data block that the iterator considered @@ -172,6 +170,11 @@ type singleLevelIterator[I any, PI indexBlockIterator[I], D any, PD dataBlockIte transforms IterTransforms + // All fields above this field are cleared when resetting the iterator for reuse. + clearForResetBoundary struct{} + + index I + data D // inPool is set to true before putting the iterator in the reusable pool; // used to detect double-close. inPool bool @@ -183,6 +186,9 @@ type singleLevelIterator[I any, PI indexBlockIterator[I], D any, PD dataBlockIte // If the iterator is embedded within a twoLevelIterator, pool is nil and // the twoLevelIterator.pool field may be non-nil. pool *sync.Pool + + // NOTE: any new fields should be added above the clearForResetBoundary field, + // unless they need to be retained when resetting the iterator. } // singleLevelIterator implements the base.InternalIterator interface. @@ -384,13 +390,18 @@ func (i *singleLevelIterator[I, PI, D, PD]) SetupForCompaction() { } } -func (i *singleLevelIterator[I, PI, D, PD]) resetForReuse() singleLevelIterator[I, PI, D, PD] { - return singleLevelIterator[I, PI, D, PD]{ - index: PI(&i.index).ResetForReuse(), - data: PD(&i.data).ResetForReuse(), - pool: i.pool, - inPool: true, - } +const clearLen = unsafe.Offsetof(singleLevelIteratorRowBlocks{}.clearForResetBoundary) + +// Assert that clearLen is consistent betwen the row and columnar implementations. +const clearLenColBlocks = unsafe.Offsetof(singleLevelIteratorColumnBlocks{}.clearForResetBoundary) +const _ uintptr = clearLen - clearLenColBlocks +const _ uintptr = clearLenColBlocks - clearLen + +func (i *singleLevelIterator[I, PI, D, PD]) resetForReuse() { + *(*[clearLen]byte)(unsafe.Pointer(i)) = [clearLen]byte{} + PI(&i.index).ResetForReuse() + PD(&i.data).ResetForReuse() + i.inPool = true } func (i *singleLevelIterator[I, PI, D, PD]) initBounds() { @@ -1565,7 +1576,7 @@ func firstError(err0, err1 error) error { func (i *singleLevelIterator[I, PI, D, PD]) Close() error { err := i.closeInternal() pool := i.pool - *i = i.resetForReuse() + i.resetForReuse() if pool != nil { pool.Put(i) } diff --git a/sstable/reader_iter_two_lvl.go b/sstable/reader_iter_two_lvl.go index 4e0b11ea34..a88ff6c5b6 100644 --- a/sstable/reader_iter_two_lvl.go +++ b/sstable/reader_iter_two_lvl.go @@ -1045,12 +1045,11 @@ func (i *twoLevelIterator[I, PI, D, PD]) Close() error { } pool := i.pool err := i.secondLevel.closeInternal() + i.secondLevel.resetForReuse() err = firstError(err, PI(&i.topLevelIndex).Close()) - *i = twoLevelIterator[I, PI, D, PD]{ - secondLevel: i.secondLevel.resetForReuse(), - topLevelIndex: PI(&i.topLevelIndex).ResetForReuse(), - pool: pool, - } + PI(&i.topLevelIndex).ResetForReuse() + i.useFilterBlock = false + i.lastBloomFilterMatched = false if pool != nil { pool.Put(i) } diff --git a/sstable/rowblk/rowblk_fragment_iter.go b/sstable/rowblk/rowblk_fragment_iter.go index 4aeb0a6ffb..3f47c5a89b 100644 --- a/sstable/rowblk/rowblk_fragment_iter.go +++ b/sstable/rowblk/rowblk_fragment_iter.go @@ -286,8 +286,9 @@ func (i *fragmentIter) Close() { return } + i.blockIter.ResetForReuse() *i = fragmentIter{ - blockIter: i.blockIter.ResetForReuse(), + blockIter: i.blockIter, closeCheck: i.closeCheck, startKeyBuf: i.startKeyBuf[:0], endKeyBuf: i.endKeyBuf[:0], diff --git a/sstable/rowblk/rowblk_index_iter.go b/sstable/rowblk/rowblk_index_iter.go index 87e59736bb..b02ae5991e 100644 --- a/sstable/rowblk/rowblk_index_iter.go +++ b/sstable/rowblk/rowblk_index_iter.go @@ -34,8 +34,8 @@ func (i *IndexIter) InitHandle( // ResetForReuse resets the index iterator for reuse, retaining buffers to avoid // future allocations. -func (i *IndexIter) ResetForReuse() IndexIter { - return IndexIter{iter: i.iter.ResetForReuse()} +func (i *IndexIter) ResetForReuse() { + i.iter.ResetForReuse() } // Valid returns true if the iterator is currently positioned at a valid block diff --git a/sstable/rowblk/rowblk_iter.go b/sstable/rowblk/rowblk_iter.go index e8d4b0a0aa..11a49410fa 100644 --- a/sstable/rowblk/rowblk_iter.go +++ b/sstable/rowblk/rowblk_iter.go @@ -303,13 +303,16 @@ func (i *Iter) IsDataInvalidated() bool { // ResetForReuse resets the blockIter for reuse, retaining buffers to avoid // future allocations. -func (i *Iter) ResetForReuse() Iter { - return Iter{ - fullKey: i.fullKey[:0], - cached: i.cached[:0], - cachedBuf: i.cachedBuf[:0], - firstUserKeyWithPrefixBuf: i.firstUserKeyWithPrefixBuf[:0], - data: nil, +func (i *Iter) ResetForReuse() { + fullKey := i.fullKey[:0] + cached := i.cached[:0] + cachedBuf := i.cachedBuf[:0] + firstUserKeyWithPrefixBuf := i.firstUserKeyWithPrefixBuf[:0] + *i = Iter{ + fullKey: fullKey, + cached: cached, + cachedBuf: cachedBuf, + firstUserKeyWithPrefixBuf: firstUserKeyWithPrefixBuf, } } diff --git a/sstable/rowblk/rowblk_rewrite.go b/sstable/rowblk/rowblk_rewrite.go index ed370b0cf8..78f1b1481c 100644 --- a/sstable/rowblk/rowblk_rewrite.go +++ b/sstable/rowblk/rowblk_rewrite.go @@ -89,6 +89,6 @@ func (r *Rewriter) RewriteSuffixes( end.Trailer = r.scratchKey.Trailer r.keyAlloc, end.UserKey = r.keyAlloc.Copy(r.scratchKey.UserKey) - r.iter = r.iter.ResetForReuse() + r.iter.ResetForReuse() return start, end, r.writer.Finish(), nil }