when packing ancient append vecs, organize access to index

accounts-db/src/accounts_db.rs

@@ -3750,6 +3750,7 @@ impl AccountsDb {
  accounts: &'a [StoredAccountMeta<'a>],
  stats: &ShrinkStats,
  slot_to_shrink: Slot,
+ bin: Option<usize>,
  ) -> LoadAccountsIndexForShrink<'a, T> {
  let count = accounts.len();
  let mut alive_accounts = T::with_capacity(count, slot_to_shrink);
@@ -3760,8 +3761,16 @@ impl AccountsDb {
  let mut index = 0;
  let mut all_are_zero_lamports = true;
  let mut index_entries_being_shrunk = Vec::with_capacity(accounts.len());
+ let bin_calc = &self.accounts_index.bin_calculator;
  self.accounts_index.scan(
- accounts.iter().map(|account| account.pubkey()),
+ accounts.iter().filter_map(|account| {
+ let pk = account.pubkey();
+ if let Some(bin) = bin {
+ (bin_calc.bin_from_pubkey(pk) == bin).then_some(pk)
+ } else {
+ Some(pk)
+ }
+ }),
  |pubkey, slots_refs, entry| {
  let mut result = AccountsIndexScanResult::OnlyKeepInMemoryIfDirty;
  if let Some((slot_list, ref_count)) = slots_refs {
@@ -3847,6 +3856,7 @@ impl AccountsDb {
  store: &'a Arc<AccountStorageEntry>,
  unique_accounts: &'b GetUniqueAccountsResult<'b>,
  stats: &ShrinkStats,
+ bin: Option<usize>,
  ) -> ShrinkCollect<'b, T> {
  let slot = store.slot();
 
@@ -3874,7 +3884,7 @@ impl AccountsDb {
  mut unrefed_pubkeys,
  all_are_zero_lamports,
  mut index_entries_being_shrunk,
- } = self.load_accounts_index_for_shrink(stored_accounts, stats, slot);
+ } = self.load_accounts_index_for_shrink(stored_accounts, stats, slot, bin);
 
  // collect
  alive_accounts_collect
@@ -3983,8 +3993,12 @@ impl AccountsDb {
  let unique_accounts =
  self.get_unique_accounts_from_storage_for_shrink(store, &self.shrink_stats);
  debug!("do_shrink_slot_store: slot: {}", slot);
- let shrink_collect =
- self.shrink_collect::<AliveAccounts<'_>>(store, &unique_accounts, &self.shrink_stats);
+ let shrink_collect = self.shrink_collect::<AliveAccounts<'_>>(
+ store,
+ &unique_accounts,
+ &self.shrink_stats,
+ None,
+ );
 
  // This shouldn't happen if alive_bytes/approx_stored_count are accurate
  if Self::should_not_shrink(
@@ -4539,6 +4553,7 @@ impl AccountsDb {
  old_storage,
  &unique_accounts,
  &self.shrink_ancient_stats.shrink_stats,
+ None,
  );
 
  // could follow what shrink does more closely

accounts-db/src/ancient_append_vecs.rs

@@ -612,16 +612,22 @@ impl AccountsDb {
  // `shrink_collect` all accounts in the append vecs we want to combine.
  // This also unrefs all dead accounts in those append vecs.
  let mut accounts_to_combine = self.thread_pool_clean.install(|| {
- accounts_per_storage
- .par_iter()
- .map(|(info, unique_accounts)| {
- self.shrink_collect::<ShrinkCollectAliveSeparatedByRefs<'_>>(
- &info.storage,
- unique_accounts,
- &self.shrink_ancient_stats.shrink_stats,
- )
- })
- .collect::<Vec<_>>()
+ let mut result = Vec::default();
+ for bin in 0..self.accounts_index.bins() {
+ let mut this_bin = accounts_per_storage
+ .par_iter()
+ .map(|(info, unique_accounts)| {
+ self.shrink_collect::<ShrinkCollectAliveSeparatedByRefs<'_>>(
+ &info.storage,
+ unique_accounts,
+ &self.shrink_ancient_stats.shrink_stats,
+ Some(bin),
+ )
+ })
+ .collect::<Vec<_>>();
+ result.append(&mut this_bin);
+ }
+ result
  });
 
  let mut remove = Vec::default();