From ce1d45c045e814550d118c11402910a2310b4080 Mon Sep 17 00:00:00 2001 From: Hao Hao Date: Fri, 1 Sep 2023 22:00:14 -0700 Subject: [PATCH] chore: remove eth acc root and blob in db --- firewood/src/config.rs | 3 - firewood/src/db.rs | 118 +++--------------------------------- firewood/src/db/proposal.rs | 60 ++---------------- fwdctl/src/create.rs | 1 - 4 files changed, 12 insertions(+), 170 deletions(-) diff --git a/firewood/src/config.rs b/firewood/src/config.rs index 90a114d9c..f4bfcf658 100644 --- a/firewood/src/config.rs +++ b/firewood/src/config.rs @@ -63,7 +63,4 @@ pub struct DbRevConfig { /// Maximum cached Trie objects. #[builder(default = 1 << 20)] pub merkle_ncached_objs: usize, - /// Maximum cached Blob (currently just `Account`) objects. - #[builder(default = 4096)] - pub blob_ncached_objs: usize, } diff --git a/firewood/src/db.rs b/firewood/src/db.rs index 7bc2b3967..8070602af 100644 --- a/firewood/src/db.rs +++ b/firewood/src/db.rs @@ -45,9 +45,7 @@ use self::proposal::ProposalBase; const MERKLE_META_SPACE: SpaceId = 0x0; const MERKLE_PAYLOAD_SPACE: SpaceId = 0x1; -const BLOB_META_SPACE: SpaceId = 0x2; -const BLOB_PAYLOAD_SPACE: SpaceId = 0x3; -const ROOT_HASH_SPACE: SpaceId = 0x4; +const ROOT_HASH_SPACE: SpaceId = 0x2; const SPACE_RESERVED: u64 = 0x1000; const MAGIC_STR: &[u8; 16] = b"firewood v0.1\0\0\0"; @@ -185,19 +183,15 @@ fn get_sub_universe_from_empty_delta( /// DB-wide metadata, it keeps track of the roots of the top-level tries. #[derive(Debug)] struct DbHeader { - /// The root node of the account model storage. (Where the values are [Account] objects, which - /// may contain the root for the secondary trie.) - acc_root: DiskAddress, /// The root node of the generic key-value store. kv_root: DiskAddress, } impl DbHeader { - pub const MSIZE: u64 = 16; + pub const MSIZE: u64 = 8; pub fn new_empty() -> Self { Self { - acc_root: DiskAddress::null(), kv_root: DiskAddress::null(), } } @@ -212,8 +206,7 @@ impl Storable for DbHeader { size: Self::MSIZE, })?; Ok(Self { - acc_root: raw.as_deref()[..8].into(), - kv_root: raw.as_deref()[8..].into(), + kv_root: raw.as_deref()[..8].into(), }) } @@ -223,7 +216,6 @@ impl Storable for DbHeader { fn dehydrate(&self, to: &mut [u8]) -> Result<(), ShaleError> { let mut cur = Cursor::new(to); - cur.write_all(&self.acc_root.to_le_bytes())?; cur.write_all(&self.kv_root.to_le_bytes())?; Ok(()) } @@ -233,14 +225,12 @@ impl Storable for DbHeader { /// Necessary linear space instances bundled for the state of the entire DB. struct Universe { merkle: SubUniverse, - blob: SubUniverse, } impl Universe { fn to_mem_store_r(&self) -> Universe> { Universe { merkle: self.merkle.to_mem_store_r(), - blob: self.blob.to_mem_store_r(), } } } @@ -249,7 +239,6 @@ impl Universe> { fn new_from_other(&self) -> Universe> { Universe { merkle: self.merkle.new_from_other(), - blob: self.blob.new_from_other(), } } } @@ -258,7 +247,6 @@ impl Universe> { fn to_mem_store_r(&self) -> Universe> { Universe { merkle: self.merkle.to_mem_store_r(), - blob: self.blob.to_mem_store_r(), } } } @@ -268,14 +256,11 @@ impl Universe> { &self, merkle_meta_writes: &[SpaceWrite], merkle_payload_writes: &[SpaceWrite], - blob_meta_writes: &[SpaceWrite], - blob_payload_writes: &[SpaceWrite], ) -> Universe { Universe { merkle: self .merkle .rewind(merkle_meta_writes, merkle_payload_writes), - blob: self.blob.rewind(blob_meta_writes, blob_payload_writes), } } } @@ -338,14 +323,6 @@ impl + Send + Sync> DbRev { let valid = proof.verify_range_proof(hash, first_key, last_key, keys, values)?; Ok(valid) } - - /// Check if the account exists. - pub fn exist>(&self, key: K) -> Result { - Ok(match self.merkle.get(key, self.header.acc_root) { - Ok(r) => r.is_some(), - Err(e) => return Err(DbError::Merkle(e)), - }) - } } #[derive(Debug)] @@ -407,10 +384,6 @@ impl Db { let merkle_meta_path = file::touch_dir("meta", &merkle_path)?; let merkle_payload_path = file::touch_dir("compact", &merkle_path)?; - let blob_path = file::touch_dir("blob", &db_path)?; - let blob_meta_path = file::touch_dir("meta", &blob_path)?; - let blob_payload_path = file::touch_dir("compact", &blob_path)?; - let root_hash_path = file::touch_dir("root_hash", &db_path)?; let file0 = crate::file::File::new(0, SPACE_RESERVED, &merkle_meta_path)?; @@ -509,41 +482,11 @@ impl Db { .unwrap(), ), ), - blob: SubUniverse::new( - Arc::new( - CachedSpace::new( - &StoreConfig::builder() - .ncached_pages(cfg.meta_ncached_pages) - .ncached_files(cfg.meta_ncached_files) - .space_id(BLOB_META_SPACE) - .file_nbit(params.meta_file_nbit) - .rootdir(blob_meta_path) - .build(), - disk_requester.clone(), - ) - .unwrap(), - ), - Arc::new( - CachedSpace::new( - &StoreConfig::builder() - .ncached_pages(cfg.payload_ncached_pages) - .ncached_files(cfg.payload_ncached_files) - .space_id(BLOB_PAYLOAD_SPACE) - .file_nbit(params.payload_file_nbit) - .rootdir(blob_payload_path) - .build(), - disk_requester.clone(), - ) - .unwrap(), - ), - ), }; [ data_cache.merkle.meta.as_ref(), data_cache.merkle.payload.as_ref(), - data_cache.blob.meta.as_ref(), - data_cache.blob.payload.as_ref(), root_hash_cache.as_ref(), ] .into_iter() @@ -559,7 +502,6 @@ impl Db { let base = Universe { merkle: get_sub_universe_from_empty_delta(&data_cache.merkle), - blob: get_sub_universe_from_empty_delta(&data_cache.blob), }; let db_header_ref = Db::get_db_header_ref(&base.merkle.meta)?; @@ -567,18 +509,11 @@ impl Db { let merkle_payload_header_ref = Db::get_payload_header_ref(&base.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE)?; - let blob_payload_header_ref = Db::get_payload_header_ref(&base.blob.meta, DbHeader::MSIZE)?; - - let header_refs = ( - db_header_ref, - merkle_payload_header_ref, - blob_payload_header_ref, - ); + let header_refs = (db_header_ref, merkle_payload_header_ref); let base_revision = Db::new_revision( header_refs, (base.merkle.meta.clone(), base.merkle.payload.clone()), - (base.blob.meta.clone(), base.blob.payload.clone()), params.payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -618,10 +553,8 @@ impl Db { let merkle_payload_header: DiskAddress = DiskAddress::from(offset); offset += CompactSpaceHeader::MSIZE as usize; assert!(offset <= SPACE_RESERVED as usize); - let blob_payload_header: DiskAddress = DiskAddress::null(); let mut merkle_meta_store = StoreRevMut::new(cached_space.merkle.meta.clone()); - let mut blob_meta_store = StoreRevMut::new(cached_space.blob.meta.clone()); if reset_store_headers { // initialize store headers @@ -636,13 +569,6 @@ impl Db { db_header.into(), &shale::to_dehydrated(&DbHeader::new_empty())?, ); - blob_meta_store.write( - blob_payload_header.into(), - &shale::to_dehydrated(&shale::compact::CompactSpaceHeader::new( - NonZeroUsize::new(SPACE_RESERVED as usize).unwrap(), - NonZeroUsize::new(SPACE_RESERVED as usize).unwrap(), - ))?, - ); } let store = Universe { @@ -650,10 +576,6 @@ impl Db { Arc::new(merkle_meta_store), Arc::new(StoreRevMut::new(cached_space.merkle.payload.clone())), ), - blob: SubUniverse::new( - Arc::new(blob_meta_store), - Arc::new(StoreRevMut::new(cached_space.blob.payload.clone())), - ), }; let db_header_ref = Db::get_db_header_ref(store.merkle.meta.as_ref())?; @@ -663,18 +585,11 @@ impl Db { Db::PARAM_SIZE + DbHeader::MSIZE, )?; - let blob_payload_header_ref = Db::get_payload_header_ref(store.blob.meta.as_ref(), 0)?; - - let header_refs = ( - db_header_ref, - merkle_payload_header_ref, - blob_payload_header_ref, - ); + let header_refs = (db_header_ref, merkle_payload_header_ref); let mut rev: DbRev> = Db::new_revision( header_refs, (store.merkle.meta.clone(), store.merkle.payload.clone()), - (store.blob.meta.clone(), store.blob.payload.clone()), payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -703,13 +618,8 @@ impl Db { } fn new_revision>>( - header_refs: ( - Obj, - Obj, - Obj, - ), + header_refs: (Obj, Obj), merkle: (T, T), - _blob: (T, T), payload_regn_nbit: u64, payload_max_walk: u64, cfg: &DbRevConfig, @@ -737,13 +647,12 @@ impl Db { let merkle = Merkle::new(Box::new(merkle_space)); - if db_header_ref.acc_root.is_null() { + if db_header_ref.kv_root.is_null() { let mut err = Ok(()); // create the sentinel node db_header_ref .write(|r| { err = (|| { - r.acc_root = merkle.init_root()?; r.kv_root = merkle.init_root()?; Ok(()) })(); @@ -863,14 +772,10 @@ impl Db { Some(u) => u.to_mem_store_r().rewind( &ash.0[&MERKLE_META_SPACE].undo, &ash.0[&MERKLE_PAYLOAD_SPACE].undo, - &ash.0[&BLOB_META_SPACE].undo, - &ash.0[&BLOB_PAYLOAD_SPACE].undo, ), None => inner_lock.cached_space.to_mem_store_r().rewind( &ash.0[&MERKLE_META_SPACE].undo, &ash.0[&MERKLE_PAYLOAD_SPACE].undo, - &ash.0[&BLOB_META_SPACE].undo, - &ash.0[&BLOB_PAYLOAD_SPACE].undo, ), }; revisions.inner.push_back(u); @@ -891,19 +796,12 @@ impl Db { Db::get_payload_header_ref(&space.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE) .unwrap(); - let blob_payload_header_ref = Db::get_payload_header_ref(&space.blob.meta, 0).unwrap(); - - let header_refs = ( - db_header_ref, - merkle_payload_header_ref, - blob_payload_header_ref, - ); + let header_refs = (db_header_ref, merkle_payload_header_ref); Revision { rev: Db::new_revision( header_refs, (space.merkle.meta.clone(), space.merkle.payload.clone()), - (space.blob.meta.clone(), space.blob.payload.clone()), self.payload_regn_nbit, 0, &self.cfg.rev, diff --git a/firewood/src/db/proposal.rs b/firewood/src/db/proposal.rs index 984e91c84..debece604 100644 --- a/firewood/src/db/proposal.rs +++ b/firewood/src/db/proposal.rs @@ -1,7 +1,7 @@ use super::{ get_sub_universe_from_deltas, get_sub_universe_from_empty_delta, Db, DbConfig, DbError, - DbHeader, DbInner, DbRev, DbRevInner, SharedStore, Store, Universe, BLOB_META_SPACE, - BLOB_PAYLOAD_SPACE, MERKLE_META_SPACE, MERKLE_PAYLOAD_SPACE, ROOT_HASH_SPACE, + DbHeader, DbInner, DbRev, DbRevInner, SharedStore, Store, Universe, MERKLE_META_SPACE, + MERKLE_PAYLOAD_SPACE, ROOT_HASH_SPACE, }; use crate::{ merkle::{TrieHash, TRIE_HASH_LEN}, @@ -64,18 +64,11 @@ impl Proposal { Db::PARAM_SIZE + DbHeader::MSIZE, )?; - let blob_payload_header_ref = Db::get_payload_header_ref(store.blob.meta.as_ref(), 0)?; - - let header_refs = ( - db_header_ref, - merkle_payload_header_ref, - blob_payload_header_ref, - ); + let header_refs = (db_header_ref, merkle_payload_header_ref); let mut rev = Db::new_revision( header_refs, (store.merkle.meta.clone(), store.merkle.payload.clone()), - (store.blob.meta.clone(), store.blob.payload.clone()), cfg.payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -155,8 +148,6 @@ impl Proposal { // clear the staging layer and apply changes to the CachedSpace let (merkle_payload_redo, merkle_payload_wal) = self.store.merkle.payload.delta(); let (merkle_meta_redo, merkle_meta_wal) = self.store.merkle.meta.delta(); - let (blob_payload_redo, blob_payload_wal) = self.store.blob.payload.delta(); - let (blob_meta_redo, blob_meta_wal) = self.store.blob.meta.delta(); let mut rev_inner = self.m.write(); let merkle_meta_undo = rev_inner @@ -171,18 +162,6 @@ impl Proposal { .payload .update(&merkle_payload_redo) .unwrap(); - let blob_meta_undo = rev_inner - .cached_space - .blob - .meta - .update(&blob_meta_redo) - .unwrap(); - let blob_payload_undo = rev_inner - .cached_space - .blob - .payload - .update(&blob_payload_redo) - .unwrap(); // update the rolling window of past revisions let latest_past = Universe { @@ -191,11 +170,6 @@ impl Proposal { merkle_meta_undo, merkle_payload_undo, ), - blob: get_sub_universe_from_deltas( - &rev_inner.cached_space.blob, - blob_meta_undo, - blob_payload_undo, - ), }; let max_revisions = revisions.max_revisions; @@ -206,12 +180,6 @@ impl Proposal { rev.merkle .payload .set_base_space(latest_past.merkle.payload.inner().clone()); - rev.blob - .meta - .set_base_space(latest_past.blob.meta.inner().clone()); - rev.blob - .payload - .set_base_space(latest_past.blob.payload.inner().clone()); } revisions.inner.push_front(latest_past); while revisions.inner.len() > max_revisions { @@ -220,7 +188,6 @@ impl Proposal { let base = Universe { merkle: get_sub_universe_from_empty_delta(&rev_inner.cached_space.merkle), - blob: get_sub_universe_from_empty_delta(&rev_inner.cached_space.blob), }; let db_header_ref = Db::get_db_header_ref(&base.merkle.meta)?; @@ -228,18 +195,11 @@ impl Proposal { let merkle_payload_header_ref = Db::get_payload_header_ref(&base.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE)?; - let blob_payload_header_ref = Db::get_payload_header_ref(&base.blob.meta, 0)?; - - let header_refs = ( - db_header_ref, - merkle_payload_header_ref, - blob_payload_header_ref, - ); + let header_refs = (db_header_ref, merkle_payload_header_ref); let base_revision = Db::new_revision( header_refs, (base.merkle.meta.clone(), base.merkle.payload.clone()), - (base.blob.meta.clone(), base.blob.payload.clone()), 0, self.cfg.payload_max_walk, &self.cfg.rev, @@ -269,14 +229,6 @@ impl Proposal { space_id: self.store.merkle.meta.id(), delta: merkle_meta_redo, }, - BufferWrite { - space_id: self.store.blob.payload.id(), - delta: blob_payload_redo, - }, - BufferWrite { - space_id: self.store.blob.meta.id(), - delta: blob_meta_redo, - }, BufferWrite { space_id: rev_inner.root_hash_staging.id(), delta: root_hash_redo, @@ -286,8 +238,6 @@ impl Proposal { [ (MERKLE_META_SPACE, merkle_meta_wal), (MERKLE_PAYLOAD_SPACE, merkle_payload_wal), - (BLOB_META_SPACE, blob_meta_wal), - (BLOB_PAYLOAD_SPACE, blob_payload_wal), (ROOT_HASH_SPACE, root_hash_wal), ] .into(), @@ -310,8 +260,6 @@ impl Drop for Proposal { // drop the staging changes self.store.merkle.payload.reset_deltas(); self.store.merkle.meta.reset_deltas(); - self.store.blob.payload.reset_deltas(); - self.store.blob.meta.reset_deltas(); self.m.read().root_hash_staging.reset_deltas(); } } diff --git a/fwdctl/src/create.rs b/fwdctl/src/create.rs index 27612515a..1ae83c3b7 100644 --- a/fwdctl/src/create.rs +++ b/fwdctl/src/create.rs @@ -269,7 +269,6 @@ pub fn initialize_db_config(opts: &Options) -> DbConfig { truncate: opts.truncate, rev: DbRevConfig { merkle_ncached_objs: opts.merkle_ncached_objs, - blob_ncached_objs: opts.blob_ncached_objs, }, buffer: DiskBufferConfig { max_buffered: opts.max_buffered,