Move shale into firewood as a module (#338)

.github/check-license-headers.yaml

@@ -10,7 +10,7 @@
       "RELEASE.md",
       "rpc/**",
       "README*",
-      "*/README*",
+      "**/README*",
       "Cargo.toml",
       "*/Cargo.toml",
       "libaio/**",
@@ -27,7 +27,7 @@
       "RELEASE.md",
       "rpc/**",
       "README*",
-      "*/README*",
+      "**/README*",
       "Cargo.toml",
       "*/Cargo.toml",
       "libaio/**",

Cargo.toml

@@ -5,7 +5,6 @@ members = [
   "growth-ring",
   "libaio",
   "rpc",
-  "shale",
 ]
 resolver = "2"
 

firewood/Cargo.toml

@@ -22,7 +22,6 @@ bytemuck = { version = "1.13.1", features = ["derive"] }
 enum-as-inner = "0.6.0"
 growth-ring = { version = "0.0.4", path = "../growth-ring" }
 libaio = {version = "0.0.4", path = "../libaio" }
-shale = { version = "0.0.4", path = "../shale" }
 futures = "0.3.24"
 hex = "0.4.3"
 lru = "0.12.0"

firewood/benches/hashops.rs

@@ -7,17 +7,17 @@ use criterion::{criterion_group, criterion_main, profiler::Profiler, BatchSize,
 use firewood::{
     db::{BatchOp, DbConfig},
     merkle::{Merkle, TrieHash, TRIE_HASH_LEN},
+    shale::{
+        cached::PlainMem,
+        compact::{CompactHeader, CompactSpace},
+        disk_address::DiskAddress,
+        CachedStore, ObjCache, Storable, StoredView,
+    },
     storage::WalConfig,
     v2::api::{Db, Proposal},
 };
 use pprof::ProfilerGuard;
 use rand::{distributions::Alphanumeric, rngs::StdRng, Rng, SeedableRng};
-use shale::{
-    cached::PlainMem,
-    compact::{CompactHeader, CompactSpace},
-    disk_address::DiskAddress,
-    CachedStore, ObjCache, Storable, StoredView,
-};
 use std::{fs::File, iter::repeat_with, ops::Deref, os::raw::c_int, path::Path, sync::Arc};
 
 const ZERO_HASH: TrieHash = TrieHash([0u8; TRIE_HASH_LEN]);

shale/benches/shale-bench.rs → firewood/benches/shale-bench.rs

@@ -4,14 +4,14 @@
 use criterion::{
     black_box, criterion_group, criterion_main, profiler::Profiler, Bencher, Criterion,
 };
-use pprof::ProfilerGuard;
-use rand::Rng;
-use shale::{
+use firewood::shale::{
     cached::{DynamicMem, PlainMem},
     compact::{CompactHeader, CompactSpaceHeader},
     disk_address::DiskAddress,
     CachedStore, Obj, StoredView,
 };
+use pprof::ProfilerGuard;
+use rand::Rng;
 use std::{fs::File, os::raw::c_int, path::Path};
 
 const BENCH_MEM_SIZE: usize = 2_000_000;

firewood/src/db.rs

@@ -1,6 +1,12 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
+use crate::shale::{
+    self,
+    compact::{CompactSpace, CompactSpaceHeader},
+    disk_address::DiskAddress,
+    CachedStore, Obj, ShaleError, ShaleStore, SpaceId, Storable, StoredView,
+};
 pub use crate::{
     config::{DbConfig, DbRevConfig},
     storage::{buffer::DiskBufferConfig, WalConfig},
@@ -21,11 +27,6 @@ use async_trait::async_trait;
 use bytemuck::{cast_slice, AnyBitPattern};
 use metered::metered;
 use parking_lot::{Mutex, RwLock};
-use shale::{
-    compact::{CompactSpace, CompactSpaceHeader},
-    disk_address::DiskAddress,
-    CachedStore, Obj, ShaleError, ShaleStore, SpaceId, Storable, StoredView,
-};
 use std::{
     collections::VecDeque,
     error::Error,

firewood/src/db/proposal.rs

@@ -6,14 +6,14 @@ use super::{
     DbHeader, DbInner, DbRev, DbRevInner, SharedStore, Store, Universe, MERKLE_META_SPACE,
     MERKLE_PAYLOAD_SPACE, ROOT_HASH_SPACE,
 };
+use crate::shale::CachedStore;
 use crate::{
     merkle::{TrieHash, TRIE_HASH_LEN},
     storage::{buffer::BufferWrite, AshRecord, StoreRevMut},
     v2::api::{self, KeyType, ValueType},
 };
 use async_trait::async_trait;
 use parking_lot::{Mutex, RwLock};
-use shale::CachedStore;
 use std::{io::ErrorKind, sync::Arc};
 use tokio::task::block_in_place;
 

firewood/src/lib.rs

@@ -193,5 +193,7 @@ pub mod storage;
 
 pub mod config;
 pub mod nibbles;
+// TODO: shale should not be pub, but there are integration test dependencies :(
+pub mod shale;
 
 pub mod v2;

firewood/src/merkle.rs

@@ -1,9 +1,9 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
+use crate::shale::{self, disk_address::DiskAddress, ObjWriteError, ShaleError, ShaleStore};
 use crate::{nibbles::Nibbles, v2::api::Proof};
 use sha3::Digest;
-use shale::{self, disk_address::DiskAddress, ObjWriteError, ShaleError, ShaleStore};
 use std::{
     cmp::Ordering,
     collections::HashMap,

firewood/src/merkle/node.rs

@@ -1,11 +1,11 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
+use crate::shale::{disk_address::DiskAddress, CachedStore, ShaleError, ShaleStore, Storable};
 use bincode::{Error, Options};
 use enum_as_inner::EnumAsInner;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sha3::{Digest, Keccak256};
-use shale::{disk_address::DiskAddress, CachedStore, ShaleError, ShaleStore, Storable};
 use std::{
     fmt::{self, Debug},
     io::{Cursor, Read, Write},
@@ -850,7 +850,7 @@ pub(super) mod tests {
     use std::array::from_fn;
 
     use super::*;
-    use shale::cached::PlainMem;
+    use crate::shale::cached::PlainMem;
     use test_case::test_case;
 
     pub fn leaf(path: Vec<u8>, data: Vec<u8>) -> Node {

firewood/src/merkle/trie_hash.rs

@@ -1,7 +1,7 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
-use shale::{CachedStore, ShaleError, Storable};
+use crate::shale::{CachedStore, ShaleError, Storable};
 use std::{
     fmt::{self, Debug},
     io::Write,

firewood/src/merkle_util.rs

@@ -1,15 +1,15 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
+use crate::shale::{
+    self, cached::DynamicMem, compact::CompactSpace, disk_address::DiskAddress, CachedStore,
+    ShaleStore, StoredView,
+};
 use crate::{
     merkle::{Merkle, Node, Ref, RefMut, TrieHash},
     proof::ProofError,
     v2::api::Proof,
 };
-use shale::{
-    cached::DynamicMem, compact::CompactSpace, disk_address::DiskAddress, CachedStore, ShaleStore,
-    StoredView,
-};
 use std::{num::NonZeroUsize, sync::Arc};
 use thiserror::Error;
 

firewood/src/proof.rs

@@ -4,11 +4,9 @@
 use std::cmp::Ordering;
 use std::ops::Deref;
 
+use crate::shale::{disk_address::DiskAddress, ShaleError, ShaleStore};
 use nix::errno::Errno;
 use sha3::Digest;
-use shale::disk_address::DiskAddress;
-use shale::ShaleError;
-use shale::ShaleStore;
 use thiserror::Error;
 
 use crate::nibbles::Nibbles;

firewood/src/shale/README.md

@@ -0,0 +1,3 @@
+# shale
+
+This directory was forked from [`shale`](https://github.com/Determinant/shale) at commit [`caa6d75`](https://github.com/Determinant/shale/commit/caa6d7543d253e2172c51a65d226d65d232a9b9a), under MIT license.

shale/src/cached.rs → firewood/src/shale/cached.rs

@@ -1,7 +1,7 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
-use crate::{CachedStore, CachedView, SendSyncDerefMut, SpaceId};
+use crate::shale::{CachedStore, CachedView, SendSyncDerefMut, SpaceId};
 use std::{
     borrow::BorrowMut,
     fmt::Debug,
@@ -10,7 +10,7 @@ use std::{
 };
 
 /// Purely volatile, vector-based implementation for [CachedStore]. This is good for testing or trying
-/// out stuff (persistent data structures) built on [ShaleStore](crate::ShaleStore) in memory, without having to write
+/// out stuff (persistent data structures) built on [ShaleStore](super::ShaleStore) in memory, without having to write
 /// your own [CachedStore] implementation.
 #[derive(Debug)]
 pub struct PlainMem {

shale/src/compact.rs → firewood/src/shale/compact.rs

@@ -1,7 +1,7 @@
 // Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE.md for licensing terms.
 
-use crate::ObjCache;
+use crate::shale::ObjCache;
 
 use super::disk_address::DiskAddress;
 use super::{CachedStore, Obj, ObjRef, ShaleError, ShaleStore, Storable, StoredView};
@@ -589,7 +589,7 @@ impl<T: Storable + 'static, M: CachedStore + Send + Sync> ShaleStore<T> for Comp
 mod tests {
     use sha3::Digest;
 
-    use crate::{cached::DynamicMem, ObjCache};
+    use crate::shale::{self, cached::DynamicMem, ObjCache};
 
     use super::*;
 
@@ -648,7 +648,7 @@ mod tests {
         let compact_header = DiskAddress::from(0x1);
         dm.write(
             compact_header.unwrap().get(),
-            &crate::to_dehydrated(&CompactSpaceHeader::new(
+            &shale::to_dehydrated(&CompactSpaceHeader::new(
                 reserved.0.unwrap(),
                 reserved.0.unwrap(),
             ))

shale/src/disk_address.rs → firewood/src/shale/disk_address.rs

@@ -8,7 +8,7 @@ use std::ops::{Deref, DerefMut};
 
 use bytemuck::NoUninit;
 
-use crate::{CachedStore, ShaleError, Storable};
+use crate::shale::{CachedStore, ShaleError, Storable};
 
 /// The virtual disk address of an object
 #[repr(C)]

firewood/src/storage/buffer.rs

@@ -11,6 +11,7 @@ use std::sync::Arc;
 use std::{cell::RefCell, collections::HashMap};
 
 use super::{AshRecord, FilePool, Page, StoreDelta, StoreError, WalConfig, PAGE_SIZE_NBIT};
+use crate::shale::SpaceId;
 use crate::storage::DeltaPage;
 use aiofut::{AioBuilder, AioError, AioManager};
 use futures::future::join_all;
@@ -19,7 +20,6 @@ use growthring::{
     walerror::WalError,
     WalFileImpl, WalStoreImpl,
 };
-use shale::SpaceId;
 use tokio::{
     sync::{
         mpsc,
@@ -621,14 +621,14 @@ mod tests {
     use tokio::task::block_in_place;
 
     use super::*;
+    use crate::shale::CachedStore;
     use crate::{
         file,
         storage::{
             Ash, CachedSpace, DeltaPage, MemStoreR, StoreConfig, StoreRevMut, StoreRevMutDelta,
             StoreRevShared, ZeroStore,
         },
     };
-    use shale::CachedStore;
 
     const STATE_SPACE: SpaceId = 0x0;
     const HASH_SIZE: usize = 32;

firewood/src/storage/mod.rs

@@ -4,10 +4,10 @@
 // TODO: try to get rid of the use `RefCell` in this file
 use self::buffer::DiskBufferRequester;
 use crate::file::File;
+use crate::shale::{self, CachedStore, CachedView, SendSyncDerefMut, SpaceId};
 use nix::fcntl::{flock, FlockArg};
 use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
-use shale::{CachedStore, CachedView, SendSyncDerefMut, SpaceId};
 use std::{
     collections::HashMap,
     fmt::{self, Debug},

firewood/tests/merkle.rs

@@ -5,10 +5,11 @@ use firewood::{
     merkle::Node,
     merkle_util::{new_merkle, DataStoreError, MerkleSetup},
     proof::ProofError,
+    // TODO: we should not be using shale from an integration test
+    shale::{cached::DynamicMem, compact::CompactSpace},
     v2::api::Proof,
 };
 use rand::Rng;
-use shale::{cached::DynamicMem, compact::CompactSpace};
 use std::collections::HashMap;
 
 type Store = CompactSpace<Node, DynamicMem>;