Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove support for storage v7 #2309

Merged
merged 1 commit into from
Feb 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/internet_identity/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ internet_identity_interface.workspace = true
hex.workspace = true
include_dir.workspace = true
lazy_static.workspace = true
serde = { version = "1", features = ["rc"] }
serde.workspace = true
serde_bytes.workspace = true
serde_cbor.workspace = true
serde_json = { version = "1.0", default-features = false, features = ["std"] }
Expand Down
49 changes: 7 additions & 42 deletions src/internet_identity/src/archive.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ use internet_identity_interface::internet_identity::types::*;
use serde_bytes::ByteBuf;
use sha2::Digest;
use sha2::Sha256;
use std::rc::Rc;
use std::time::Duration;
use ArchiveState::{Configured, Created, CreationInProgress, NotConfigured};
use CanisterInstallMode::Upgrade;
Expand Down Expand Up @@ -54,21 +53,11 @@ pub struct ArchiveData {
pub sequence_number: u64,
// Canister id of the archive canister
pub archive_canister: Principal,
// Entries to be fetched by the archive canister sorted in ascending order by sequence_number.
// Once the limit has been reached, II will refuse further changes to anchors in stable memory
// until the archive acknowledges entries and they can safely be deleted from this buffer.
// The limit is configurable (entries_buffer_limit).
// This is an Rc to avoid unnecessary copies of (potentially) a lot of data when cloning.
pub entries_buffer: Rc<Vec<BufferedEntry>>,
}

impl ArchiveData {
pub fn buffered_entries_count(&self) -> usize {
state::storage_borrow(|s| match s.version() {
7 => self.entries_buffer.len(),
8 => s.archive_entries_count(),
_ => trap("unsupported storage version"),
})
state::storage_borrow(|s| s.archive_entries_count())
}
}

Expand Down Expand Up @@ -200,7 +189,6 @@ async fn create_archive(config: ArchiveConfig) -> Result<Principal, String> {
data: ArchiveData {
sequence_number: 0,
archive_canister: canister_id,
entries_buffer: Rc::new(vec![]),
},
config,
}
Expand Down Expand Up @@ -327,8 +315,7 @@ pub fn archive_operation(anchor_number: AnchorNumber, caller: Principal, operati
return;
};

// For layout versions < 6 this will always be false because nothing is ever added to the buffer
if data.entries_buffer.len() as u64 >= config.entries_buffer_limit {
if data.buffered_entries_count() as u64 >= config.entries_buffer_limit {
trap("cannot archive operation, archive entries buffer limit reached")
}

Expand All @@ -348,17 +335,9 @@ pub fn archive_operation(anchor_number: AnchorNumber, caller: Principal, operati
};

// add entry to buffer (which is emptied by the archive periodically, see fetch_entries and acknowledge entries)
match state::storage_borrow(|s| s.version()) {
7 => {
state::archive_data_mut(|data| {
Rc::make_mut(&mut data.entries_buffer).push(buffered_entry);
});
}
8 => state::storage_borrow_mut(|s| {
s.add_archive_entry(buffered_entry);
}),
_ => trap("unsupported storage version"),
}
state::storage_borrow_mut(|s| {
s.add_archive_entry(buffered_entry);
});

state::archive_data_mut(|data| {
data.sequence_number += 1;
Expand All @@ -373,16 +352,7 @@ pub fn fetch_entries() -> Vec<BufferedEntry> {

// buffered entries are ordered by sequence number
// i.e. this takes the first (by sequence numbers) entries_fetch_limit many entries
match state::storage_borrow(|s| s.version()) {
7 => data
.entries_buffer
.iter()
.take(config.entries_fetch_limit as usize)
.cloned()
.collect(),
8 => state::storage_borrow_mut(|s| s.get_archive_entries(config.entries_fetch_limit)),
_ => trap("unsupported storage version"),
}
state::storage_borrow_mut(|s| s.get_archive_entries(config.entries_fetch_limit))
}

pub fn acknowledge_entries(sequence_number: u64) {
Expand All @@ -393,12 +363,7 @@ pub fn acknowledge_entries(sequence_number: u64) {
trap_if_caller_not_archive(data);

// Only keep entries with higher sequence number as the highest acknowledged.
match state::storage_borrow(|s| s.version()) {
7 => Rc::make_mut(&mut data.entries_buffer)
.retain(|e| e.sequence_number > sequence_number),
8 => state::storage_borrow_mut(|s| s.prune_archive_entries(sequence_number)),
_ => trap("unsupported storage version"),
}
state::storage_borrow_mut(|s| s.prune_archive_entries(sequence_number))
});
}

Expand Down
19 changes: 0 additions & 19 deletions src/internet_identity/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ use internet_identity_interface::internet_identity::types::vc_mvp::{
use internet_identity_interface::internet_identity::types::*;
use serde_bytes::ByteBuf;
use std::collections::HashMap;
use std::ops::Deref;
use storage::{Salt, Storage};

mod activity_stats;
Expand Down Expand Up @@ -410,24 +409,6 @@ fn post_upgrade(maybe_arg: Option<InternetIdentityInit>) {
// load the persistent state after initializing storage, otherwise the memory address to load it from cannot be calculated
state::load_persistent_state();

// Migrate the archive entries buffer if necessary.
state::storage_borrow_mut(|storage| {
if storage.version() == 7 {
// We need to update the storage layout to version 8.
let entries = match state::archive_state() {
// If the archive has been created, we need to migrate the entries to the archive buffer.
// The buffer in persistent state is not modified yet. It will be cleaned up when it
// is moved into its own stable memory.
ArchiveState::Created { data, .. } => data.entries_buffer.deref().clone(),
// If not, we migrate using the empty buffer.
ArchiveState::NotConfigured
| ArchiveState::Configured { .. }
| ArchiveState::CreationInProgress { .. } => vec![],
};
storage.migrate_to_stable_memory_archive_buffer(entries);
}
});

apply_install_arg(maybe_arg);
}

Expand Down
41 changes: 5 additions & 36 deletions src/internet_identity/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,10 @@ mod storable_anchor;
mod tests;

/// * version 0: invalid
/// * version 1-6: no longer supported
/// * version 7: 4KB anchors, candid anchor record layout, persistent state with archive pull config,
/// with memory manager (from 2nd page on)
/// * version 8: same as 7, but archive entries buffer in stable memory
const SUPPORTED_LAYOUT_VERSIONS: RangeInclusive<u8> = 7..=8;
/// * version 1-7: no longer supported
/// * version 8: 4KB anchors, candid anchor record layout, persistent state with archive pull config,
/// with memory manager (from 2nd page on), archive entries buffer in stable memory
const SUPPORTED_LAYOUT_VERSIONS: RangeInclusive<u8> = 8..=8;

const DEFAULT_ENTRY_SIZE: u16 = 4096;
const EMPTY_SALT: [u8; 32] = [0; 32];
Expand Down Expand Up @@ -290,7 +289,7 @@ impl<M: Memory + Clone> Storage<M> {
}

match header.version {
7 | 8 => {
8 => {
let header_memory = RestrictedMemory::new(memory.clone(), 0..1);
let managed_memory = RestrictedMemory::new(memory, 1..MAX_MANAGED_WASM_PAGES);
let memory_manager =
Expand Down Expand Up @@ -417,22 +416,12 @@ impl<M: Memory + Clone> Storage<M> {

/// Add a new archive entry to the buffer.
pub fn add_archive_entry(&mut self, entry: BufferedEntry) {
assert_eq!(
self.version(),
8,
"archive entry buffer is only supported in version 8"
);
self.archive_entries_buffer
.insert(entry.sequence_number, BufferedEntryWrapper(entry));
}

/// Get the first `max_entries` archive entries from the buffer.
pub fn get_archive_entries(&mut self, max_entries: u16) -> Vec<BufferedEntry> {
assert_eq!(
self.version(),
8,
"archive entry buffer is only supported in version 8"
);
self.archive_entries_buffer
.iter()
.take(max_entries as usize)
Expand All @@ -442,11 +431,6 @@ impl<M: Memory + Clone> Storage<M> {

/// Prune all archive entries with sequence numbers less than or equal to the given sequence number.
pub fn prune_archive_entries(&mut self, sequence_number: u64) {
assert_eq!(
self.version(),
8,
"archive entry buffer is only supported in version 8"
);
let entries_to_prune = self
.archive_entries_buffer
.range(..=sequence_number)
Expand All @@ -462,21 +446,6 @@ impl<M: Memory + Clone> Storage<M> {
self.archive_entries_buffer.iter().count()
}

/// Migrates storage to version 8, by adding all pending archive entries to the archive buffer.
pub fn migrate_to_stable_memory_archive_buffer(&mut self, entries: Vec<BufferedEntry>) {
assert_eq!(
self.version(),
7,
"migration to stable memory archive entry buffer is only supported from version 7"
);
for entry in entries {
self.archive_entries_buffer
.insert(entry.sequence_number, BufferedEntryWrapper(entry));
}
self.header.version = 8;
self.flush();
}

fn anchor_number_to_record(&self, anchor_number: u64) -> Result<u32, StorageError> {
if anchor_number < self.header.id_range_lo || anchor_number >= self.header.id_range_hi {
return Err(StorageError::AnchorNumberOutOfRange {
Expand Down
15 changes: 0 additions & 15 deletions src/internet_identity/src/storage/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use internet_identity_interface::internet_identity::types::{
ArchiveConfig, DeviceProtection, KeyType, Purpose,
};
use serde_bytes::ByteBuf;
use std::rc::Rc;

const HEADER_SIZE: usize = 58;

Expand Down Expand Up @@ -42,19 +41,6 @@ fn should_serialize_header_v8() {
assert_eq!(buf, hex::decode("49494308000000000100000000000000020000000000000000100505050505050505050505050505050505050505050505050505050505050505").unwrap());
}

#[test]
fn should_recover_header_from_memory_v7() {
let memory = VectorMemory::default();
memory.grow(1);
memory.write(0, &hex::decode("494943070500000040e2010000000000f1fb090000000000000843434343434343434343434343434343434343434343434343434343434343430002000000000000000000000000000000000000000000000000").unwrap());

let storage = Storage::from_memory(memory).unwrap();
assert_eq!(storage.assigned_anchor_number_range(), (123456, 654321));
assert_eq!(storage.salt().unwrap(), &[67u8; 32]);
assert_eq!(storage.anchor_count(), 5);
assert_eq!(storage.version(), 7);
}

#[test]
fn should_recover_header_from_memory_v8() {
let memory = VectorMemory::default();
Expand Down Expand Up @@ -164,7 +150,6 @@ fn sample_persistent_state() -> PersistentState {
data: ArchiveData {
sequence_number: 39,
archive_canister: Principal::from_text("2h5ob-7aaaa-aaaad-aacya-cai").unwrap(),
entries_buffer: Rc::new(vec![]),
},
config: ArchiveConfig {
module_hash: [99u8; 32],
Expand Down
Loading