From 6a426751ffc5e4c0ce706c4b13a97a306a92f2f1 Mon Sep 17 00:00:00 2001 From: Seph Gentle Date: Wed, 10 Jan 2024 14:38:25 +1100 Subject: [PATCH] Tweaks for getting detailed stats --- examples/posstats.rs | 81 ++++++++++++++++++++++++------- src/list/encoding/encode_oplog.rs | 7 ++- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/examples/posstats.rs b/examples/posstats.rs index 2468d41..9399859 100644 --- a/examples/posstats.rs +++ b/examples/posstats.rs @@ -48,7 +48,8 @@ fn print_stats_for_testdata(name: &str) { let test_data = load_testing_data(&filename); assert_eq!(test_data.start_content.len(), 0); println!("\n\nLoaded testing data from {}\n ({} patches in {} txns -> docsize {} chars)", - filename, + // filename, + name, test_data.len(), test_data.txns.len(), test_data.end_content.chars().count() @@ -78,22 +79,24 @@ fn print_stats_for_testdata(name: &str) { println!("Branch size {}", doc.len()); // println!("---\nEncoded size {} (?? What do we include here?)", as_bytes.len()); - let out_file = format!("{}.dt", name); - let data = doc.oplog.encode(EncodeOptions { - user_data: None, - store_start_branch_content: false, - experimentally_store_end_branch_content: false, - store_inserted_content: true, - store_deleted_content: false, - compress_content: true, - verbose: true - }); - println!("Regular file size {} bytes", data.len()); - std::fs::write(out_file.clone(), data.as_slice()).unwrap(); - println!("Saved to {}", out_file); + // let out_file = format!("{}.dt", name); + // let data = doc.oplog.encode(EncodeOptions { + // user_data: None, + // store_start_branch_content: false, + // experimentally_store_end_branch_content: false, + // store_inserted_content: true, + // store_deleted_content: false, + // compress_content: true, + // verbose: true + // }); + // println!("Regular file size {} bytes", data.len()); + // std::fs::write(out_file.clone(), data.as_slice()).unwrap(); + // println!("Saved to {}", out_file); - #[cfg(feature = "gen_test_data")] - write_stats(name, &doc.oplog); + // #[cfg(feature = "gen_test_data")] + // write_stats(name, &doc.oplog); + + print_stats_for_oplog(name, &doc.oplog); } #[allow(unused)] @@ -102,9 +105,9 @@ fn print_stats_for_file(name: &str) { println!("\n\nLoaded testing data from {} ({} bytes)", name, contents.len()); #[cfg(feature = "memusage")] - let start_bytes = get_thread_memory_usage(); + let start_bytes = get_thread_memory_usage(); #[cfg(feature = "memusage")] - let start_count = get_thread_num_allocations(); + let start_count = get_thread_num_allocations(); let oplog = ListOpLog::load_from(&contents).unwrap(); #[cfg(feature = "memusage")] @@ -113,8 +116,26 @@ fn print_stats_for_file(name: &str) { get_thread_num_allocations() - start_count); oplog.print_stats(false); + print_stats_for_oplog(name, &oplog); +} + +fn print_stats_for_oplog(name: &str, oplog: &ListOpLog) { // oplog.make_time_dag_graph("node_cc.svg"); + println!("---- Saving normally ----"); + let data = oplog.encode(EncodeOptions { + user_data: None, + store_start_branch_content: false, + experimentally_store_end_branch_content: false, + store_inserted_content: true, + store_deleted_content: false, + compress_content: true, + verbose: true + }); + println!("Regular file size {} bytes", data.len()); + + + println!("---- Saving smol mode ----"); let data_smol = oplog.encode(EncodeOptions { user_data: None, store_start_branch_content: false, @@ -126,6 +147,30 @@ fn print_stats_for_file(name: &str) { }); println!("Smol size {}", data_smol.len()); + println!("---- Saving uncompressed ----"); + let data_uncompressed = oplog.encode(EncodeOptions { + user_data: None, + store_start_branch_content: false, + experimentally_store_end_branch_content: false, + store_inserted_content: true, + store_deleted_content: false, + compress_content: false, + verbose: true + }); + println!("Uncompressed size {}", data_uncompressed.len()); + + println!("---- Saving smol uncompressed ----"); + let data_uncompressed = oplog.encode(EncodeOptions { + user_data: None, + store_start_branch_content: false, + experimentally_store_end_branch_content: true, + store_inserted_content: false, + store_deleted_content: false, + compress_content: false, + verbose: true + }); + println!("Uncompressed size {}", data_uncompressed.len()); + oplog.bench_writing_xf_since(&[]); // oplog.make_time_dag_graph_with_merge_bubbles(&format!("{name}.svg")); diff --git a/src/list/encoding/encode_oplog.rs b/src/list/encoding/encode_oplog.rs index 5650045..64f657f 100644 --- a/src/list/encoding/encode_oplog.rs +++ b/src/list/encoding/encode_oplog.rs @@ -15,7 +15,7 @@ use crate::list::encoding::encode_tools::{Merger, push_leb_chunk, push_leb_str, use crate::list::encoding::leb::{encode_leb_u32, encode_leb_usize, num_encode_zigzag_isize_old}; use crate::listmerge::plan::M1PlanAction; -const ALLOW_VERBOSE: bool = false; +const ALLOW_VERBOSE: bool = true; /// Write an operation to the passed writer. fn write_op(dest: &mut Vec, op: &ListOpMetrics, cursor: &mut usize) { @@ -622,6 +622,9 @@ impl ListOpLog { write_local_version(&mut end_branch, self.cg.version.as_ref(), &mut agent_mapping, self); let branch_here = ListBranch::new_at_tip(self); + if verbose { + println!("End content length (uncompressed) {}", branch_here.content.len_bytes()); + } write_content_rope(&mut end_branch, &branch_here.content.borrow(), compress_bytes.as_mut()); Some(end_branch) @@ -653,7 +656,7 @@ impl ListOpLog { // chunk goes first in the file, so if we compress anything, it needs to be filled up. let inserted_content = inserted_content.and_then(|inserted_content| { if verbose { - println!("Inserted text length {}", inserted_content.content.len()); + println!("Inserted text length (uncompressed) {}", inserted_content.content.len()); } inserted_content.flush(compress_bytes.as_mut())