Merge branch 'master' of github.com:josephg/diamond-types

README.md

@@ -18,6 +18,8 @@ This project was initially created as a prototype to see how fast a well optimiz
 
 For detail about how to *use* diamond types, see the [package level documentation at docs.rs](https://docs.rs/diamond-types/latest/diamond_types/).
 
+Note the package published to cargo is quite out of date, both in terms of API and performance.
+
 For much more detail about how this library *works*, see:
 
 - The talk I gave on this library at a recent [braid user meetings](https://braid.org/meeting-14) or

crates/bench/Cargo.toml

@@ -8,5 +8,6 @@ edition = "2021"
 [dependencies]
 diamond-types = { path = "../.." }
 rle = { path = "../rle" }
-criterion = "0.4.0"
+#criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.5.1", features = [] }
 crdt-testdata = { path = "../crdt-testdata" }

crates/bench/src/main.rs

@@ -15,7 +15,7 @@ fn testing_data(name: &str) -> TestData {
 }
 
 const LINEAR_DATASETS: &[&str] = &["automerge-paper", "rustcode", "sveltecomponent", "seph-blog1", "friendsforever_flat"];
-const COMPLEX_DATASETS: &[&str] = &["node_nodecc", "git-makefile", "friendsforever"];
+const COMPLEX_DATASETS: &[&str] = &["node_nodecc", "git-makefile", "friendsforever", "clownschool"];
 
 fn local_benchmarks(c: &mut Criterion) {
     for name in LINEAR_DATASETS {

crates/content-tree/src/unsafe_cursor.rs

@@ -87,6 +87,8 @@ impl<E: ContentTraits, I: TreeMetrics<E>, const IE: usize, const LE: usize> Unsa
         }
     }
 
+    /// Go to the next entry marker
+    /// Returns true if successful, or false if we've reached the end of the document.
     #[inline(always)]
     pub fn next_entry(&mut self) -> bool {
         self.next_entry_marker(None)
@@ -185,10 +187,6 @@ impl<E: ContentTraits, I: TreeMetrics<E>, const IE: usize, const LE: usize> Unsa
         &mut node.data[self.idx]
     }
 
-    /// This is a terrible name. This method modifies a cursor at the end of an entry
-    /// to be a cursor to the start of the next entry - potentially in the following leaf.
-    ///
-    /// Returns false if the resulting cursor location points past the end of the tree.
     pub(crate) fn roll_to_next_entry_internal<F: FnOnce(&mut Self)>(&mut self, f: F) -> bool {
         unsafe {
             if self.offset == usize::MAX {
@@ -208,6 +206,10 @@ impl<E: ContentTraits, I: TreeMetrics<E>, const IE: usize, const LE: usize> Unsa
         }
     }
 
+    /// This is a terrible name. This method modifies a cursor at the end of an entry
+    /// to be a cursor to the start of the next entry - potentially in the following leaf.
+    ///
+    /// Returns false if the resulting cursor location points past the end of the tree.
     pub fn roll_to_next_entry(&mut self) -> bool {
         self.roll_to_next_entry_internal(|_| {})
     }

crates/diamond-core-old/src/lib.rs

@@ -1,4 +1,5 @@
 use std::cmp::Ordering;
+use std::fmt::{Debug, Formatter};
 
 pub mod alloc;
 
@@ -10,12 +11,26 @@ pub type ItemCount = u32;
 
 pub const CLIENT_INVALID: AgentId = AgentId::MAX;
 
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Copy, Clone, Eq, PartialEq)]
 pub struct CRDTId {
     pub agent: AgentId,
     pub seq: u32,
 }
 
+impl Debug for CRDTId {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        if self.agent == CLIENT_INVALID {
+            f.write_str("ROOT")
+        } else {
+            // f.debug_tuple("CRDTId")
+            f.debug_list()
+                .entry(&self.agent)
+                .entry(&self.seq)
+                .finish()
+        }
+    }
+}
+
 impl Default for CRDTId {
     fn default() -> Self {
         CRDTId {

crates/diamond-types-old/README.md

@@ -0,0 +1,3 @@
+This is a fully working, optimized implementation of the sequence CRDT from diamond types, but implemented as a traditional CRDT rather than using the new "time travel" algorithm.
+
+This code is kept alive mostly for comparative benchmarking, since this codebase employs all the same optimization tricks as the other implementation.

crates/diamond-types-old/examples/stats.rs

@@ -11,7 +11,7 @@ use crdt_testdata::{load_testing_data, TestPatch, TestTxn};
 use diamond_types_old::list::*;
 
 #[cfg(feature = "memusage")]
-use diamond_core::alloc::*;
+use diamond_core_old::alloc::*;
 #[cfg(feature = "memusage")]
 use humansize::{FileSize, file_size_opts};
 use diamond_types_old::list::positional::PositionalOpRef;

crates/diamond-types-old/src/crdtspan.rs

@@ -1,5 +1,5 @@
 use diamond_core_old::CRDTId;
-use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, SplitableSpanHelpers};
 
 use content_tree::ContentLength;
 use rle::Searchable;

crates/diamond-types-old/src/lib.rs

@@ -1,5 +1,5 @@
 #![allow(dead_code)] // TODO: turn this off and clean up before releasing.
-#![allow(unused)]
+// #![allow(unused)]
 
 use crate::list::external_txn::RemoteId;
 

crates/diamond-types-old/src/list/check.rs

@@ -91,10 +91,10 @@ impl ListCRDT {
                 assert!(child.parents.iter().any(|p| txn.contains(*p)));
             }
 
-            if parents[0] == ROOT_LV {
+            if parents.is_empty() || parents[0] == ROOT_LV {
                 // The root order will be sorted out of order, but it doesn't matter because
                 // if it shows up at all it should be the only item in parents.
-                debug_assert_eq!(parents.len(), 1);
+                assert!(parents.len() <= 1);
                 if txn.time == 0 { expect_shadow = ROOT_LV; }
                 assert!(txn.parent_indexes.is_empty());
             } else {

crates/diamond-types-old/src/list/doc.rs

@@ -266,7 +266,21 @@ impl ListCRDT {
         span.1.len - span_offset
     }
 
-    // fn integrate(&mut self, agent: AgentId, item: YjsSpan, ins_content: Option<&str>, cursor_hint: Option<UnsafeCursor<YjsSpan, DocRangeIndex, DOC_IE, DOC_LE>>) -> UnsafeCursor<YjsSpan, DocRangeIndex, DOC_IE, DOC_LE> {
+    fn get_right_parent(&self, item: &YjsSpan, offset: u32) -> (Cursor<'_, YjsSpan, DocRangeIndex>, LV) {
+        let mut right_cursor = self.get_cursor_before(item.origin_right);
+        let right_parent = if let Some(origin_right) = right_cursor.try_get_raw_entry() {
+            if origin_right.origin_left == item.origin_left_at_offset(offset) {
+                // Right parent is the item's origin_right.
+                item.origin_right
+            } else {
+                // No right parent. Reset the right_cursor to point to the end of the list.
+                right_cursor = self.range_tree.cursor_at_end();
+                ROOT_LV
+            }
+        } else { ROOT_LV };
+        (right_cursor, right_parent)
+    }
+
     pub(super) fn integrate(&mut self, agent: AgentId, item: YjsSpan, ins_content: Option<&str>, cursor_hint: Option<UnsafeCursor<YjsSpan, DocRangeIndex, DOC_IE, DOC_LE>>) {
         // if cfg!(debug_assertions) {
         //     let next_order = self.get_next_order();
@@ -319,8 +333,15 @@ impl ListCRDT {
                 Ordering::Less => { break; } // Top row
                 Ordering::Greater => { } // Bottom row. Continue.
                 Ordering::Equal => {
-                    if item.origin_right == other_entry.origin_right {
-                        // Origin_right matches. Items are concurrent. Order by agent names.
+                    let (my_right_cursor, right_parent) = self.get_right_parent(&item, 0);
+                    // We need to pass the cursor offset in here because the cursor might be
+                    // pointing in the middle of an item, and we need to adjust origin_left
+                    // appropriately in that case.
+                    let (other_right_cursor, other_parent) = self.get_right_parent(&other_entry, cursor.offset as u32);
+
+                    // if item.origin_right == other_entry.origin_right {
+                    if right_parent == other_parent {
+                        // Items are concurrent and "double siblings". Order by agent names.
                         let my_name = self.get_agent_name(agent);
                         let other_loc = self.client_with_time.get(other_order);
                         let other_name = self.get_agent_name(other_loc.agent);
@@ -330,8 +351,7 @@ impl ListCRDT {
                         let ins_here = match my_name.cmp(other_name) {
                             Ordering::Less => true,
                             Ordering::Equal => {
-                                self.get_crdt_location(item.time).seq < self.get_crdt_location(other_entry.time).seq
-                                // oplog.time_to_crdt_id(item.id.start) < oplog.time_to_crdt_id(other_entry.id.start)
+                                self.get_crdt_location(item.time).seq < other_loc.seq
                             }
                             Ordering::Greater => false,
                         };
@@ -344,8 +364,8 @@ impl ListCRDT {
                         }
                     } else {
                         // Set scanning based on how the origin_right entries are ordered.
-                        let my_right_cursor = self.get_cursor_before(item.origin_right);
-                        let other_right_cursor = self.get_cursor_before(other_entry.origin_right);
+                        // let my_right_cursor = self.get_cursor_before(item.origin_right);
+                        // let other_right_cursor = self.get_cursor_before(other_entry.origin_right);
 
                         if other_right_cursor < my_right_cursor {
                             if !scanning {
@@ -446,6 +466,8 @@ impl ListCRDT {
             }
         }
 
+        // println!("insert_txn_internal {:?} {:?}", txn_parents, range);
+
         // let parents = replace(&mut self.frontier, txn_parents);
         let mut shadow = range.start;
         while shadow >= 1 && txn_parents.contains(&(shadow - 1)) {
@@ -471,6 +493,7 @@ impl ListCRDT {
                 // Interestingly the parent_idx array will always end up the same length as parents
                 // because it would be invalid for multiple parents to point to the same entry in
                 // txns. (That would imply one parent is a descendant of another.)
+
                 debug_assert!(!parent_indexes.contains(&parent_idx));
                 parent_indexes.push(parent_idx);
 
@@ -542,7 +565,7 @@ impl ListCRDT {
     pub fn apply_remote_txn(&mut self, txn: &RemoteTxn) {
         let agent = self.get_or_create_agent_id(txn.id.agent.as_str());
         let client = &self.client_data[agent as usize];
-        let next_seq = client.get_next_seq();
+        // let next_seq = client.get_next_seq();
 
         // Check that the txn hasn't already been applied.
         assert!(client.item_localtime.find(txn.id.seq).is_none());
@@ -856,6 +879,26 @@ impl ListCRDT {
         }
     }
 
+    #[allow(unused)]
+    pub fn debug_print_ids(&self) {
+        for span in self.range_tree.iter() {
+            let id = self.get_crdt_location(span.time);
+            let left = self.get_crdt_location(span.origin_left);
+            let right = self.get_crdt_location(span.origin_right);
+
+            let parent_order = self.get_right_parent(&span, 0).1;
+            let parent = self.get_crdt_location(parent_order);
+
+            println!("{:?} (len {}) left {:?} right {:?} right_parent {:?}\t(LV: {})",
+                     id, span.len, left, right, parent, span.time
+            );
+        }
+
+        // dbg!(&self.deletes);
+        // self.debug_print_del();
+    }
+
+
     #[allow(unused)]
     pub fn debug_print_del(&self) {
         for e in self.deletes.iter() {

crates/diamond-types-old/src/list/encoding/mod.rs

@@ -7,7 +7,7 @@ use num_enum::TryFromPrimitive;
 use smallvec::SmallVec;
 
 use diamond_core_old::CRDTId;
-use rle::{HasLength, MergableSpan, MergeableIterator, SplitableSpan, SplitableSpanHelpers, SplitAndJoinSpan};
+use rle::{HasLength, MergableSpan, MergeableIterator, SplitableSpanHelpers, SplitAndJoinSpan};
 
 use crate::crdtspan::CRDTSpan;
 use crate::list::{ListCRDT, LV};

crates/diamond-types-old/src/list/external_txn.rs

@@ -9,7 +9,7 @@ use smartstring::alias::String as SmartString;
 
 use content_tree::Toggleable;
 use diamond_core_old::{AgentId, CRDT_DOC_ROOT, CRDTId};
-use rle::{AppendRle, HasLength};
+use rle::{AppendRle, HasLength, MergableSpan};
 
 use crate::crdtspan::CRDTSpan;
 use crate::list::{Branch, ListCRDT, LV, ROOT_LV};
@@ -63,6 +63,46 @@ pub enum RemoteCRDTOp {
     }
 }
 
+impl MergableSpan for RemoteIdSpan {
+    fn can_append(&self, other: &Self) -> bool {
+        self.id.agent == other.id.agent && self.id.seq + self.len == other.id.seq
+    }
+
+    fn append(&mut self, other: Self) {
+        self.len += other.len
+    }
+}
+
+
+impl MergableSpan for RemoteCRDTOp {
+    fn can_append(&self, other: &Self) -> bool {
+        // We're assuming the IDs are adjacent.
+        match (self, other) {
+            // Can't append inserts because we don't have the ID of the operation, and we would
+            // need to check that other.origin_left == other.id_start - 1.
+            //
+            // We'll just merge deletes.
+            (Del { id, len }, Del { id: other_id, .. }) => {
+                id.agent == other_id.agent
+                    && id.seq + len == other_id.seq
+            }
+            _ => false,
+        }
+    }
+
+    fn append(&mut self, other: Self) {
+        match (self, other) {
+            (Ins { len, .. }, Ins { len: other_len, .. }) => {
+                *len += other_len;
+            },
+            (Del { len, .. }, Del { len: other_len, .. }) => {
+                *len += other_len;
+            },
+            _ => panic!("Cannot append mismatched operations")
+        }
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct RemoteTxn {

crates/diamond-types-old/src/list/markers.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::ptr::NonNull;
 
-use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, SplitableSpanHelpers};
 
 use content_tree::*;
 use content_tree::ContentTraits;

crates/diamond-types-old/src/list/mod.rs

@@ -1,5 +1,5 @@
 use std::pin::Pin;
-use jumprope::{JumpRope, JumpRopeBuf};
+use jumprope::JumpRopeBuf;
 
 use smallvec::SmallVec;
 use smartstring::alias::String as SmartString;
@@ -131,7 +131,7 @@ pub struct ListCRDT {
     /// TODO: Consider renaming this field
     txns: RleVec<TxnSpan>,
 
-    // Temporary. This will be moved out into a reference to another data structure I think.
+    /// The document state.
     text_content: Option<JumpRopeBuf>,
     /// This is a big ol' string containing everything that's been deleted (self.deletes) in order.
     deleted_content: Option<String>,

crates/diamond-types-old/src/list/ot/positionmap.rs

@@ -1,9 +1,8 @@
 // TODO: This file ended up being a kitchen sink for the logic here. Separate this logic out into a
 // few files!
 
-use std::borrow::Borrow;
 use std::pin::Pin;
-use jumprope::{JumpRope, JumpRopeBuf};
+use jumprope::JumpRopeBuf;
 
 use smallvec::SmallVec;
 
@@ -562,6 +561,7 @@ fn map_to_traversal(map: &PositionMap, resulting_doc: &JumpRopeBuf) -> Traversal
 
 #[cfg(test)]
 mod test {
+    use jumprope::JumpRope;
     use rand::prelude::*;
     use smallvec::smallvec;
 

crates/diamond-types-old/src/list/ot/traversal.rs

@@ -16,7 +16,7 @@
 
 use smartstring::alias::{String as SmartString};
 use smallvec::{SmallVec, smallvec};
-use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, SplitableSpanHelpers};
 use TraversalComponent::*;
 use crate::list::LV;
 use crate::list::positional::{PositionalOp, PositionalComponent, InsDelTag};

crates/diamond-types-old/src/list/positional.rs

@@ -7,7 +7,7 @@ use jumprope::{JumpRope, JumpRopeBuf};
 
 use smartstring::alias::{String as SmartString};
 use smallvec::{SmallVec, smallvec};
-use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, SplitableSpanHelpers};
 use InsDelTag::*;
 use crate::unicount::{chars_to_bytes, count_chars};
 use rle::AppendRle;

crates/diamond-types-old/src/list/span.rs

@@ -1,5 +1,5 @@
 use std::fmt::{Debug, DebugStruct, Formatter};
-use rle::{HasLength, MergableSpan, Searchable, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, Searchable, SplitableSpanHelpers};
 
 use content_tree::ContentLength;
 use content_tree::Toggleable;

crates/diamond-types-old/src/list/time/external_patches.rs

@@ -5,7 +5,7 @@
 use smallvec::{SmallVec, smallvec};
 use crate::list::{ListCRDT, LV, PositionalComponent};
 use smartstring::alias::{String as SmartString};
-use rle::{AppendRle, HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{AppendRle, HasLength, MergableSpan, SplitableSpanHelpers};
 use crate::list::external_txn::{RemoteId, RemoteIdSpan};
 use crate::list::time::docpatchiter::PositionalOpWalk;
 use crate::list::txn::TxnSpan;

crates/diamond-types-old/src/list/time/positionmap.rs

@@ -1,7 +1,7 @@
 use std::iter::FromIterator;
 use std::mem::take;
 use content_tree::*;
-use rle::{HasLength, MergableSpan, SplitableSpan, SplitableSpanHelpers};
+use rle::{HasLength, MergableSpan, SplitableSpanHelpers};
 
 use smartstring::alias::{String as SmartString};
 use crate::list::time::positionmap::MapTag::*;