From b954ee889d86f89854668ea843099df1f472fb59 Mon Sep 17 00:00:00 2001 From: Tin Rabzelj Date: Wed, 28 Aug 2024 01:48:41 +0200 Subject: [PATCH] Update ID generator --- bomboni_common/Cargo.toml | 1 + bomboni_common/src/id/mod.rs | 142 ++++++++++++++---- .../src/id/{generator.rs => worker.rs} | 71 +++------ bomboni_proto/build.rs | 24 +++ bomboni_proto/src/protobuf/empty.rs | 23 ++- bomboni_request/src/parse/mod.rs | 4 +- 6 files changed, 173 insertions(+), 92 deletions(-) rename bomboni_common/src/id/{generator.rs => worker.rs} (68%) diff --git a/bomboni_common/Cargo.toml b/bomboni_common/Cargo.toml index a70cd73..60a50a4 100644 --- a/bomboni_common/Cargo.toml +++ b/bomboni_common/Cargo.toml @@ -34,6 +34,7 @@ bomboni_wasm = { path = "../bomboni_wasm", version = "0.1.60", features = [ thiserror = "1.0.63" regex = "1.10.5" time = { version = "0.3.36", features = ["formatting", "parsing"] } +ulid = "1.1.3" tokio = { version = "1.39.1", features = ["time", "sync"], optional = true } parking_lot = { version = "0.12.3", optional = true } diff --git a/bomboni_common/src/id/mod.rs b/bomboni_common/src/id/mod.rs index faacdfe..8d22e11 100644 --- a/bomboni_common/src/id/mod.rs +++ b/bomboni_common/src/id/mod.rs @@ -1,24 +1,22 @@ //! # Id //! //! Semi-globally unique and sortable identifiers. + use std::{ fmt::{self, Display, Formatter}, - num::ParseIntError, str::FromStr, }; +use thiserror::Error; +use ulid::Ulid; #[cfg(feature = "serde")] use serde::{de::Unexpected, Deserialize, Deserializer, Serialize, Serializer}; use crate::date_time::UtcDateTime; -pub mod generator; #[cfg(feature = "postgres")] mod postgres; - -const TIMESTAMP_BITS: i64 = 64; -const WORKER_BITS: i64 = 16; -const SEQUENCE_BITS: i64 = 16; +pub mod worker; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr( @@ -36,15 +34,43 @@ const SEQUENCE_BITS: i64 = 16; )] pub struct Id(u128); +#[derive(Error, Debug, Clone, PartialEq, Eq)] +pub enum ParseIdError { + #[error("invalid id string")] + InvalidString, +} + +const TIMESTAMP_BITS: i64 = 64; +const WORKER_BITS: i64 = 16; +const SEQUENCE_BITS: i64 = 16; + impl Id { #[must_use] pub const fn new(id: u128) -> Self { Self(id) } - /// Encodes the Id from parts. + /// Generates a new random sortable id. + #[must_use] + pub fn generate() -> Self { + Self(Ulid::new().0) + } + + /// Generate multiple random sortable ids. + /// Generated ids are monotonically increasing. #[must_use] - pub fn from_parts(time: UtcDateTime, worker: u16, sequence: u16) -> Self { + pub fn generate_multiple(count: usize) -> Vec { + let mut ids = Vec::with_capacity(count); + let mut g = ulid::Generator::new(); + for _ in 0..count { + ids.push(Self::new(g.generate().unwrap().0)); + } + ids + } + + /// Encodes the [`Id`] from worker parts. + #[must_use] + pub fn from_worker_parts(time: UtcDateTime, worker: u16, sequence: u16) -> Self { let timestamp = time.unix_timestamp() as u128; let worker = u128::from(worker); let sequence = u128::from(sequence); @@ -60,24 +86,17 @@ impl Id { ) } - /// Decodes Id's parts. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// use bomboni_common::{id::Id, date_time::UtcDateTime}; - /// - /// let time = UtcDateTime::from_timestamp(1337, 0).unwrap(); - /// let id = Id::from_parts(time, 42, 1); - /// let (timestamp, worker, sequence) = id.decode(); - /// assert_eq!(timestamp, time); - /// assert_eq!(worker, 42); - /// assert_eq!(sequence, 1); - /// ``` + /// Encodes the [`Id`] from time and a random number. #[must_use] - pub fn decode(self) -> (UtcDateTime, u16, u16) { + pub fn from_time_and_random(time: UtcDateTime, random: u128) -> Self { + let timestamp_ms = time.unix_timestamp_nanos() / 1_000_000; + let id = Ulid::from_parts(timestamp_ms as u64, random); + Self::new(id.0) + } + + /// Decodes [`Id`]'s worker parts. + #[must_use] + pub fn decode_worker(self) -> (UtcDateTime, u16, u16) { let timestamp = UtcDateTime::from_timestamp((self.0 >> (WORKER_BITS + SEQUENCE_BITS)) as i64, 0) .unwrap(); @@ -85,19 +104,36 @@ impl Id { let sequence = (self.0 & ((1 << SEQUENCE_BITS) - 1)) as u16; (timestamp, worker, sequence) } + + /// Decodes [`Id`]'s time and randomness parts. + #[must_use] + pub fn decode_time_and_random(self) -> (UtcDateTime, u128) { + let id = Ulid::from(self.0); + let timestamp_ms = id.timestamp_ms(); + let seconds = timestamp_ms / 1000; + let nanoseconds = timestamp_ms % 1000 * 1_000_000; + ( + UtcDateTime::from_timestamp(seconds as i64, nanoseconds as u32).unwrap(), + id.random(), + ) + } } impl Display for Id { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{:x}", self.0) + let buf = self.0.to_be_bytes(); + for b in buf { + write!(f, "{b:02X}")?; + } + Ok(()) } } impl FromStr for Id { - type Err = ParseIntError; + type Err = ParseIdError; fn from_str(s: &str) -> Result { - let value = u128::from_str_radix(s, 16)?; + let value = u128::from_str_radix(s, 16).map_err(|_| ParseIdError::InvalidString)?; Ok(Self::new(value)) } } @@ -119,6 +155,18 @@ impl From for u128 { } } +impl From for Id { + fn from(ulid: Ulid) -> Self { + Self(ulid.into()) + } +} + +impl From for Ulid { + fn from(id: Id) -> Self { + Ulid::from(id.0) + } +} + #[cfg(feature = "serde")] impl Serialize for Id { fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> @@ -145,14 +193,39 @@ impl<'de> Deserialize<'de> for Id { #[cfg(test)] mod tests { + use super::*; #[test] - fn it_works() { + fn generate_random() { + use std::collections::HashMap; + const N: usize = 10; + + let mut ids = HashMap::new(); + for _ in 0..N { + let id = Id::generate(); + ids.insert(id.to_string(), id); + } + assert_eq!(ids.len(), N); + + ids = Id::generate_multiple(N) + .into_iter() + .map(|id| (id.to_string(), id)) + .collect(); + assert_eq!(ids.len(), N); + + for (id_str, id) in ids { + let decoded: Id = id_str.parse().unwrap(); + assert_eq!(decoded, id); + } + } + + #[test] + fn worker_parts() { let ts = UtcDateTime::from_timestamp(10, 0).unwrap(); - let id = Id::from_parts(ts, 1, 1); + let id = Id::from_worker_parts(ts, 1, 1); assert_eq!(id, Id(0b1010_0000_0000_0000_0001_0000_0000_0000_0001)); - let (timestamp, worker, sequence) = id.decode(); + let (timestamp, worker, sequence) = id.decode_worker(); assert_eq!(timestamp, ts); assert_eq!(worker, 1); assert_eq!(sequence, 1); @@ -161,7 +234,10 @@ mod tests { #[cfg(feature = "serde")] #[test] fn serialize() { - let id = Id::from_parts(UtcDateTime::from_timestamp(3, 0).unwrap(), 5, 7); - assert_eq!(serde_json::to_string(&id).unwrap(), r#""300050007""#); + let id = Id::from_worker_parts(UtcDateTime::from_timestamp(3, 0).unwrap(), 5, 7); + assert_eq!( + serde_json::to_string(&id).unwrap(), + r#""00000000000000000000000300050007""# + ); } } diff --git a/bomboni_common/src/id/generator.rs b/bomboni_common/src/id/worker.rs similarity index 68% rename from bomboni_common/src/id/generator.rs rename to bomboni_common/src/id/worker.rs index b05d20a..6319231 100644 --- a/bomboni_common/src/id/generator.rs +++ b/bomboni_common/src/id/worker.rs @@ -1,13 +1,6 @@ use std::thread; use std::time::Duration; -#[cfg(all( - target_family = "wasm", - not(any(target_os = "emscripten", target_os = "wasi")), - feature = "wasm" -))] -use wasm_bindgen::prelude::*; - #[cfg(feature = "tokio")] use parking_lot::Mutex; #[cfg(feature = "tokio")] @@ -17,28 +10,20 @@ use crate::date_time::UtcDateTime; use crate::id::Id; #[derive(Debug, Clone, Copy)] -#[cfg_attr( - all( - target_family = "wasm", - not(any(target_os = "emscripten", target_os = "wasi")), - feature = "wasm" - ), - wasm_bindgen(js_name = IdGenerator) -)] -pub struct IdGenerator { +pub struct WorkerIdGenerator { worker: u16, next: u16, } #[cfg(feature = "tokio")] #[derive(Debug, Clone)] -pub struct IdGeneratorArc(Arc>); +pub struct WorkerIdGeneratorArc(Arc>); /// Duration to sleep after overflowing the sequence number. /// Used to avoid collisions. const SLEEP_DURATION: Duration = Duration::from_secs(1); -impl IdGenerator { +impl WorkerIdGenerator { #[must_use] pub const fn new(worker: u16) -> Self { Self { next: 0, worker } @@ -51,13 +36,13 @@ impl IdGenerator { /// Basic usage: /// /// ``` - /// use bomboni_common::id::generator::IdGenerator; + /// use bomboni_common::id::worker::WorkerIdGenerator; /// - /// let mut g = IdGenerator::new(1); + /// let mut g = WorkerIdGenerator::new(1); /// assert_ne!(g.generate(), g.generate()); /// ``` pub fn generate(&mut self) -> Id { - let id = Id::from_parts(UtcDateTime::now(), self.worker, self.next); + let id = Id::from_worker_parts(UtcDateTime::now(), self.worker, self.next); self.next += 1; if self.next == u16::MAX { @@ -73,7 +58,7 @@ impl IdGenerator { /// The same as [`generate`] but async. #[cfg(feature = "tokio")] pub async fn generate_async(&mut self) -> Id { - let id = Id::from_parts(UtcDateTime::now(), self.worker, self.next); + let id = Id::from_worker_parts(UtcDateTime::now(), self.worker, self.next); self.next += 1; if self.next == u16::MAX { @@ -93,9 +78,9 @@ impl IdGenerator { /// /// ``` /// # use std::collections::HashSet; - /// use bomboni_common::id::generator::IdGenerator; + /// use bomboni_common::id::worker::WorkerIdGenerator; /// - /// let mut g = IdGenerator::new(1); + /// let mut g = WorkerIdGenerator::new(1); /// let ids = g.generate_multiple(3); /// let id_set: HashSet<_> = ids.iter().collect(); /// assert_eq!(id_set.len(), ids.len()); @@ -109,7 +94,7 @@ impl IdGenerator { let mut now = UtcDateTime::now(); for _ in 0..count { - let id = Id::from_parts(now, self.worker, self.next); + let id = Id::from_worker_parts(now, self.worker, self.next); ids.push(id); self.next += 1; @@ -136,7 +121,7 @@ impl IdGenerator { let mut now = UtcDateTime::now(); for _ in 0..count { - let id = Id::from_parts(now, self.worker, self.next); + let id = Id::from_worker_parts(now, self.worker, self.next); ids.push(id); self.next += 1; @@ -151,34 +136,16 @@ impl IdGenerator { } } -#[cfg(all( - target_family = "wasm", - not(any(target_os = "emscripten", target_os = "wasi")), - feature = "wasm", -))] -#[wasm_bindgen(js_class = IdGenerator)] -impl IdGenerator { - #[wasm_bindgen(constructor)] - pub fn wasm_new(worker: u16) -> Self { - Self { next: 0, worker } - } - - #[wasm_bindgen(js_name = generate)] - pub fn wasm_generate(&mut self) -> Id { - self.generate() - } -} - #[cfg(feature = "tokio")] const _: () = { - impl IdGeneratorArc { + impl WorkerIdGeneratorArc { pub fn new(worker: u16) -> Self { - Self(Arc::new(Mutex::new(IdGenerator::new(worker)))) + Self(Arc::new(Mutex::new(WorkerIdGenerator::new(worker)))) } } - impl Deref for IdGeneratorArc { - type Target = Mutex; + impl Deref for WorkerIdGeneratorArc { + type Target = Mutex; fn deref(&self) -> &Self::Target { &self.0 @@ -192,12 +159,12 @@ mod tests { #[test] fn it_works() { - let mut id_generator = IdGenerator::new(42); + let mut id_generator = WorkerIdGenerator::new(42); let id = id_generator.generate(); - let (_timestamp, worker, sequence) = id.decode(); + let (_timestamp, worker, sequence) = id.decode_worker(); assert_eq!(worker, 42); let id = id_generator.generate(); - assert_ne!(sequence, id.decode().2); + assert_ne!(sequence, id.decode_worker().2); } #[cfg(feature = "tokio")] @@ -206,7 +173,7 @@ mod tests { use std::collections::HashSet; const N: usize = 10; - let mut g = IdGenerator::new(1); + let mut g = WorkerIdGenerator::new(1); let mut ids = HashSet::new(); ids.extend(g.generate_multiple_async(N / 2).await); diff --git a/bomboni_proto/build.rs b/bomboni_proto/build.rs index 92c4f29..434e0e0 100644 --- a/bomboni_proto/build.rs +++ b/bomboni_proto/build.rs @@ -212,6 +212,18 @@ fn build_wasm(config: &mut Config) { ); if cfg!(feature = "js") { + config.message_attribute( + ".google.protobuf.Empty", + r#" + #[derive(bomboni_wasm::Wasm)] + #[wasm( + bomboni_crate = crate::bomboni, + wasm_abi, + js_value, + override_type = "undefined | null", + )] + "#, + ); config.message_attribute( ".google.protobuf.Timestamp", r#" @@ -225,6 +237,18 @@ fn build_wasm(config: &mut Config) { "#, ); } else { + config.message_attribute( + ".google.protobuf.Empty", + r#" + #[derive(bomboni_wasm::Wasm)] + #[wasm( + bomboni_crate = crate::bomboni, + wasm_abi, + js_value, + override_type = "null", + )] + "#, + ); config.message_attribute( ".google.protobuf.Timestamp", " diff --git a/bomboni_proto/src/protobuf/empty.rs b/bomboni_proto/src/protobuf/empty.rs index 23896bd..a7db772 100644 --- a/bomboni_proto/src/protobuf/empty.rs +++ b/bomboni_proto/src/protobuf/empty.rs @@ -32,10 +32,23 @@ impl<'de> Deserialize<'de> for Empty { feature = "wasm", ))] const _: () = { - use wasm_bindgen::prelude::*; + use wasm_bindgen::JsValue; - #[wasm_bindgen(typescript_custom_section)] - const TS_APPEND_CONTENT: &'static str = r#" - export type Empty = {}; - "#; + impl From for JsValue { + fn from(_: Empty) -> Self { + if cfg!(feature = "js") { + JsValue::undefined() + } else { + JsValue::null() + } + } + } + + impl TryFrom for Empty { + type Error = JsValue; + + fn try_from(_: JsValue) -> Result { + Ok(Self {}) + } + } }; diff --git a/bomboni_request/src/parse/mod.rs b/bomboni_request/src/parse/mod.rs index ecd7076..92b4a9e 100644 --- a/bomboni_request/src/parse/mod.rs +++ b/bomboni_request/src/parse/mod.rs @@ -2271,7 +2271,7 @@ mod tests { assert_eq!( ParsedItem::parse(Item { value: 42, - id: Some("2a".into()), + id: Some("0000000000000000000000000000002A".into()), nested: vec![1, 2, 3], }) .unwrap(), @@ -2293,7 +2293,7 @@ mod tests { }), Item { value: 42, - id: Some("2a".into()), + id: Some("0000000000000000000000000000002A".into()), nested: vec![1, 2, 3], } );